fix: encrypt stored GitHub credentials (#797)

This commit is contained in:
Thomas Durieux
2026-09-08 13:04:28 +02:00
committed by GitHub
parent b070d7e281
commit 3fb376dd53
32 changed files with 1098 additions and 188 deletions
+2 -21
View File
@@ -1,3 +1,4 @@
import { getCredentialToken } from "./credentials";
import { RepositoryStatus } from "./types";
import User from "./User";
import UserModel from "./model/users/users.model";
@@ -46,27 +47,7 @@ export default class Gist {
}
async getToken() {
let owner = this.owner.model;
if (owner && !owner.accessTokens.github) {
const temp = await UserModel.findById(owner._id);
if (temp) {
owner = temp;
}
}
if (owner && owner.accessTokens && owner.accessTokens.github) {
if (owner.accessTokens.github != this._model.source.accessToken) {
this._model.source.accessToken = owner.accessTokens.github;
}
return owner.accessTokens.github;
}
if (this._model.source.accessToken) {
try {
return this._model.source.accessToken;
} catch {
logger.warn("invalid token", { code: "invalid_token", httpStatus: 401, gistId: this._model.source.gistId });
}
}
return config.GITHUB_TOKEN;
return (await getCredentialToken(this.owner.id, "github", { collection: "anonymizedgists", id: this._model._id })) || config.GITHUB_TOKEN;
}
async download() {
+20 -44
View File
@@ -4,7 +4,7 @@ import { createClient, RedisClientType } from "redis";
import AnonymousError from "./AnonymousError";
import Repository from "./Repository";
import UserModel from "./model/users/users.model";
import { getCredential, replaceCredential, getCredentialToken } from "./credentials";
import config from "../config";
import { createLogger } from "./logger";
@@ -271,36 +271,21 @@ export async function checkToken(token: string) {
}
}
const checkedRepositoryTokens = new WeakMap<Repository, string>();
export async function getToken(repository: Repository) {
logger.debug("getToken", { repoId: repository.repoId });
// if (repository.model.source.accessToken) {
// // only check the token if the repo has been visited less than 10 minutes ago
// if (
// repository.status == RepositoryStatus.READY &&
// repository.model.lastView > new Date(Date.now() - 1000 * 60 * 10)
// ) {
// return repository.model.source.accessToken;
// } else if (await checkToken(repository.model.source.accessToken)) {
// return repository.model.source.accessToken;
// }
// }
if (!repository.owner.model.accessTokens?.github) {
const query = await UserModel.findById(repository.owner.id, {
accessTokens: 1,
accessTokenDates: 1,
});
if (query?.accessTokens) {
repository.owner.model.accessTokens = query.accessTokens;
repository.owner.model.accessTokenDates = query.accessTokenDates;
}
}
const ownerAccessToken = repository.owner.model.accessTokens?.github;
const credential = await getCredential(repository.owner.id);
const ownerAccessToken = credential?.token;
if (ownerAccessToken) {
const tokenAge = repository.owner.model.accessTokenDates?.github;
if (checkedRepositoryTokens.get(repository) === ownerAccessToken) {
return ownerAccessToken;
}
const tokenAge = credential?.updatedAt;
// if the token is older than 7 days, refresh it
if (
!tokenAge ||
tokenAge < new Date(Date.now() - 1000 * 60 * 60 * 24 * 7)
credential?.persisted &&
(!tokenAge || tokenAge < new Date(Date.now() - 1000 * 60 * 60 * 24 * 7))
) {
const url = `https://api.github.com/applications/${config.CLIENT_ID}/token`;
const headers = {
@@ -339,23 +324,11 @@ export async function getToken(repository: Repository) {
? resBody.token
: null;
if (refreshed) {
repository.owner.model.accessTokens.github = refreshed;
if (!repository.owner.model.accessTokenDates) {
repository.owner.model.accessTokenDates = { github: new Date() };
} else {
repository.owner.model.accessTokenDates.github = new Date();
if (await replaceCredential(repository.owner.id, ownerAccessToken, refreshed)) {
checkedRepositoryTokens.set(repository, refreshed);
return refreshed;
}
await UserModel.updateOne(
{ _id: repository.owner.model._id },
{
$set: {
"accessTokens.github": refreshed,
"accessTokenDates.github":
repository.owner.model.accessTokenDates.github,
},
}
).exec();
return refreshed;
return (await getCredentialToken(repository.owner.id)) || config.GITHUB_TOKEN;
}
}
logger.warn("token refresh failed; falling back", {
@@ -367,9 +340,12 @@ export async function getToken(repository: Repository) {
}
const check = await checkToken(ownerAccessToken);
if (check) {
repository.model.source.accessToken = ownerAccessToken;
checkedRepositoryTokens.set(repository, ownerAccessToken);
return ownerAccessToken;
}
return config.GITHUB_TOKEN;
}
return config.GITHUB_TOKEN;
return (await getCredentialToken(repository.owner.id, "github", {
collection: "anonymizedrepositories", id: repository.model._id,
})) || config.GITHUB_TOKEN;
}
+2 -25
View File
@@ -1,3 +1,4 @@
import { getCredentialToken } from "./credentials";
import { RepositoryStatus } from "./types";
import User from "./User";
import UserModel from "./model/users/users.model";
@@ -24,31 +25,7 @@ export default class PullRequest {
}
async getToken() {
let owner = this.owner.model;
if (owner && !owner.accessTokens.github) {
const temp = await UserModel.findById(owner._id);
if (temp) {
owner = temp;
}
}
if (owner && owner.accessTokens && owner.accessTokens.github) {
if (owner.accessTokens.github != this._model.source.accessToken) {
this._model.source.accessToken = owner.accessTokens.github;
}
return owner.accessTokens.github;
}
if (this._model.source.accessToken) {
try {
return this._model.source.accessToken;
} catch {
logger.warn("invalid token", {
code: "invalid_token",
httpStatus: 401,
pullRequestId: this._model.source.pullRequestId,
});
}
}
return config.GITHUB_TOKEN;
return (await getCredentialToken(this.owner.id, "github", { collection: "anonymizedpullrequests", id: this._model._id })) || config.GITHUB_TOKEN;
}
async download() {
+1 -16
View File
@@ -70,23 +70,8 @@ export default class Repository {
this.owner.model.isNew = false;
}
private checkedToken: boolean = false;
async getToken() {
if (this.checkedToken) return this._model.source.accessToken as string;
const originalToken = this._model.source.accessToken;
const token = await getToken(this);
if (originalToken != token) {
this._model.source.accessToken = token;
if (isConnected) {
await AnonymizedRepositoryModel.updateOne(
{ _id: this._model._id },
{ $set: { "source.accessToken": token } }
).exec();
}
}
this.checkedToken = true;
return token;
return getToken(this);
}
get source() {
+8 -4
View File
@@ -1,3 +1,4 @@
import { getCredentialToken } from "./credentials";
import AnonymizedRepositoryModel from "./model/anonymizedRepositories/anonymizedRepositories.model";
import RepositoryModel from "./model/repositories/repositories.model";
import UserModel from "./model/users/users.model";
@@ -31,8 +32,8 @@ export default class User {
return !!this._model.isAdmin;
}
get accessToken(): string {
return this._model.accessTokens.github;
async getAccessToken(): Promise<string> {
return getCredentialToken(this.id);
}
get photo(): string | undefined {
@@ -64,7 +65,7 @@ export default class User {
opt?.force === true
) {
// get the list of repo from github
const oct = octokit(this.accessToken);
const oct = octokit(await this.getAccessToken());
const repositories = (
await oct.paginate("GET /user/repos", {
visibility: "all",
@@ -213,6 +214,9 @@ export default class User {
}
toJSON() {
return this._model.toJSON();
const value = this._model.toJSON();
delete (value as Partial<typeof value>).accessTokens;
delete value.apiTokens;
return value;
}
}
+65
View File
@@ -0,0 +1,65 @@
import { createCipheriv, createDecipheriv, randomBytes } from "crypto";
export interface EncryptedToken {
version: number;
keyId: string;
nonce: string;
ciphertext: string;
tag: string;
}
export function createTokenCipher(rawKeys: string, activeKeyId: string) {
let parsed: Record<string, unknown>;
try {
parsed = JSON.parse(rawKeys);
} catch {
throw new Error("CREDENTIAL_KEYS must be a JSON object of base64 keys");
}
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
throw new Error("CREDENTIAL_KEYS must be a JSON object");
}
const keys = new Map<string, Buffer>();
for (const [id, value] of Object.entries(parsed)) {
if (typeof value !== "string" || !id ||
Buffer.from(value, "base64").length !== 32 ||
Buffer.from(value, "base64").toString("base64") !== value) {
throw new Error("Credential keys must be canonical base64-encoded 32-byte keys");
}
keys.set(id, Buffer.from(value, "base64"));
}
if (!keys.has(activeKeyId)) throw new Error("CREDENTIAL_ACTIVE_KEY_ID is missing from CREDENTIAL_KEYS");
const aad = (ownerId: string, provider: string) =>
Buffer.from(JSON.stringify(["credentials", ownerId, provider, "encryptedToken", 1]));
return {
encrypt(token: string, ownerId: string, provider: string): EncryptedToken {
if (!token) throw new Error("Cannot encrypt an empty credential");
const nonce = randomBytes(12);
const cipher = createCipheriv("aes-256-gcm", keys.get(activeKeyId)!, nonce);
cipher.setAAD(aad(ownerId, provider));
const ciphertext = Buffer.concat([cipher.update(token, "utf8"), cipher.final()]);
return { version: 1, keyId: activeKeyId, nonce: nonce.toString("base64"),
ciphertext: ciphertext.toString("base64"), tag: cipher.getAuthTag().toString("base64") };
},
decrypt(value: EncryptedToken, ownerId: string, provider: string): string {
try {
if (!value || value.version !== 1 || !keys.has(value.keyId)) throw new Error();
const decode = (s: string) => {
if (typeof s !== "string" || !s) throw new Error();
const b = Buffer.from(s, "base64");
if (b.toString("base64") !== s) throw new Error();
return b;
};
const nonce = decode(value.nonce);
const tag = decode(value.tag);
if (nonce.length !== 12 || tag.length !== 16) throw new Error();
const decipher = createDecipheriv("aes-256-gcm", keys.get(value.keyId)!, nonce, { authTagLength: 16 });
decipher.setAAD(aad(ownerId, provider));
decipher.setAuthTag(tag);
return Buffer.concat([decipher.update(decode(value.ciphertext)), decipher.final()]).toString("utf8");
} catch {
// Never attach the token, envelope, key, or underlying crypto error.
throw new Error("Credential decryption failed");
}
},
};
}
+62
View File
@@ -0,0 +1,62 @@
import { Types } from "mongoose";
import config from "../config";
import { createTokenCipher } from "./credential-crypto";
import CredentialModel from "./model/credentials/credentials.model";
import UserModel from "./model/users/users.model";
export function credentialCipher() {
return createTokenCipher(config.CREDENTIAL_KEYS, config.CREDENTIAL_ACTIVE_KEY_ID);
}
export async function getCredential(ownerId: string, provider = "github") {
const credential = await CredentialModel.findOne({ ownerId, provider }).select("+encryptedToken").lean();
if (credential) {
return { token: credentialCipher().decrypt(credential.encryptedToken, String(credential.ownerId), provider),
updatedAt: credential.updatedAt, persisted: true };
}
if (config.CREDENTIAL_LEGACY_READS && provider === "github") {
const user = await UserModel.findById(ownerId).select("+accessTokens.github accessTokenDates");
const token = user?.accessTokens?.github;
if (token) return { token, updatedAt: user?.accessTokenDates?.github, persisted: false };
}
return null;
}
export async function getCredentialToken(ownerId: string, provider = "github", resource?: {
collection: "anonymizedrepositories" | "anonymizedgists" | "anonymizedpullrequests";
id: unknown;
}): Promise<string> {
const credential = await getCredential(ownerId, provider);
if (credential) return credential.token;
if (config.CREDENTIAL_LEGACY_READS && resource) {
const row = await CredentialModel.db.collection(resource.collection).findOne({
_id: resource.id as Types.ObjectId,
owner: new Types.ObjectId(ownerId),
}, { projection: { "source.accessToken": 1 } });
if (typeof row?.source?.accessToken === "string") return row.source.accessToken;
}
return "";
}
export async function setCredential(ownerId: string, token: string, provider = "github") {
ownerId = new Types.ObjectId(ownerId).toHexString();
const encryptedToken = credentialCipher().encrypt(token, ownerId, provider);
await CredentialModel.updateOne({ ownerId, provider }, {
$set: { encryptedToken, updatedAt: new Date() },
}, { upsert: true, runValidators: true });
}
// A refresh must not overwrite a credential replaced by a concurrent login.
export async function replaceCredential(ownerId: string, previous: string, token: string) {
const current = await CredentialModel.findOne({ ownerId, provider: "github" }).select("+encryptedToken").lean();
if (!current) {
// Legacy credentials are migrated by the migration script or next login.
return false;
}
const cipher = credentialCipher();
if (cipher.decrypt(current.encryptedToken, String(ownerId), "github") !== previous) return false;
const result = await CredentialModel.updateOne({ _id: current._id, encryptedToken: current.encryptedToken }, {
$set: { encryptedToken: cipher.encrypt(token, String(ownerId), "github"), updatedAt: new Date() },
});
return result.modifiedCount === 1;
}
+3 -1
View File
@@ -1,3 +1,4 @@
import { redactSecrets } from "./redact-secrets";
import { createClient, RedisClientType } from "redis";
import config from "../config";
@@ -251,6 +252,7 @@ function persistError(entry: {
function emit(level: Level, module: string, args: unknown[]) {
if (LEVEL_ORDER[level] < threshold) return;
args = args.map(arg => redactSecrets(arg instanceof Error ? serializeError(arg) : arg));
const ts = new Date().toISOString();
const formatted = args.map(formatArg);
const line = `${ts} ${level.toUpperCase()} [${module}] ${formatted.join(" ")}`;
@@ -341,5 +343,5 @@ export function serializeError(err: unknown): Record<string, unknown> {
// a stack for handled HTTP errors but keeps debuggability for plain Errors.
if (!out.status && !out.httpStatus && e.stack) out.stack = e.stack;
return out;
return redactSecrets(out) as Record<string, unknown>;
}
+128
View File
@@ -0,0 +1,128 @@
import { mongo } from "mongoose";
import { createTokenCipher, EncryptedToken } from "./credential-crypto";
type Cipher = ReturnType<typeof createTokenCipher>;
const resources = ["anonymizedrepositories", "anonymizedgists", "anonymizedpullrequests"];
const legacyQuery = { $or: [{ "source.accessToken": { $exists: true } }, { accessToken: { $exists: true } }] };
export interface MigrationOptions {
apply?: boolean;
removeLegacy?: boolean;
preferOwnerToken?: boolean;
batchSize?: number;
report?: (event: { collection: string; id: string; issue: string }) => void;
}
/** Run with all application writers stopped. Reruns never replace an existing credential. */
export async function migrateCredentials(db: mongo.Db, cipher: Cipher, options: MigrationOptions = {}) {
const credentials = db.collection("credentials");
const users = db.collection("users");
const batchSize = options.batchSize || 100;
const counts = { owners: 0, created: 0, removed: 0, issues: 0 };
const issue = (collection: string, id: unknown, reason: string) => {
counts.issues++;
options.report?.({ collection, id: String(id), issue: reason });
};
if (options.apply) await credentials.createIndex({ ownerId: 1, provider: 1 }, { unique: true });
for await (const user of users.find({}, { projection: { accessTokens: 1, accessTokenDates: 1, status: 1 } }).batchSize(batchSize)) {
counts.owners++;
const ownerId = user._id;
const existing = await credentials.findOne({ ownerId, provider: "github" });
let selected: string | undefined;
let valid = true;
if (existing) {
try { selected = cipher.decrypt(existing.encryptedToken as EncryptedToken, String(ownerId), "github"); }
catch { issue("credentials", existing._id, "decryption_failed"); continue; }
}
const ownerToken = user.accessTokens?.github;
const authoritative = !!(selected || (typeof ownerToken === "string" && ownerToken));
const inspect = (value: unknown, collection: string, id: unknown) => {
if (value === undefined || value === null || value === "") return;
if (typeof value !== "string") {
issue(collection, id, "malformed_token"); valid = false; return;
}
if (!selected) selected = value;
else if (selected !== value && !(options.preferOwnerToken && authoritative)) {
issue(collection, id, "conflicting_token"); valid = false;
}
};
inspect(ownerToken, "users", ownerId);
for (const name of resources) {
for await (const row of db.collection(name).find({ owner: ownerId, ...legacyQuery }, {
projection: { source: 1, accessToken: 1 },
}).batchSize(batchSize)) {
inspect(row.source?.accessToken, name, row._id);
inspect(row.accessToken, name, row._id);
}
}
if (!valid) continue;
// Removed accounts must never regain credentials during backfill.
if (user.status === "removed") {
if (options.apply && options.removeLegacy) await credentials.deleteMany({ ownerId });
} else if (selected && !existing) {
const encryptedToken = cipher.encrypt(selected, String(ownerId), "github");
if (cipher.decrypt(encryptedToken, String(ownerId), "github") !== selected) throw new Error("Credential verification failed");
if (options.apply) {
await credentials.updateOne({ ownerId, provider: "github" }, { $setOnInsert: {
encryptedToken, updatedAt: user.accessTokenDates?.github || new Date(),
} }, { upsert: true });
const stored = await credentials.findOne({ ownerId, provider: "github" });
if (!stored || cipher.decrypt(stored.encryptedToken as EncryptedToken, String(ownerId), "github") !== selected) {
issue("users", ownerId, "credential_changed_retry"); continue;
}
}
counts.created++;
}
if (options.apply && options.removeLegacy) {
// The command requires maintenance mode: no login, refresh, deletion, or resource writes.
const result = await users.updateOne({ _id: ownerId }, { $unset: { accessTokens: "", accessTokenDates: "" } });
counts.removed += result.modifiedCount;
for (const name of resources) {
const result = await db.collection(name).updateMany({ owner: ownerId, ...legacyQuery }, {
$unset: { "source.accessToken": "", accessToken: "" },
});
counts.removed += result.modifiedCount;
}
}
}
// Credentials attached to missing owners cannot be assigned safely.
for (const name of resources) {
for await (const row of db.collection(name).find(legacyQuery, { projection: { owner: 1 } }).batchSize(batchSize)) {
if (!row.owner || !(await users.findOne({ _id: row.owner }, { projection: { _id: 1 } }))) {
issue(name, row._id, "missing_owner");
}
}
}
return counts;
}
export async function verifyCredentials(db: mongo.Db, cipher: Cipher) {
let checked = 0;
for await (const row of db.collection("credentials").find({})) {
cipher.decrypt(row.encryptedToken as EncryptedToken, String(row.ownerId), row.provider);
if (!(await db.collection("users").findOne({ _id: row.ownerId, status: { $ne: "removed" } }))) {
throw new Error("Credential has no active owner");
}
checked++;
}
let legacy = await db.collection("users").countDocuments({ accessTokens: { $exists: true } });
for (const name of resources) legacy += await db.collection(name).countDocuments(legacyQuery);
return { checked, legacy };
}
/** Preserve existing validators while forbidding legacy credential storage. */
export async function enforceCredentialStorage(db: mongo.Db, cipher: Cipher) {
const verification = await verifyCredentials(db, cipher);
if (verification.legacy) throw new Error("Legacy credentials remain");
for (const name of ["users", ...resources]) {
const info = await db.listCollections({ name }).next();
if (!info) await db.createCollection(name);
const previous = info && "options" in info ? info.options?.validator : undefined;
const absent = name === "users" ? { accessTokens: { $exists: false } } : {
"source.accessToken": { $exists: false }, accessToken: { $exists: false },
};
await db.command({ collMod: name, validator: previous && Object.keys(previous).length ? {
$and: [previous, absent],
} : absent, validationLevel: "strict", validationAction: "error" });
}
return verification;
}
@@ -18,7 +18,7 @@ const AnonymizedGistSchema = new Schema({
conference: String,
source: {
gistId: String,
accessToken: String,
accessToken: { type: String, select: false },
},
options: {
terms: [String],
@@ -19,7 +19,7 @@ const AnonymizedPullRequestSchema = new Schema({
source: {
pullRequestId: Number,
repositoryFullName: String,
accessToken: String,
accessToken: { type: String, select: false },
},
options: {
terms: [String],
@@ -14,7 +14,7 @@ const AnonymizedRepositorySchema = new Schema({
anonymizeDate: Date,
lastView: Date,
pageView: Number,
accessToken: String,
accessToken: { type: String, select: false },
owner: {
type: Schema.Types.ObjectId,
ref: "user",
@@ -36,7 +36,7 @@ const AnonymizedRepositorySchema = new Schema({
commitDate: Date,
repositoryId: String,
repositoryName: String,
accessToken: String,
accessToken: { type: String, select: false },
},
truncatedFolders: {
type: [String],
@@ -0,0 +1,24 @@
import { model, Schema, Types } from "mongoose";
import { EncryptedToken } from "../../credential-crypto";
export interface ICredential {
ownerId: Types.ObjectId;
provider: string;
encryptedToken: EncryptedToken;
updatedAt: Date;
}
const envelope = new Schema({
version: { type: Number, required: true, enum: [1] },
keyId: { type: String, required: true },
nonce: { type: String, required: true },
ciphertext: { type: String, required: true },
tag: { type: String, required: true },
}, { _id: false });
const schema = new Schema<ICredential>({
ownerId: { type: Schema.Types.ObjectId, required: true, ref: "user" },
provider: { type: String, required: true, enum: ["github"] },
encryptedToken: { type: envelope, required: true, select: false },
updatedAt: { type: Date, required: true },
}, { collection: "credentials" });
schema.index({ ownerId: 1, provider: 1 }, { unique: true });
export default model<ICredential>("Credential", schema);
+1 -1
View File
@@ -2,7 +2,7 @@ import { Schema } from "mongoose";
const UserSchema = new Schema({
accessTokens: {
github: { type: String },
github: { type: String, select: false },
},
accessTokenDates: {
github: { type: Date },
+2 -1
View File
@@ -1,7 +1,8 @@
import { Document, Model } from "mongoose";
export interface IUser {
accessTokens: {
/** Legacy storage, read only when CREDENTIAL_LEGACY_READS is enabled. */
accessTokens?: {
github: string;
};
accessTokenDates?: {
+16
View File
@@ -0,0 +1,16 @@
const sensitive = /^(?:authorization|proxy-authorization|cookie|set-cookie|token|access_?tokens?|refresh_?token|encryptedToken|ciphertext|nonce|tag|password|client_?secret|CREDENTIAL_KEYS)$/i;
export function redactSecrets(value: unknown, seen = new WeakSet<object>()): unknown {
if (typeof value === "string") return value
.replace(/\b(?:gh[pousr]_[A-Za-z0-9_]+|github_pat_[A-Za-z0-9_]+)\b/g, "[REDACTED]")
.replace(/((?:access_token|refresh_token|token)=)[^&\s]+/gi, "$1[REDACTED]")
.replace(/\b(Bearer|Basic)\s+[A-Za-z0-9+/=._-]+/gi, "$1 [REDACTED]");
if (!value || typeof value !== "object" || value instanceof Date) return value;
if (seen.has(value)) return "[Circular]";
seen.add(value);
if (Array.isArray(value)) return value.map(item => redactSecrets(item, seen));
const result: Record<string, unknown> = {};
for (const [key, item] of Object.entries(value)) {
result[key] = sensitive.test(key) ? "[REDACTED]" : redactSecrets(item, seen);
}
return result;
}