mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-18 14:14:37 +01:00
Secrets check stops flagging three code shapes and catches concatenated and commented secrets (GST-3004) (#4457)
The secrets check no longer stops a push on three kinds of ordinary code that only look like secrets: TypeScript types with spaced generic arguments, Python arguments that read a settings constant, and camelCase dotted field paths. It also catches two kinds of real secrets it used to miss, one built with `+` from short pieces and one written in a comment, so pushes are both quieter and safer. Nothing to do; a secret in any of the loosened positions still fails the check. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> Review-Finding: p1/adversarial-native#1 reader=adversarial-native Review-Finding: p1/outside-voice#1 reader=outside-voice Review-Finding: p1/outside-voice#2 reader=outside-voice
This commit is contained in:
1 parent
bc64f5e210
commit
e1cfceae6a
6 files changed
+379
-14
No files matched your search
+152
-14
@@ -1065,6 +1065,33 @@ function expressionTail(line: string, comment: string): string | null {
|
||||
return depth > 0 || /(?:[=+\\]|\?\?|\|\||&&|\b(?:or|and))\s*$/.test(rest) ? null : rest;
|
||||
}
|
||||
|
||||
/**
|
||||
* GST-3004: the value capture stops at a type's first space, so an annotation
|
||||
* with spaced generic arguments (`session: Awaited<ReturnType<typeof launch>>;`)
|
||||
* never reached the name-chain test as a closed type. Read on its line, the
|
||||
* type is names, `<>`, `[]`, `|`, `&`, `,` and `?` only; it ends at a `,`
|
||||
* `;`, `)` or `=` outside the brackets, or at the line end, and the rest of
|
||||
* the line carries no secret literal or template. A string-literal type
|
||||
* (CAPY-1's `Pick<Session, "id" | "forkedFrom">`) still reports: an entropy
|
||||
* test on the literal alone let random hex through, so that shape is a known
|
||||
* limit, allowlisted by value.
|
||||
*/
|
||||
const GENERIC_TYPE_HEAD = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*</;
|
||||
function isGenericType(line: string): boolean {
|
||||
let depth = 0;
|
||||
let i = 0;
|
||||
for (; i < line.length; i += 1) {
|
||||
const c = line[i]!;
|
||||
if (c === "<") depth += 1;
|
||||
else if (c === ">") {
|
||||
if (--depth < 0) return false;
|
||||
} else if (depth === 0 && /[,;)=]/.test(c)) break;
|
||||
else if (!/[\w$.\s|&[\],?]/.test(c)) return false;
|
||||
}
|
||||
const rest = line.slice(i);
|
||||
return depth === 0 && !carriesSecretLiteral(rest) && !rest.includes("`");
|
||||
}
|
||||
|
||||
function isSourceExpression(span: string, match: RegExpExecArray, sourcePath: string | undefined): boolean {
|
||||
if (!sourcePath || !CODE_SOURCE.test(sourcePath)) return false;
|
||||
const { start, end } = spanBounds(match);
|
||||
@@ -1078,7 +1105,8 @@ function isSourceExpression(span: string, match: RegExpExecArray, sourcePath: st
|
||||
} else {
|
||||
const head = match[0].slice(0, match[0].length - span.length).trimEnd();
|
||||
const separator = head.slice(-1);
|
||||
if (separator !== ":" || !CODE_NAME_CHAIN.test(span.replace(/[,;)=|]+$/, ""))) return false;
|
||||
if (separator !== ":") return false;
|
||||
if (!CODE_NAME_CHAIN.test(span.replace(/[,;)=|]+$/, ""))) return GENERIC_TYPE_HEAD.test(span) && isGenericType(match.input.slice(start).split("\n", 1)[0]!) && !insideString(match.input, start, sourcePath);
|
||||
const rest = (span.endsWith("=") ? "=" : "") + match.input.slice(end).split("\n", 1)[0];
|
||||
if (!/[,;]$/.test(span)) {
|
||||
const tail = expressionTail(rest, /\.pyi?$/i.test(sourcePath) ? "#" : "//");
|
||||
@@ -1096,15 +1124,22 @@ function isSourceExpression(span: string, match: RegExpExecArray, sourcePath: st
|
||||
* the same line inside a `"""`/`'''` block still reports. Every segment must be
|
||||
* name-shaped (snake_case, camelCase, PascalCase or CONSTANT_CASE), so a dotted
|
||||
* random token still reports, and only a `;` and a comment may follow it on
|
||||
* the line: a literal after it (`or "…"`, `, "…"`), a continuation or a
|
||||
* keyword argument's comma still reports. A quoted value, a bare name, a
|
||||
* the line: a literal after it (`or "…"`, `, "…"`) or a continuation still
|
||||
* reports. A keyword argument's closing `,` or `)` is allowed only when the
|
||||
* last segment is a CONSTANT_CASE setting
|
||||
* (`aws_secret_access_key=settings.S3_SECRET_KEY,`, #4289), so
|
||||
* `password=request.password,` still reports. A quoted value, a bare name, a
|
||||
* config, .env or TS file, and a scan without a path are unchanged.
|
||||
*/
|
||||
const ATTRIBUTE_SEGMENT = /^_*(?:[a-z][a-z0-9]*(?:_+[a-z0-9]+)*|[A-Z][A-Z0-9]*(?:_+[A-Z0-9]+)*|[A-Za-z][a-z0-9]*(?:[A-Z][a-z0-9]+)*)_*$/;
|
||||
const CONSTANT_SEGMENT = /^_*[A-Z][A-Z0-9]*(?:_+[A-Z0-9]+)*_*$/;
|
||||
function isAttributeReference(span: string, match: RegExpExecArray, sourcePath: string | undefined): boolean {
|
||||
if (!sourcePath || !STRING_SPAN_SOURCE.test(sourcePath)) return false;
|
||||
const chain = span.replace(/;$/, "").split(".");
|
||||
const read = span.replace(/[,;)]+$/, "");
|
||||
const chain = read.split(".");
|
||||
if (chain.length < 2 || !chain.every((segment) => ATTRIBUTE_SEGMENT.test(segment))) return false;
|
||||
const closer = span.slice(read.length);
|
||||
if (closer !== "" && closer !== ";" && !CONSTANT_SEGMENT.test(chain.at(-1)!)) return false;
|
||||
const { start, end } = spanBounds(match);
|
||||
if (!/=[ \t]*$/.test(match.input.slice(match.index, start))) return false;
|
||||
const rest = match.input.slice(end).split("\n", 1)[0];
|
||||
@@ -1112,6 +1147,93 @@ function isAttributeReference(span: string, match: RegExpExecArray, sourcePath:
|
||||
return !insideString(match.input, start, sourcePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* GST-3004: text that names or builds a key rather than holding one. Read with
|
||||
* its whitespace removed, so a password with spaces is read whole, it must be
|
||||
* short (under 8), a placeholder, under 3.0 bits, or letters-only words,
|
||||
* lowercase or camelCase with an optional leading capital, joined by
|
||||
* `_ . - | : / #` (`over_tough`, `secretsOperations`, `|over_tough|`). A
|
||||
* generated credential carries digits or adjacent capitals, so it keeps
|
||||
* reporting; a letters-only passphrase in this shape (`correctHorseBattery`)
|
||||
* is the accepted residue. Comment prose is read run by run instead
|
||||
* (isCommentText).
|
||||
*/
|
||||
const KEY_TEXT = /^[_.\-|:/#]*[A-Za-z][a-z]+(?:[A-Z][a-z]+)*(?:[_.\-|:/#]+[A-Za-z][a-z]+(?:[A-Z][a-z]+)*)*[_.\-|:/#]*$/;
|
||||
function isSecretRun(run: string): boolean {
|
||||
return run.length >= 8 && !isPlaceholderSpan(run) && shannonEntropy(run) >= 3.0 && !KEY_TEXT.test(run);
|
||||
}
|
||||
function holdsSecretText(text: string): boolean {
|
||||
return isSecretRun(text.replace(/\s+/g, ""));
|
||||
}
|
||||
|
||||
/** The text of the `"…"` or `'…'` literal whose quote is at `i` and the index after it, or null when it does not close on the line. */
|
||||
function quotedAt(line: string, i: number): { text: string; end: number } | null {
|
||||
for (let j = i + 1; j < line.length; j += line[j] === "\\" ? 2 : 1) if (line[j] === line[i]) return { text: line.slice(i + 1, j), end: j + 1 };
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* #4289: `KEY = "form.x" + "<secret>"` builds the value from pieces, and the
|
||||
* value capture read only the first. When the value is a `+` chain of quoted
|
||||
* literals and names on its line, the literals' text is joined, so a secret
|
||||
* placed after a short piece or split across pieces is whole again, and checked
|
||||
* with holdsSecretText. Names add no text. A chain that breaks off at a call,
|
||||
* template or other operand is checked as far as it reads.
|
||||
*/
|
||||
const CHAIN_NAME = /[A-Za-z_$][\w$]*(?:\??\.[A-Za-z_$][\w$]*)*/y;
|
||||
const CHAIN_PLUS = /[ \t]*\+[ \t]*/y;
|
||||
function concatenationHoldsSecret(match: RegExpExecArray): boolean {
|
||||
const { start } = spanBounds(match);
|
||||
const line = match.input.slice(/["']/.test(match.input[start - 1] ?? "") ? start - 1 : start).split("\n", 1)[0]!;
|
||||
let joined = "";
|
||||
let pieces = 0;
|
||||
for (let i = 0; ; i = CHAIN_PLUS.lastIndex) {
|
||||
const literal = line[i] === '"' || line[i] === "'" ? quotedAt(line, i) : null;
|
||||
CHAIN_NAME.lastIndex = i;
|
||||
if (!literal && !CHAIN_NAME.exec(line)) break;
|
||||
pieces += 1;
|
||||
if (literal) joined += literal.text;
|
||||
CHAIN_PLUS.lastIndex = literal ? literal.end : CHAIN_NAME.lastIndex;
|
||||
if (!CHAIN_PLUS.exec(line)) break;
|
||||
}
|
||||
return pieces > 1 && holdsSecretText(joined);
|
||||
}
|
||||
|
||||
/**
|
||||
* #4289: the regex also starts a name right after a comment marker, and
|
||||
* comments are prose: `# Auth: authentication, authorization, sessions`. A
|
||||
* match that starts after other text on its line is a comment's, and its value,
|
||||
* less trailing punctuation, reports only when it is a secret run by
|
||||
* isSecretRun. Code before the comment is matched from the line start and
|
||||
* checked as before.
|
||||
*/
|
||||
function isCommentText(span: string, match: RegExpExecArray): boolean {
|
||||
const lineStart = match.input.lastIndexOf("\n", match.index - 1) + 1;
|
||||
if (/^(?:[ +-]?\+|[ +-]?-(?! [^ \t]))?[ \t]*$/.test(match.input.slice(lineStart, match.index))) return false;
|
||||
return !isSecretRun(span.replace(/[,;.)]+$/, ""));
|
||||
}
|
||||
|
||||
/**
|
||||
* #4289: a quoted dotted path of three or more letters-only segments, each
|
||||
* lowercase or camelCase and at least one camelCase
|
||||
* (`KEY = "form.section.fieldName"`), names a form field. A segment with a
|
||||
* digit, a capital first letter or adjacent capitals (`"form.<secret>"`)
|
||||
* reports, and so do a path of one or two segments and an all-lowercase one,
|
||||
* which #4209 keeps reporting as a quoted literal (`session = "self.client.session"`).
|
||||
* Only source files qualify, and the rest of the line must carry no secret
|
||||
* literal or template, so `PASSWORD="form.section.fieldName"` in a config
|
||||
* file and `password = "form.section.fieldName" or "<secret>"` still report.
|
||||
*/
|
||||
const DOTTED_FIELD_PATH = /^[a-z]+(?:[A-Z][a-z]+)*(?:\.[a-z]+(?:[A-Z][a-z]+)*){2,}$/;
|
||||
function isDottedFieldPath(span: string, match: RegExpExecArray, sourcePath: string | undefined): boolean {
|
||||
if (!sourcePath || !CODE_SOURCE.test(sourcePath)) return false;
|
||||
const { start, end } = spanBounds(match);
|
||||
const quote = match.input[start - 1];
|
||||
if ((quote !== '"' && quote !== "'") || match.input[end] !== quote || !DOTTED_FIELD_PATH.test(span) || !/[A-Z]/.test(span)) return false;
|
||||
const rest = match.input.slice(end + 1).split("\n", 1)[0]!;
|
||||
return !carriesSecretLiteral(rest) && !rest.includes("`");
|
||||
}
|
||||
|
||||
/** The files isPropertyRead exempts in: TS, Dart and Python. JSX text in .tsx and .jsx can hold a host on any line, so those report as before. */
|
||||
const PROPERTY_READ_SOURCE = /\.(?:[cm]?ts|dart|pyi?)$/i;
|
||||
/** How far back on its line isPropertyRead reads from a span. A span further from its line start than this is never exempted. */
|
||||
@@ -1565,7 +1687,15 @@ export const PATTERNS: RedactPattern[] = [
|
||||
// and text is a Markdown bullet or YAML list item, not a marker, so those
|
||||
// read as they did before. ENV_KV_NAME and SCHEMA_FIELD_NAME take the same
|
||||
// marker so the name-shape and schema exemptions hold on a diff line.
|
||||
regex: /^(?:[ +-]?\+|[ +-]?-(?! [^ \t]))?[ \t]*(?:export[ \t]+)?["']?[A-Za-z0-9_.-]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?|DSN|AUTH|COOKIE|SESSION|PRIVATE)["']?[ \t]*[:=][ \t]*["']?([^\s'"]{8,})["']?/i,
|
||||
// GST-3004 (#4289): a name may also start right after a comment marker
|
||||
// (`#`, `//`, `/*`, `<!--`) at the line start or after a space, so a
|
||||
// secret in a trailing or whole-line comment reports; that start is a
|
||||
// bounded lookbehind (a run of spaces is matched after it, not inside it,
|
||||
// so a long run stays linear), and the code before the comment is still
|
||||
// matched on its own from the line start. A short first piece followed by
|
||||
// `+` is captured so concatenationHoldsSecret can read the rest of the
|
||||
// chain.
|
||||
regex: /(?:^(?:[ +-]?\+|[ +-]?-(?! [^ \t]))?|(?<=(?:^|[ \t])(?:#{1,6}|\/{2,3}|\/\*{1,2}|<!--)))[ \t]*(?:export[ \t]+)?["']?[A-Za-z0-9_.-]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?|DSN|AUTH|COOKIE|SESSION|PRIVATE)["']?[ \t]*[:=][ \t]*["']?([^\s'"]{8,}|[^\s'"]{1,7}(?=["']?[ \t]*\+))["']?/i,
|
||||
// Only fire on credential-shaped names with high-entropy values — kills
|
||||
// `FOO_KEY=changeme` and `cacheKey: <entropic-id>` FPs. #2912: a value
|
||||
// that is exactly an environment read (`os.environ["X"]`,
|
||||
@@ -1578,17 +1708,25 @@ export const PATTERNS: RedactPattern[] = [
|
||||
// #4209: so is a dotted attribute read assigned with = in Python or Dart (isAttributeReference).
|
||||
// #3208: so is a schema field identifier (isSchemaFieldIdentifier).
|
||||
// GST-1894: so is a self-named fixture label in a test file (isTestFixtureLabel).
|
||||
// GST-3004: so is a dotted field path (isDottedFieldPath, #4289), and so
|
||||
// is key-naming text in a comment
|
||||
// (isCommentText). A secret literal in a `+` chain reports first,
|
||||
// whatever the exemptions say about the chain's first piece
|
||||
// (concatenationHoldsSecret).
|
||||
validate: (span, match, opts) =>
|
||||
isCredentialShapedEnvName(match[0]) &&
|
||||
!isPlaceholderSpan(span) &&
|
||||
!/^\$\{?[A-Za-z_]/.test(span) &&
|
||||
!isBareEnvRead(span, match) &&
|
||||
!isCallExpression(span, match) &&
|
||||
!isSourceExpression(span, match, opts?.sourcePath) &&
|
||||
!isAttributeReference(span, match, opts?.sourcePath) &&
|
||||
!isSchemaFieldIdentifier(span, match) &&
|
||||
!isTestFixtureLabel(span, opts?.sourcePath) &&
|
||||
shannonEntropy(span) >= 3.0,
|
||||
(concatenationHoldsSecret(match) ||
|
||||
(!isPlaceholderSpan(span) &&
|
||||
!/^\$\{?[A-Za-z_]/.test(span) &&
|
||||
!isBareEnvRead(span, match) &&
|
||||
!isCallExpression(span, match) &&
|
||||
!isSourceExpression(span, match, opts?.sourcePath) &&
|
||||
!isAttributeReference(span, match, opts?.sourcePath) &&
|
||||
!isDottedFieldPath(span, match, opts?.sourcePath) &&
|
||||
!isCommentText(span, match) &&
|
||||
!isSchemaFieldIdentifier(span, match) &&
|
||||
!isTestFixtureLabel(span, opts?.sourcePath) &&
|
||||
shannonEntropy(span) >= 3.0)),
|
||||
},
|
||||
{
|
||||
id: "auth.bearer",
|
||||
|
||||
Reference in new issue
Block a user