mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-07 09:23:50 +01:00
Merge origin/main into feat/design-lint; stamp 1.91.251.0
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
commit
4b4cadcb9c
289 files changed
+16151
-2303
No files matched your search
+30
-8
@@ -2,8 +2,9 @@
|
||||
* changelog-check — CHANGELOG re-heading and checks for gstack-restamp (C2)
|
||||
* and the tree receipt (C3). A restamp moves the PR's section (`[Unreleased]`
|
||||
* or the version currently stamped on the branch) to the top under the new
|
||||
* version, above its predecessor's entry. Duplicate version headings and a
|
||||
* top entry that is not VERSION are errors. The tree receipt uses
|
||||
* version, above its predecessor's entry. Duplicate version headings beyond
|
||||
* those the base branch already carries, and a top entry that is not VERSION,
|
||||
* are errors. The tree receipt uses
|
||||
* `normalizeChangelog` to accept a heading move as a stamp, never a change.
|
||||
*/
|
||||
import type { ResultCodeName } from './result-codes';
|
||||
@@ -44,15 +45,36 @@ export function parseChangelog(text: string): ParsedChangelog {
|
||||
return { header, sections };
|
||||
}
|
||||
|
||||
/** Duplicate headings, and (when `expectedTop` is given) the top entry equals VERSION. */
|
||||
export function checkChangelog(text: string, expectedTop: string | null, file = 'CHANGELOG.md'): ChangelogError[] {
|
||||
/**
|
||||
* Duplicate headings, and (when `expectedTop` is given) the top entry equals
|
||||
* VERSION. `base` holds readings of the base branch's CHANGELOG (the base tip,
|
||||
* and the commit the PR's own lines are counted from): a heading it already carries N
|
||||
* times may appear N times here (history the PR inherited and may not
|
||||
* rewrite); only an occurrence beyond that count is a duplicate the PR
|
||||
* introduced. The count is per heading, not per section body: an inherited
|
||||
* copy the PR edits or replaces is caught by the train's released-history
|
||||
* checks (`ownEntryAboveMain`, `changelogDamage`), not here.
|
||||
*/
|
||||
export function checkChangelog(text: string, expectedTop: string | null, file = 'CHANGELOG.md', base: string[] = []): ChangelogError[] {
|
||||
const errors: ChangelogError[] = [];
|
||||
const parsed = parseChangelog(text);
|
||||
const seen = new Map<string, number>();
|
||||
const key = (s: ChangelogSection) => (s.unreleased ? 'Unreleased' : s.version!);
|
||||
const inherited = new Map<string, number>();
|
||||
for (const reading of base) {
|
||||
const counts = new Map<string, number>();
|
||||
for (const s of parseChangelog(reading).sections) counts.set(key(s), (counts.get(key(s)) ?? 0) + 1);
|
||||
for (const [k, n] of counts) inherited.set(k, Math.max(inherited.get(k) ?? 0, n));
|
||||
}
|
||||
const seen = new Map<string, number[]>();
|
||||
for (const s of parsed.sections) {
|
||||
const key = s.unreleased ? 'Unreleased' : s.version!;
|
||||
if (seen.has(key)) errors.push({ code: 'CHANGELOG_DUPLICATE_HEADING', path: `${file}:${s.start + 1}`, message: `${key} already headed at line ${seen.get(key)! + 1}` });
|
||||
else seen.set(key, s.start);
|
||||
const k = key(s);
|
||||
const starts = seen.get(k) ?? [];
|
||||
const allowed = Math.max(1, inherited.get(k) ?? 0);
|
||||
if (starts.length >= allowed) {
|
||||
const where = allowed > 1 ? `headed ${starts.length + 1} times (the base carries it ${allowed} times; first at line ${starts[0]! + 1})` : `already headed at line ${starts[0]! + 1}`;
|
||||
errors.push({ code: 'CHANGELOG_DUPLICATE_HEADING', path: `${file}:${s.start + 1}`, message: `${k} ${where}` });
|
||||
}
|
||||
seen.set(k, [...starts, s.start]);
|
||||
}
|
||||
if (expectedTop) {
|
||||
const top = parsed.sections[0];
|
||||
|
||||
+518
-121
@@ -18,13 +18,16 @@
|
||||
* Token figures come from one of two sources, always named in the output:
|
||||
* ESTIMATE (default, offline) bytes / TOKEN_DIVISOR, calibrated against real
|
||||
* count_tokens measurements.
|
||||
* EXACT (--exact, opt-in) Anthropic's count_tokens for every file the
|
||||
* bill touches. Sends file content off-machine,
|
||||
* so it is never implicit: an egress receipt is
|
||||
* written before the POSTs (sink
|
||||
* 'context-bill-exact'), and if the receipt
|
||||
* cannot be written the run degrades to the
|
||||
* offline estimate with a warning instead of
|
||||
* EXACT (--exact, opt-in) Anthropic's count_tokens for every billed .md
|
||||
* file inside the tree, the linked gstack
|
||||
* checkout or $GSTACK_STATE_ROOT/render; anything
|
||||
* else a symlink reaches is named and estimated,
|
||||
* and the bill is labelled mixed. Sends file
|
||||
* content off-machine, so it is never implicit:
|
||||
* an egress receipt is written before the POSTs
|
||||
* (sink 'context-bill-exact'), and if the
|
||||
* receipt cannot be written the run degrades to
|
||||
* the offline estimate with a warning instead of
|
||||
* sending unrecorded.
|
||||
* Both bytes and tokens are always shown, and the estimate's measured error
|
||||
* band is printed with it. The tool never writes state anywhere (the egress
|
||||
@@ -34,6 +37,7 @@ import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { writeReceipt } from "./egress-receipt";
|
||||
import { resolveStateRoot } from "./state-root";
|
||||
|
||||
const FORCED_PHRASE = "for every invocation";
|
||||
// Backticked reference in prose. `<...>` is excluded: a path template such as
|
||||
@@ -43,6 +47,17 @@ const PROSE_REF = /`(references\/[^`<>]+\.md)`/g;
|
||||
const ROUTER_KEYS = new Set(["name", "description", "version", "allowed-tools", "triggers", "preamble-tier"]);
|
||||
// Skill-shaped files other hosts drop into scanner scope.
|
||||
const FOREIGN_SKILL_FILE = /^(skill\.(ya?ml|json)|agents?\.md|\.cursorrules|\.windsurfrules)$/i;
|
||||
/** Directory entries one walkMdFiles or findSkillDirs call may examine before it refuses the tree. */
|
||||
const WALK_ENTRY_CAP = 200_000;
|
||||
/** tokenSource prefix of a bill that --exact priced partly by estimate. */
|
||||
const MIXED_PREFIX = "mixed: ";
|
||||
|
||||
function entryCapError(dir: string, cap: number): Error {
|
||||
return new Error(
|
||||
`${dir} has more than ${cap} entries to walk; a symlink in it probably points at a large tree. ` +
|
||||
`Nothing was billed or sent. Remove or narrow that link and run again.`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bytes per token, per content class, fitted to real count_tokens measurements.
|
||||
@@ -175,9 +190,15 @@ function sumTokens(entries: { tokens: number }[]): number {
|
||||
return entries.reduce((n, e) => n + e.tokens, 0);
|
||||
}
|
||||
|
||||
/** Cache key for a SKILL.md's frontmatter block, which is a slice, not a whole file. */
|
||||
function frontmatterKey(skillMdPath: string): string {
|
||||
return `${skillMdPath}#frontmatter`;
|
||||
/**
|
||||
* Cache key for a .md file's frontmatter block, which is a slice, not a whole
|
||||
* file. The bill looks it up through a SKILL.md path; --exact measures it once
|
||||
* per real SKILL.md the bill prices, keyed by the first path the send set
|
||||
* reached that file by, so a SKILL.md first walked under another name still
|
||||
* has one.
|
||||
*/
|
||||
function frontmatterKey(mdPath: string): string {
|
||||
return `${mdPath}#frontmatter`;
|
||||
}
|
||||
|
||||
function parseFrontmatter(text: string): { bytes: number; keys: string[]; block: string } {
|
||||
@@ -195,55 +216,197 @@ function parseFrontmatter(text: string): { bytes: number; keys: string[]; block:
|
||||
return { bytes: Buffer.byteLength(block, "utf8"), keys, block };
|
||||
}
|
||||
|
||||
/**
|
||||
* Every .md file under a tree, for the on-disk total and for exact
|
||||
* measurement. Skips node_modules and dot-directories: a skills tree that is
|
||||
* also a repo checkout (dev symlink installs) would otherwise bill its
|
||||
* dependency tree and CI state as skill content.
|
||||
*/
|
||||
export function walkMd(dir: string): string[] {
|
||||
const out: string[] = [];
|
||||
let entries: fs.Dirent[];
|
||||
/** Whether a path is a regular file once symlinks are followed; a dangling link is not. */
|
||||
function isFile(p: string): boolean {
|
||||
try {
|
||||
entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
return fs.statSync(p).isFile();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What a directory entry is once symlinks are followed. Installed skills are
|
||||
* symlinks into the gstack checkout (`review/SKILL.md -> <checkout>/review/SKILL.md`,
|
||||
* `office-hours/sections -> <checkout>/office-hours/sections`), so a plain
|
||||
* Dirent type check sees none of them. Dangling links and anything that is
|
||||
* neither a file nor a directory are null.
|
||||
*/
|
||||
function entryKind(p: string, e: fs.Dirent): "file" | "dir" | null {
|
||||
if (e.isFile()) return "file";
|
||||
if (e.isDirectory()) return "dir";
|
||||
if (!e.isSymbolicLink()) return null;
|
||||
try {
|
||||
const st = fs.statSync(p);
|
||||
return st.isFile() ? "file" : st.isDirectory() ? "dir" : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The realpath of `p`, resolved one component at a time the way the walk
|
||||
* reads directories, so a lexical path that crosses more links than the
|
||||
* kernel's limit still resolves.
|
||||
*/
|
||||
function realpathStepwise(p: string): string {
|
||||
const abs = path.resolve(p);
|
||||
const { root } = path.parse(abs);
|
||||
let real = fs.realpathSync(root);
|
||||
for (const segment of abs.slice(root.length).split(path.sep)) {
|
||||
if (segment) real = fs.realpathSync(path.join(real, segment));
|
||||
}
|
||||
return real;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether directory `outer` contains (or is) `inner`, both realpaths. A
|
||||
* followed directory symlink that contains the tree being walked (`..`, `/`,
|
||||
* `$HOME`) would climb out of that tree, so walkers refuse it.
|
||||
*/
|
||||
function encloses(outer: string, inner: string): boolean {
|
||||
const rel = path.relative(outer, inner);
|
||||
return rel === "" || (rel !== ".." && !rel.startsWith(`..${path.sep}`) && !path.isAbsolute(rel));
|
||||
}
|
||||
|
||||
/** walkMdFiles(dir)'s lexical paths, with no billedRoot fence or nested-skill skip. Only tests call it; the bill and --exact use walkMdFiles. */
|
||||
export function walkMd(dir: string): string[] {
|
||||
return walkMdFiles(dir).map((f) => f.path);
|
||||
}
|
||||
|
||||
export interface WalkedFile {
|
||||
/** The lexical path under the walked dir; the key the bill and --exact use. */
|
||||
path: string;
|
||||
/** The file's realpath when it was walked. */
|
||||
real: string;
|
||||
/** The file's identity, taken without following links when it was walked; --exact reads only that same file. */
|
||||
dev: number;
|
||||
ino: number;
|
||||
/** Its size and mtime when it was walked; a measurement is reused only for a file that still has them. */
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every .md file under a tree, with its realpath and identity, for the
|
||||
* on-disk total and for exact measurement. Skips node_modules and
|
||||
* dot-directories: a skills tree that is also a repo checkout (dev symlink
|
||||
* installs) would otherwise bill its dependency tree and CI state as skill
|
||||
* content. A child repo checkout (the
|
||||
* `gstack/` link inside ~/.claude/skills) is skipped for the same reason.
|
||||
*
|
||||
* Symlinked files and directories are followed, and every returned path stays
|
||||
* the lexical path under `dir`, so buildBill and --exact key the same files.
|
||||
* Each real directory is expanded once and each real file returned once per
|
||||
* walk, under the first lexical path reached with entries in sorted order: a
|
||||
* skill loads a file once however many links reach it, and links that fan out
|
||||
* into each other cannot multiply the walk. A directory that encloses `dir`,
|
||||
* or `billedRoot` (the tree a skill is billed under), is never entered. A link
|
||||
* named *.md counts only when its target is a .md file too, so --exact never
|
||||
* sends a non-markdown file through a markdown-named link. More than
|
||||
* WALK_ENTRY_CAP entries is an error, never a silent cut.
|
||||
*
|
||||
* With `skipNestedSkills`, a top-level subdirectory holding its own SKILL.md is
|
||||
* not entered, so a container skill keeps the files it shares with a nested
|
||||
* skill instead of losing them to whichever path sorted first. The root
|
||||
* container also skips a nested skill reached through a directory link at any
|
||||
* depth: discovery bills that skill on its own, and the plain walk this
|
||||
* replaced never followed directory links, so counting it here too would bill
|
||||
* it twice.
|
||||
*
|
||||
* Every directory is read through its realpath, so a lookup crosses at most
|
||||
* one link however deep the lexical path runs: a chain of links never hits
|
||||
* the kernel's link-depth limit (ELOOP) and silently loses the files past it.
|
||||
*/
|
||||
export function walkMdFiles(dir: string, opts: { billedRoot?: string; skipNestedSkills?: boolean } = {}): WalkedFile[] {
|
||||
const out: WalkedFile[] = [];
|
||||
let rootReal: string;
|
||||
let billedReal: string;
|
||||
try {
|
||||
rootReal = fs.realpathSync(dir);
|
||||
billedReal = fs.realpathSync(opts.billedRoot ?? dir);
|
||||
} catch {
|
||||
return out;
|
||||
}
|
||||
for (const e of entries) {
|
||||
if (e.name.startsWith(".") || e.name === "node_modules") continue;
|
||||
const p = path.join(dir, e.name);
|
||||
if (e.isDirectory()) out.push(...walkMd(p));
|
||||
else if (e.isFile() && e.name.endsWith(".md")) out.push(p);
|
||||
}
|
||||
const seen = new Set<string>([rootReal]);
|
||||
const container = rootReal === billedReal;
|
||||
let entryCount = 0;
|
||||
const walk = (current: string, currentReal: string, viaLink: boolean) => {
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = fs.readdirSync(currentReal, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
||||
for (const e of entries) {
|
||||
if (++entryCount > WALK_ENTRY_CAP) throw entryCapError(dir, WALK_ENTRY_CAP);
|
||||
if (e.name.startsWith(".") || e.name === "node_modules") continue;
|
||||
const viaReal = path.join(currentReal, e.name);
|
||||
const kind = entryKind(viaReal, e);
|
||||
if (kind === "file") {
|
||||
if (!e.name.endsWith(".md")) continue;
|
||||
} else if (
|
||||
kind !== "dir" ||
|
||||
fs.existsSync(path.join(viaReal, ".git")) ||
|
||||
(opts.skipNestedSkills &&
|
||||
(current === dir || (container && (viaLink || e.isSymbolicLink()))) &&
|
||||
isFile(path.join(viaReal, "SKILL.md")))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
let real: string;
|
||||
try {
|
||||
real = fs.realpathSync(viaReal);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (seen.has(real)) continue;
|
||||
const p = path.join(current, e.name);
|
||||
if (kind === "file") {
|
||||
if (!real.endsWith(".md")) continue;
|
||||
let st: fs.Stats;
|
||||
try {
|
||||
st = fs.lstatSync(real);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!st.isFile()) continue;
|
||||
seen.add(real);
|
||||
out.push({ path: p, real, dev: st.dev, ino: st.ino, size: st.size, mtimeMs: st.mtimeMs });
|
||||
continue;
|
||||
}
|
||||
if (encloses(real, rootReal) || encloses(real, billedReal)) continue;
|
||||
seen.add(real);
|
||||
walk(p, real, viaLink || e.isSymbolicLink());
|
||||
}
|
||||
};
|
||||
walk(dir, rootReal, false);
|
||||
return out;
|
||||
}
|
||||
|
||||
function totalMd(dir: string, tokensOf: TokensOf): { bytes: number; tokens: number } {
|
||||
function totalMd(dir: string, tokensOf: TokensOf, billedRoot: string): { bytes: number; tokens: number } {
|
||||
let bytes = 0;
|
||||
let tokens = 0;
|
||||
for (const p of walkMd(dir)) {
|
||||
// A skill dir that CONTAINS other skill dirs (the gstack root skill wraps
|
||||
// the whole tree) must not swallow its children's files: each nested
|
||||
// skill reports its own totalMd, and the grand total sums per-skill
|
||||
// figures — counting them here again double-counted every nested skill
|
||||
// in the TOTAL line (v1.63 deferred polish, fixed in fork port wave 2).
|
||||
const rel = path.relative(dir, p);
|
||||
const topSeg = rel.split(path.sep)[0];
|
||||
if (
|
||||
topSeg &&
|
||||
topSeg !== rel && // p is inside a subdirectory
|
||||
fs.existsSync(path.join(dir, topSeg, "SKILL.md"))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const b = bytesOf(p) ?? 0;
|
||||
// A skill dir that CONTAINS other skill dirs (the gstack root skill wraps
|
||||
// the whole tree) must not swallow its children's files: each nested
|
||||
// skill reports its own totalMd, and the grand total sums per-skill
|
||||
// figures — counting them here again double-counted every nested skill
|
||||
// in the TOTAL line (v1.63 deferred polish, fixed in fork port wave 2).
|
||||
for (const { path: p, real } of walkMdFiles(dir, { billedRoot, skipNestedSkills: true })) {
|
||||
const b = bytesOf(real) ?? 0;
|
||||
bytes += b;
|
||||
tokens += tokensOf(p, b);
|
||||
}
|
||||
return { bytes, tokens };
|
||||
}
|
||||
|
||||
export function parseSkill(skillDir: string, name: string, tokensOf: TokensOf = estimateTokensOf): SkillBill {
|
||||
export function parseSkill(
|
||||
skillDir: string,
|
||||
name: string,
|
||||
tokensOf: TokensOf = estimateTokensOf,
|
||||
billedRoot: string = skillDir,
|
||||
): SkillBill {
|
||||
const skillMdPath = path.join(skillDir, "SKILL.md");
|
||||
const text = fs.readFileSync(skillMdPath, "utf8");
|
||||
const skillMdBytes = bytesOf(skillMdPath) ?? 0;
|
||||
@@ -275,14 +438,14 @@ export function parseSkill(skillDir: string, name: string, tokensOf: TokensOf =
|
||||
// Foreign-host skill files sitting next to SKILL.md.
|
||||
const foreignFiles: { path: string; bytes: number; tokens: number }[] = [];
|
||||
for (const entry of fs.readdirSync(skillDir, { withFileTypes: true })) {
|
||||
if (entry.isFile() && FOREIGN_SKILL_FILE.test(entry.name)) {
|
||||
const abs = path.join(skillDir, entry.name);
|
||||
const abs = path.join(skillDir, entry.name);
|
||||
if (FOREIGN_SKILL_FILE.test(entry.name) && entryKind(abs, entry) === "file") {
|
||||
const bytes = bytesOf(abs) ?? 0;
|
||||
foreignFiles.push({ path: entry.name, bytes, tokens: tokensOf(abs, bytes) });
|
||||
}
|
||||
}
|
||||
|
||||
const total = totalMd(skillDir, tokensOf);
|
||||
const total = totalMd(skillDir, tokensOf, billedRoot);
|
||||
const eagerBytes = skillMdBytes + sumBytes(forcedRefs);
|
||||
const eagerTokens = skillMdTokens + sumTokens(forcedRefs);
|
||||
return {
|
||||
@@ -333,52 +496,59 @@ export function parseSkill(skillDir: string, name: string, tokensOf: TokensOf =
|
||||
* repo checkout (`gstack/`, with .git). Its nested SKILL.md files are the
|
||||
* repo's sources, not installed skills of the tree being billed.
|
||||
*
|
||||
* Directory symlinks are followed (setup's shell glob follows them, so a
|
||||
* symlinked skill like connect-chrome/ is real scanner load); a realpath
|
||||
* seen-set breaks cycles.
|
||||
* Symlinks are followed. Directory links: setup's shell glob follows them, so
|
||||
* a symlinked skill like connect-chrome/ is real scanner load. File links:
|
||||
* setup installs every skill as a real directory whose SKILL.md is a symlink
|
||||
* into the checkout, so skipping them billed a real install as two skills.
|
||||
* A realpath seen-set breaks cycles, a link to a directory that encloses the
|
||||
* root is never followed, and every returned path stays under the root.
|
||||
*/
|
||||
export function findSkillDirs(root: string): string[] {
|
||||
export function findSkillDirs(root: string, entryCap: number = WALK_ENTRY_CAP): string[] {
|
||||
const out: string[] = [];
|
||||
const visited = new Set<string>();
|
||||
const walk = (dir: string, isRoot: boolean) => {
|
||||
const rootDir = path.resolve(root);
|
||||
let entryCount = 0;
|
||||
let rootReal: string;
|
||||
try {
|
||||
rootReal = fs.realpathSync(rootDir);
|
||||
} catch {
|
||||
return out;
|
||||
}
|
||||
// Each directory is read through its realpath (see walkMdFiles), so a deep
|
||||
// chain of links cannot hit the kernel's link limit and drop a skill.
|
||||
const walk = (dir: string, real: string, isRoot: boolean) => {
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
entries = fs.readdirSync(real, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (entries.some((e) => e.isFile() && e.name === "SKILL.md")) {
|
||||
if (entries.some((e) => e.name === "SKILL.md" && entryKind(path.join(real, e.name), e) === "file")) {
|
||||
out.push(dir);
|
||||
// Two symlinked paths to the same skill dir are BOTH billed (each is
|
||||
// real scanner load); only container recursion below is cycle-guarded.
|
||||
if (!isRoot) return;
|
||||
}
|
||||
// Cycle guard for container recursion (a symlink loop of directories).
|
||||
let real: string;
|
||||
try {
|
||||
real = fs.realpathSync(dir);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (visited.has(real)) return;
|
||||
visited.add(real);
|
||||
for (const e of entries) {
|
||||
if (++entryCount > entryCap) throw entryCapError(rootDir, entryCap);
|
||||
if (e.name.startsWith(".") || e.name === "node_modules") continue;
|
||||
const child = path.join(dir, e.name);
|
||||
let isDir = e.isDirectory();
|
||||
if (!isDir && e.isSymbolicLink()) {
|
||||
try {
|
||||
isDir = fs.statSync(child).isDirectory();
|
||||
} catch {
|
||||
continue; // dangling symlink
|
||||
}
|
||||
const viaReal = path.join(real, e.name);
|
||||
if (entryKind(viaReal, e) !== "dir") continue;
|
||||
if (fs.existsSync(path.join(viaReal, ".git"))) continue; // repo checkout, not a skill
|
||||
let childReal: string;
|
||||
try {
|
||||
childReal = fs.realpathSync(viaReal);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!isDir) continue;
|
||||
if (fs.existsSync(path.join(child, ".git"))) continue; // repo checkout, not a skill
|
||||
walk(child, false);
|
||||
if (encloses(childReal, rootReal)) continue; // would climb out of the root
|
||||
walk(path.join(dir, e.name), childReal, false);
|
||||
}
|
||||
};
|
||||
walk(path.resolve(root), true);
|
||||
walk(rootDir, rootReal, true);
|
||||
return out.sort();
|
||||
}
|
||||
|
||||
@@ -404,25 +574,32 @@ export interface Bill {
|
||||
|
||||
export function buildBill(
|
||||
root: string,
|
||||
{ tokensOf = estimateTokensOf, tokenSource, calibration }: {
|
||||
{ tokensOf = estimateTokensOf, tokenSource, calibration, missedKeys }: {
|
||||
tokensOf?: TokensOf;
|
||||
tokenSource?: string;
|
||||
calibration?: Calibration;
|
||||
/** Keys a measured tokensOf had to estimate; filled while the bill is priced. */
|
||||
missedKeys?: ReadonlySet<string>;
|
||||
} = {},
|
||||
): Bill {
|
||||
const resolved = path.resolve(root);
|
||||
if (!fs.existsSync(resolved)) throw new Error(`No such tree: ${resolved}`);
|
||||
const skills = findSkillDirs(resolved).map((dir) =>
|
||||
parseSkill(dir, path.relative(resolved, dir) || path.basename(resolved), tokensOf),
|
||||
parseSkill(dir, path.relative(resolved, dir) || path.basename(resolved), tokensOf, resolved),
|
||||
);
|
||||
const total = skills.reduce((n, s) => n + s.totalMdBytes, 0);
|
||||
const totalTokens = skills.reduce((n, s) => n + s.totalMdTokens, 0);
|
||||
const estimated = tokenSource ? (missedKeys?.size ?? 0) : 0;
|
||||
return {
|
||||
root: resolved,
|
||||
// Named so a reader never has to guess whether a figure was measured.
|
||||
tokenSource: tokenSource ?? "estimate: calibrated bytes/token per content class",
|
||||
tokenSource: !tokenSource
|
||||
? "estimate: calibrated bytes/token per content class"
|
||||
: estimated
|
||||
? `${MIXED_PREFIX}${tokenSource}, with ${estimated} item(s) estimated offline`
|
||||
: tokenSource,
|
||||
tokenEstimate: TOKEN_DIVISORS,
|
||||
tokenEstimateErrorPct: tokenSource ? 0 : TOKEN_ESTIMATE_ERROR_PCT,
|
||||
tokenEstimateErrorPct: tokenSource && !estimated ? 0 : TOKEN_ESTIMATE_ERROR_PCT,
|
||||
// Present only under --exact: how far the offline estimate was off, per file.
|
||||
...(calibration ? { calibration } : {}),
|
||||
skills,
|
||||
@@ -605,6 +782,12 @@ export function tokenDisclaimer(bill: Pick<Bill, "tokenEstimateErrorPct" | "toke
|
||||
if (bill.tokenEstimateErrorPct === 0) {
|
||||
return `Token counts measured with ${bill.tokenSource}. Bytes are exact.`;
|
||||
}
|
||||
if (bill.tokenSource.startsWith(MIXED_PREFIX)) {
|
||||
return (
|
||||
`Token counts are MIXED (${bill.tokenSource.slice(MIXED_PREFIX.length)}). Any figure that includes an estimated ` +
|
||||
`item can be off by up to ${bill.tokenEstimateErrorPct}% for that item. Bytes are always exact.`
|
||||
);
|
||||
}
|
||||
const per = Object.entries(TOKEN_DIVISORS).map(([k, v]) => `${k} /${v}`).join(", ");
|
||||
return (
|
||||
`Token counts are ESTIMATES: bytes divided per content class (${per}), calibrated against ` +
|
||||
@@ -685,17 +868,132 @@ async function countTokens(text: string, { model, apiKey, fetchImpl }: CountToke
|
||||
return json.input_tokens;
|
||||
}
|
||||
|
||||
export interface ExactSendSet {
|
||||
/** Each real file once, under the first lexical path a skill's walk reaches it by. */
|
||||
files: WalkedFile[];
|
||||
/** Files the bill walks outside the root, the gstack checkout and gstack's render folder: never sent, estimated offline. */
|
||||
refused: WalkedFile[];
|
||||
/** Realpath of the gstack checkout the root's gstack/ entry links to, when there is one. */
|
||||
checkout?: string;
|
||||
/** Realpath of gstack's render folder ($GSTACK_STATE_ROOT/render) when it is a real directory; a render link is not trusted. */
|
||||
render?: string;
|
||||
/** Realpaths of the SKILL.md files whose frontmatter the bill prices. */
|
||||
frontmatter: Set<string>;
|
||||
}
|
||||
|
||||
/** The realpath of `dir` when it is a real directory, not a link; otherwise (missing, unreadable, a file) undefined. */
|
||||
function realDirectory(dir: string): string | undefined {
|
||||
try {
|
||||
return fs.lstatSync(dir).isDirectory() ? fs.realpathSync(dir) : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** A git checkout with VERSION, setup and a SKILL.md named gstack. */
|
||||
function isGstackCheckout(dir: string): boolean {
|
||||
if (!fs.existsSync(path.join(dir, ".git"))) return false;
|
||||
if (!["VERSION", "setup", "SKILL.md"].every((name) => isFile(path.join(dir, name)))) return false;
|
||||
const { block } = parseFrontmatter(fs.readFileSync(path.join(dir, "SKILL.md"), "utf8"));
|
||||
return /^name:\s*gstack\s*$/m.test(block);
|
||||
}
|
||||
|
||||
/**
|
||||
* What --exact sends: the same per-skill walks totalMd bills, with the same
|
||||
* fences, so nothing is walked or sent that no skill bills. Of those, only
|
||||
* files inside the root or inside the gstack checkout the skills link into
|
||||
* (the root's gstack/ entry, when it is a git checkout whose SKILL.md is named
|
||||
* gstack and which has VERSION and setup) or inside gstack's render folder
|
||||
* ($GSTACK_STATE_ROOT/render, where setup renders the SKILL.md files some
|
||||
* installs link to; trusted only as a real directory, not a link) are sent; anything else a link reaches is refused and
|
||||
* estimated offline. Each file is judged by its realpath, so a link out of
|
||||
* one of those folders is refused too.
|
||||
*
|
||||
* The root is resolved before keying. buildBill resolves its root, so a
|
||||
* relative root here would produce keys that never match and every lookup
|
||||
* would fall back to the estimate.
|
||||
*/
|
||||
export function exactSendSet(root: string, stateRoot: string = resolveStateRoot()): ExactSendSet {
|
||||
const resolved = path.resolve(root);
|
||||
if (!fs.existsSync(resolved)) throw new Error(`No such tree: ${resolved}`);
|
||||
const files: WalkedFile[] = [];
|
||||
const refused: WalkedFile[] = [];
|
||||
const frontmatter = new Set<string>();
|
||||
const seen = new Set<string>();
|
||||
const rootReal = fs.realpathSync(resolved);
|
||||
const gstackEntry = path.join(resolved, "gstack");
|
||||
const checkout = isGstackCheckout(gstackEntry) ? fs.realpathSync(gstackEntry) : undefined;
|
||||
const render = realDirectory(path.join(stateRoot, "render"));
|
||||
const approved = [rootReal, checkout, render].filter((d): d is string => d !== undefined);
|
||||
for (const dir of findSkillDirs(resolved)) {
|
||||
frontmatter.add(fs.realpathSync(path.join(dir, "SKILL.md")));
|
||||
for (const file of walkMdFiles(dir, { billedRoot: resolved, skipNestedSkills: true })) {
|
||||
if (seen.has(file.real)) continue;
|
||||
seen.add(file.real);
|
||||
if (!approved.some((d) => encloses(d, file.real))) {
|
||||
refused.push(file);
|
||||
continue;
|
||||
}
|
||||
files.push(file);
|
||||
}
|
||||
}
|
||||
return { files, refused, frontmatter, checkout, render };
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the exact file the walk approved: its realpath, opened without
|
||||
* following a final link, and only if it is still the same regular file at
|
||||
* the same realpath. A link retargeted, a file replaced, or a parent directory
|
||||
* swapped for a link after the walk is refused, so nothing the bill did not
|
||||
* walk can be sent. The open does not block, so a file swapped for a FIFO is
|
||||
* refused as not a regular file instead of hanging the run.
|
||||
*/
|
||||
function readWalkedFile({ path: file, real, dev, ino, size, mtimeMs }: WalkedFile, root: string): string {
|
||||
const changed = () => new Error(`${file} changed after it was walked; nothing from ${root} was sent. Run again.`);
|
||||
let fd: number;
|
||||
try {
|
||||
fd = fs.openSync(real, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) | (fs.constants.O_NONBLOCK ?? 0));
|
||||
} catch {
|
||||
throw changed();
|
||||
}
|
||||
try {
|
||||
const st = fs.fstatSync(fd);
|
||||
if (
|
||||
!st.isFile() ||
|
||||
st.dev !== dev ||
|
||||
st.ino !== ino ||
|
||||
st.size !== size ||
|
||||
st.mtimeMs !== mtimeMs ||
|
||||
fs.realpathSync(real) !== real
|
||||
) {
|
||||
throw changed();
|
||||
}
|
||||
return fs.readFileSync(fd, "utf8");
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
function fileIdentity(real: string, { dev, ino, size, mtimeMs }: { dev: number; ino: number; size: number; mtimeMs: number }): string {
|
||||
return `${real}\0${dev}\0${ino}\0${size}\0${mtimeMs}`;
|
||||
}
|
||||
|
||||
export interface ExactMeasurement {
|
||||
tokenSource: string;
|
||||
tokensOf: TokensOf;
|
||||
/** Texts sent to count_tokens (file bodies plus frontmatter blocks). */
|
||||
measuredFiles: number;
|
||||
counts: Map<string, number>;
|
||||
/** The realpath each measured file was read from, by its key. */
|
||||
realOf: Map<string, string>;
|
||||
/** Keys priced by estimate because measurement missed them. Read after buildBill. */
|
||||
missedKeys: Set<string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Measures every text the bill will bill for. Returns a `tokensOf` lookup.
|
||||
* Measures every text in the send set: what the bill prices inside the root,
|
||||
* the linked gstack checkout or gstack's render folder. Returns a `tokensOf`
|
||||
* lookup.
|
||||
*
|
||||
* count_tokens prices a whole request, so it includes a fixed message envelope.
|
||||
* That envelope is measured once and subtracted, leaving the tokens each file's
|
||||
@@ -708,11 +1006,15 @@ export interface ExactMeasurement {
|
||||
*/
|
||||
export async function measureExactTokens(
|
||||
root: string,
|
||||
{ model, apiKey, fetchImpl = fetch, onProgress, egressHome }: {
|
||||
{ sendSet, model, apiKey, fetchImpl = fetch, onProgress, onSend, egressHome }: {
|
||||
/** The send set from exactSendSet(root), when the caller already computed it. */
|
||||
sendSet?: ExactSendSet;
|
||||
model: string;
|
||||
apiKey: string;
|
||||
fetchImpl?: FetchLike;
|
||||
onProgress?: (done: number, total: number) => void;
|
||||
/** Called with the files about to be sent, after they are read and before the receipt; not called when nothing is sent. */
|
||||
onSend?: (files: WalkedFile[]) => void;
|
||||
egressHome?: string;
|
||||
},
|
||||
): Promise<ExactMeasurement> {
|
||||
@@ -725,18 +1027,71 @@ export async function measureExactTokens(
|
||||
const opts: CountTokensOptions = { model, apiKey, fetchImpl };
|
||||
|
||||
const texts = new Map<string, string>();
|
||||
// Resolve before keying. buildBill resolves its root, so a relative root here
|
||||
// would produce keys that never match and every lookup would fall back to the
|
||||
// estimate -- exact mode silently degrading to the thing it replaces.
|
||||
for (const file of walkMd(path.resolve(root))) {
|
||||
const text = fs.readFileSync(file, "utf8");
|
||||
texts.set(file, text);
|
||||
if (path.basename(file) === "SKILL.md") {
|
||||
const fm = parseFrontmatter(text);
|
||||
if (fm.block) texts.set(frontmatterKey(file), fm.block);
|
||||
}
|
||||
// count_tokens rejects content with no non-whitespace text, so such texts
|
||||
// are not sent: an empty one is measured as zero tokens, and a
|
||||
// whitespace-only one is left unmeasured, so the bill estimates it and says
|
||||
// so. When nothing is left to send, no receipt is written and no request is
|
||||
// made.
|
||||
const counts = new Map<string, number>();
|
||||
// The send set holds each real file once, so each is sent once. The bill keys
|
||||
// the same file by every lexical path it is billed under; measuredAs maps a
|
||||
// real file, at the inode, size and mtime it was walked with, back to the
|
||||
// one key that was measured.
|
||||
const measuredAs = new Map<string, string>();
|
||||
const realOf = new Map<string, string>();
|
||||
const { files, frontmatter } = sendSet ?? exactSendSet(root);
|
||||
for (const walked of files) {
|
||||
const { path: file, real } = walked;
|
||||
const text = readWalkedFile(walked, root);
|
||||
measuredAs.set(fileIdentity(real, walked), file);
|
||||
realOf.set(file, real);
|
||||
if (/\S/.test(text)) texts.set(file, text);
|
||||
else if (text === "") counts.set(file, 0);
|
||||
if (!frontmatter.has(real)) continue;
|
||||
const fm = parseFrontmatter(text);
|
||||
if (fm.block) texts.set(frontmatterKey(file), fm.block);
|
||||
else counts.set(frontmatterKey(file), 0);
|
||||
}
|
||||
|
||||
// A key the send set does not hold (a non-.md foreign-host file, or a file
|
||||
// refused as outside the root, the checkout and the render folder) falls back to the
|
||||
// estimate rather than billing zero. So does a key whose file is no longer
|
||||
// the one measured (a link retargeted, or the file rewritten, since the walk).
|
||||
// Misses are counted, not swallowed: a bill that is part-measured and
|
||||
// part-estimated must not present itself as measured.
|
||||
const missed = new Set<string>();
|
||||
const fmSuffix = frontmatterKey("");
|
||||
const measuredKey = (key: string): string | undefined => {
|
||||
const isFm = key.endsWith(fmSuffix);
|
||||
let real: string;
|
||||
let st: fs.Stats;
|
||||
try {
|
||||
real = realpathStepwise(isFm ? key.slice(0, -fmSuffix.length) : key);
|
||||
st = fs.lstatSync(real);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
const lexical = measuredAs.get(fileIdentity(real, st));
|
||||
return lexical === undefined ? undefined : isFm ? frontmatterKey(lexical) : lexical;
|
||||
};
|
||||
const tokensOf: TokensOf = (key, bytes) => {
|
||||
const alias = measuredKey(key);
|
||||
const exact = alias === undefined ? undefined : counts.get(alias);
|
||||
if (exact !== undefined) return exact;
|
||||
missed.add(key);
|
||||
return estimateTokensOf(key, bytes);
|
||||
};
|
||||
const measurement: ExactMeasurement = {
|
||||
tokenSource: `count_tokens (${model})`,
|
||||
tokensOf,
|
||||
measuredFiles: 0,
|
||||
counts,
|
||||
realOf,
|
||||
missedKeys: missed,
|
||||
};
|
||||
if (texts.size === 0) return measurement;
|
||||
onSend?.(files.filter((f) => texts.has(f.path)));
|
||||
|
||||
// Receipt-before-send. Content-free: file count + total bytes only.
|
||||
try {
|
||||
let totalBytes = 0;
|
||||
@@ -760,7 +1115,6 @@ export async function measureExactTokens(
|
||||
// One-char body: subtracting its single content token leaves the envelope.
|
||||
const envelope = (await countTokens("x", opts)) - 1;
|
||||
|
||||
const counts = new Map<string, number>();
|
||||
const keys = [...texts.keys()];
|
||||
let next = 0;
|
||||
let done = 0;
|
||||
@@ -773,24 +1127,8 @@ export async function measureExactTokens(
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.min(EXACT_CONCURRENCY, keys.length) }, worker));
|
||||
|
||||
// A key the walk never saw (a non-.md foreign-host file) falls back to the
|
||||
// estimate rather than billing zero. Misses are counted, not swallowed: a bill
|
||||
// that is part-measured and part-estimated must not present itself as measured.
|
||||
const missed = new Set<string>();
|
||||
const tokensOf: TokensOf = (key, bytes) => {
|
||||
const exact = counts.get(key);
|
||||
if (exact !== undefined) return exact;
|
||||
missed.add(key);
|
||||
return estimateTokensOf(key, bytes);
|
||||
};
|
||||
return {
|
||||
tokenSource: `count_tokens (${model})`,
|
||||
tokensOf,
|
||||
measuredFiles: counts.size,
|
||||
counts,
|
||||
missedKeys: missed,
|
||||
};
|
||||
measurement.measuredFiles = texts.size;
|
||||
return measurement;
|
||||
}
|
||||
|
||||
export interface Calibration {
|
||||
@@ -812,11 +1150,11 @@ export interface Calibration {
|
||||
* Estimate-vs-measured residual per file. This is what makes the divisor
|
||||
* auditable: run --exact and the tool grades its own offline estimate.
|
||||
*/
|
||||
export function calibrationTable(counts: Map<string, number>, root: string): Calibration {
|
||||
export function calibrationTable(counts: Map<string, number>, root: string, realOf?: Map<string, string>): Calibration {
|
||||
const rows: Calibration["rows"] = [];
|
||||
for (const [key, tokens] of counts) {
|
||||
if (key.endsWith("#frontmatter") || tokens === 0) continue;
|
||||
const bytes = bytesOf(key);
|
||||
const bytes = bytesOf(realOf?.get(key) ?? key);
|
||||
if (bytes == null) continue;
|
||||
// Grade the estimate the tool actually uses, class divisor included.
|
||||
const estimated = Math.round(estimateTokensOf(key, bytes));
|
||||
@@ -868,7 +1206,11 @@ const USAGE =
|
||||
"\n" +
|
||||
" --exact measure tokens with Anthropic's count_tokens instead of\n" +
|
||||
" estimating. Off by default: it sends the content of every\n" +
|
||||
" .md file in the tree to api.anthropic.com. Needs\n" +
|
||||
" .md file the bill prices inside the tree, inside\n" +
|
||||
" the gstack checkout its gstack/ entry links to, or\n" +
|
||||
" inside gstack's render folder ($GSTACK_STATE_ROOT/render)\n" +
|
||||
" to api.anthropic.com. Files that symlinks reach anywhere\n" +
|
||||
" else are named, not sent, and estimated offline. Needs\n" +
|
||||
" ANTHROPIC_API_KEY; passing --exact is the consent. An\n" +
|
||||
" egress receipt is written before the send (see\n" +
|
||||
" gstack-egress); if it cannot be written, the run falls\n" +
|
||||
@@ -887,6 +1229,7 @@ export interface MainOptions {
|
||||
apiKey?: string;
|
||||
fetchImpl?: FetchLike;
|
||||
egressHome?: string;
|
||||
stateRoot?: string;
|
||||
}
|
||||
|
||||
export async function contextBillMain(argv: string[], options: MainOptions = {}): Promise<number> {
|
||||
@@ -917,29 +1260,81 @@ export async function contextBillMain(argv: string[], options: MainOptions = {})
|
||||
|
||||
// Exact mode is the only path that leaves the machine. Announce what is sent
|
||||
// before sending it, and degrade to the estimate rather than failing the run.
|
||||
const exactFor = async (tree: string): Promise<{
|
||||
const sendSet = (tree: string) => (flags.exact ? exactSendSet(tree, options.stateRoot) : undefined);
|
||||
const exactFor = async (tree: string, set: ExactSendSet | undefined): Promise<{
|
||||
tokensOf?: TokensOf;
|
||||
tokenSource?: string;
|
||||
missedKeys?: ReadonlySet<string>;
|
||||
calibration?: Calibration;
|
||||
onDone?: () => void;
|
||||
}> => {
|
||||
if (!flags.exact) return {};
|
||||
const files = walkMd(tree).length;
|
||||
stderr.write(
|
||||
`--exact: sending the content of ${files} .md file(s) under ${tree} to ` +
|
||||
`api.anthropic.com for count_tokens (${flags.exactModel}). No other data leaves this machine.\n`,
|
||||
);
|
||||
if (!set) return {};
|
||||
if (set.refused.length) {
|
||||
const outsideDirs: string[] = [];
|
||||
const kept = new Set<string>();
|
||||
for (const d of [...new Set(set.refused.map((f) => path.dirname(f.real)))].sort()) {
|
||||
let covered = false;
|
||||
for (let a = d; !covered; a = path.dirname(a)) {
|
||||
covered = kept.has(a);
|
||||
if (path.dirname(a) === a) break;
|
||||
}
|
||||
if (covered) continue;
|
||||
kept.add(d);
|
||||
outsideDirs.push(d);
|
||||
}
|
||||
stderr.write(
|
||||
`--exact: not sending ${set.refused.length} .md file(s) that symlinks reach outside ${tree}, the gstack checkout and gstack's render folder ` +
|
||||
`(under ${outsideDirs.join(", ")}); they are estimated offline:\n` +
|
||||
set.refused.map((f) => ` ${path.relative(tree, f.path)}\n`).join(""),
|
||||
);
|
||||
}
|
||||
if (set.files.length === 0) {
|
||||
if (set.refused.length) {
|
||||
stderr.write(`--exact: every billed .md file is outside ${tree}, the gstack checkout and gstack's render folder; nothing was sent, so the bill uses the offline estimate.\n`);
|
||||
} else if (set.frontmatter.size) {
|
||||
stderr.write(`--exact: no markdown skill file to measure under ${tree}; nothing was sent, so the bill uses the offline estimate.\n`);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
let announced = false;
|
||||
const announce = (sent: WalkedFile[]) => {
|
||||
announced = true;
|
||||
const treeReal = fs.realpathSync(tree);
|
||||
const outsideTree = sent.filter((f) => !encloses(treeReal, f.real));
|
||||
const inCheckout = outsideTree.filter((f) => set.checkout && encloses(set.checkout, f.real)).length;
|
||||
const inRender = outsideTree.length - inCheckout;
|
||||
const from = [
|
||||
inCheckout ? `${inCheckout} of them in the gstack checkout ${set.checkout}` : "",
|
||||
inRender ? `${inRender} in gstack's render folder ${set.render}` : "",
|
||||
].filter(Boolean);
|
||||
stderr.write(
|
||||
`--exact: sending the content of ${sent.length} .md file(s) under ${tree}` +
|
||||
(from.length ? ` (${from.join(", ")})` : "") +
|
||||
` to api.anthropic.com for count_tokens (${flags.exactModel}). No other data leaves this machine.\n`,
|
||||
);
|
||||
};
|
||||
try {
|
||||
const measured = await measureExactTokens(tree, {
|
||||
sendSet: set,
|
||||
model: flags.exactModel,
|
||||
apiKey: options.apiKey ?? process.env.ANTHROPIC_API_KEY ?? "",
|
||||
fetchImpl: options.fetchImpl,
|
||||
onSend: announce,
|
||||
egressHome: options.egressHome,
|
||||
});
|
||||
if (!announced) {
|
||||
stderr.write(
|
||||
`--exact: every .md file --exact could send under ${tree} is blank` +
|
||||
(set.refused.length ? `, and the rest are outside the folders it sends from` : "") +
|
||||
`; nothing was sent, so the bill uses the offline estimate.\n`,
|
||||
);
|
||||
return {};
|
||||
}
|
||||
return {
|
||||
tokensOf: measured.tokensOf,
|
||||
tokenSource: measured.tokenSource,
|
||||
calibration: calibrationTable(measured.counts, tree),
|
||||
missedKeys: measured.missedKeys,
|
||||
calibration: calibrationTable(measured.counts, tree, measured.realOf),
|
||||
onDone: () => {
|
||||
if (measured.missedKeys.size) {
|
||||
stderr.write(
|
||||
@@ -964,8 +1359,10 @@ export async function contextBillMain(argv: string[], options: MainOptions = {})
|
||||
}
|
||||
const treeA = path.resolve(cwd, positional[0]);
|
||||
const treeB = path.resolve(cwd, positional[1]);
|
||||
const optsA = await exactFor(treeA);
|
||||
const optsB = await exactFor(treeB);
|
||||
const setA = sendSet(treeA);
|
||||
const setB = sendSet(treeB);
|
||||
const optsA = await exactFor(treeA, setA);
|
||||
const optsB = await exactFor(treeB, setB);
|
||||
const diff = diffBills(buildBill(treeA, optsA), buildBill(treeB, optsB));
|
||||
optsA.onDone?.();
|
||||
optsB.onDone?.();
|
||||
@@ -980,7 +1377,7 @@ export async function contextBillMain(argv: string[], options: MainOptions = {})
|
||||
);
|
||||
return 2;
|
||||
}
|
||||
const exactOpts = await exactFor(tree);
|
||||
const exactOpts = await exactFor(tree, sendSet(tree));
|
||||
const bill = buildBill(tree, exactOpts);
|
||||
exactOpts.onDone?.();
|
||||
if (bill.skills.length === 0) {
|
||||
|
||||
+37
-3
@@ -103,8 +103,42 @@ export function checkCraft(html: string): CraftFinding[] {
|
||||
return findings;
|
||||
}
|
||||
|
||||
export function formatCraft(findings: CraftFinding[]): string {
|
||||
const lines = findings.map(f => `CRAFT_CHECK: ${f.id} ${f.count} ${f.fix}`);
|
||||
lines.push(findings.length === 0 ? 'CRAFT_RESULT: clean' : `CRAFT_RESULT: ${findings.length} issue(s)`);
|
||||
/** An accepted finding recorded in finalized.json's `craft_exceptions`. */
|
||||
export interface CraftException {
|
||||
id: string;
|
||||
count: number;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/** Reads `craft_exceptions` from a finalized.json; a record without one has none. Throws on a malformed record. */
|
||||
export function parseCraftExceptions(json: string): CraftException[] {
|
||||
const record = JSON.parse(json);
|
||||
if (typeof record !== 'object' || record === null || Array.isArray(record)) throw new Error('not a finalized.json object');
|
||||
const list = record.craft_exceptions ?? [];
|
||||
const valid = (e: any) => typeof e?.id === 'string' && Number.isInteger(e.count) && e.count > 0 && typeof e.reason === 'string' && e.reason.trim() !== '';
|
||||
if (!Array.isArray(list) || !list.every(valid)) throw new Error('craft_exceptions must be a list of { id, count, reason } with a positive count and a reason');
|
||||
return list.map(({ id, count, reason }) => ({ id, count, reason: reason.replace(/\s+/g, ' ').trim() }));
|
||||
}
|
||||
|
||||
/** A finding is accepted when an exception names its id with at least its count; a higher count stays an issue and says how many were accepted. */
|
||||
export function acceptFindings(findings: CraftFinding[], exceptions: CraftException[]): { issues: CraftFinding[]; accepted: CraftException[] } {
|
||||
const issues: CraftFinding[] = [];
|
||||
const accepted: CraftException[] = [];
|
||||
for (const finding of findings) {
|
||||
const exception = exceptions.filter(e => e.id === finding.id).sort((a, b) => b.count - a.count)[0];
|
||||
if (!exception) issues.push(finding);
|
||||
else if (finding.count <= exception.count) accepted.push({ id: finding.id, count: finding.count, reason: exception.reason });
|
||||
else issues.push({ ...finding, fix: `${finding.fix} (${exception.count} accepted earlier: ${exception.reason})` });
|
||||
}
|
||||
return { issues, accepted };
|
||||
}
|
||||
|
||||
export function formatCraft(findings: CraftFinding[], accepted: CraftException[] = []): string {
|
||||
const lines = [
|
||||
...findings.map(f => `CRAFT_CHECK: ${f.id} ${f.count} ${f.fix}`),
|
||||
...accepted.map(a => `CRAFT_ACCEPTED: ${a.id} ${a.count} ${a.reason}`),
|
||||
];
|
||||
const note = accepted.length === 0 ? '' : findings.length === 0 ? ` (${accepted.length} accepted)` : `, ${accepted.length} accepted`;
|
||||
lines.push(`${findings.length === 0 ? 'CRAFT_RESULT: clean' : `CRAFT_RESULT: ${findings.length} issue(s)`}${note}`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -14,6 +14,8 @@ The built page is **committed** (eng-review D2): rendering works with zero
|
||||
network at install time and render time, and there is no npm supply-chain
|
||||
surface in `./setup`. The drift test (`test/diagram-render-drift.test.ts`)
|
||||
fails CI if `dist/` is edited by hand or falls out of sync with `BUILD_INFO.json`.
|
||||
It also checks that the bundled DOMPurify is the `package.json` override and that
|
||||
the bundle carries no Google API key.
|
||||
|
||||
## Page API (window functions)
|
||||
|
||||
@@ -41,6 +43,12 @@ bun run build
|
||||
# 3. commit package.json + bun.lock + dist/ together
|
||||
```
|
||||
|
||||
`scripts/build.ts` empties the one `VITE_APP_FIREBASE_CONFIG` literal in
|
||||
`@excalidraw/excalidraw`'s npm bundle (a public Firebase client key the offline page
|
||||
never uses), because a key-shaped literal fails the pre-gate secret scan. The build
|
||||
fails if that literal is not matched exactly once or an `AIza…` key survives, so an
|
||||
excalidraw bump that changes its shape needs the strip rule updated.
|
||||
|
||||
Render contract details (securityLevel strict, htmlLabels false, print-css font
|
||||
lock, `<base href>` + `</scri` escaping) are documented in `src/entry.ts` and
|
||||
`scripts/build.ts` — read both before touching either.
|
||||
@@ -15,7 +15,7 @@
|
||||
},
|
||||
"overrides": {
|
||||
"chokidar": "4.0.3",
|
||||
"dompurify": "3.4.16",
|
||||
"dompurify": "3.4.17",
|
||||
"lodash-es": "4.18.1",
|
||||
"nanoid": "5.1.16",
|
||||
"source-map-js": "1.2.2",
|
||||
@@ -281,7 +281,7 @@
|
||||
|
||||
"detect-node-es": ["detect-node-es@1.1.0", "", {}, "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ=="],
|
||||
|
||||
"dompurify": ["dompurify@3.4.16", "", { "optionalDependencies": { "@types/trusted-types": "^2.0.7" } }, "sha512-sqo+pNp3qRhCIpbgRi1y8Tgk27Bo2Ry7w0dC1NBeNTdZChWjz9Xb/KOoZbRP/R6pQZ80Qw8YhXw13hWWBbMRnQ=="],
|
||||
"dompurify": ["dompurify@3.4.17", "", { "optionalDependencies": { "@types/trusted-types": "^2.0.7" } }, "sha512-B/pSlYVHQ58kqISD/yDjuJ1yfzI34m1gF4lAod2bwSRhiN4K5Cl3H8Af7hqB2PmTL911tdEudnnXn5FnhRoz0w=="],
|
||||
|
||||
"es-toolkit": ["es-toolkit@1.52.0", "", {}, "sha512-XTNEJQh1tY1ZJVcf6ayP/2n4ZPyaHlW2FWs7xvw5ddPuhUVjLD3olQVQS7kf58JbAB48iL0uL/jerTrjtV3lDA=="],
|
||||
|
||||
|
||||
+3
-3
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"name": "gstack-diagram-render",
|
||||
"sha256": "ec759ca41239ff888acdd0654c3f6ef505e3314b87c6e5de11903f804fc8a859",
|
||||
"srcSha256": "07238fae312bc0444f62b0a0a3404a8a38c45cef505aa1528c60a0ded17cbe06",
|
||||
"bytes": 7884089,
|
||||
"sha256": "c2d37f9f924baae2a146bc109bf9b4c308d89f49e08b1c35793edc310b579444",
|
||||
"srcSha256": "17d8921086e091252425c63f06cab01b4626d8e0fa9fe8738267e1ac4ade52e8",
|
||||
"bytes": 7884979,
|
||||
"bunVersion": "1.4.2",
|
||||
"deps": {
|
||||
"@excalidraw/excalidraw": "0.18.1",
|
||||
|
||||
+585
-585
File diff suppressed because it is too large.
Load diff
@@ -15,7 +15,7 @@
|
||||
},
|
||||
"overrides": {
|
||||
"chokidar": "4.0.3",
|
||||
"dompurify": "3.4.16",
|
||||
"dompurify": "3.4.17",
|
||||
"nanoid": "5.1.16",
|
||||
"source-map-js": "1.2.2",
|
||||
"lodash-es": "4.18.1"
|
||||
|
||||
@@ -42,8 +42,26 @@ if (!result.success) {
|
||||
}
|
||||
const js = await result.outputs[0].text();
|
||||
|
||||
// @excalidraw/excalidraw's npm bundle carries excalidraw.com's public Firebase
|
||||
// client config (an `AIza…` key) in an env literal nothing in the package
|
||||
// reads. The offline page never opens a collab session, and a key-shaped
|
||||
// literal in a committed file fails the pre-gate secret scan on every rebuild,
|
||||
// so the literal is emptied. Exactly one match is expected: a bump that changes
|
||||
// the shape must fail here rather than put the key back silently.
|
||||
const FIREBASE_CONFIG = /VITE_APP_FIREBASE_CONFIG:'\{[^']*\}'/g;
|
||||
const firebaseHits = js.match(FIREBASE_CONFIG)?.length ?? 0;
|
||||
if (firebaseHits !== 1) {
|
||||
console.error(`expected exactly one VITE_APP_FIREBASE_CONFIG literal in the excalidraw bundle, found ${firebaseHits}; update the strip rule in scripts/build.ts`);
|
||||
process.exit(1);
|
||||
}
|
||||
const strippedJs = js.replace(FIREBASE_CONFIG, "VITE_APP_FIREBASE_CONFIG:'{}'");
|
||||
if (/AIza[0-9A-Za-z_-]{35}/.test(strippedJs)) {
|
||||
console.error("a Google API key literal survived the strip; update the strip rule in scripts/build.ts");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Escape inline-script terminators (see header note).
|
||||
const inlineJs = js.replaceAll("</scri", "<\\/scri");
|
||||
const inlineJs = strippedJs.replaceAll("</scri", "<\\/scri");
|
||||
|
||||
const head = `<!doctype html>
|
||||
<html>
|
||||
|
||||
+68
-9
@@ -46,12 +46,35 @@ export function resolvesToCommit(ref: string, cwd: string): string | null {
|
||||
return gitText(['rev-parse', '--verify', '-q', `${ref}^{commit}`], cwd);
|
||||
}
|
||||
|
||||
/** origin/HEAD's target → `main` → `master` (the gstack-diff-scope chain), as a ref name. */
|
||||
/**
|
||||
* A ref that could be a branch name: not a sha, not relative or symbolic
|
||||
* (`HEAD~1`, `main^2`, `@{u}`, `HEAD`, `FETCH_HEAD`), not a full `refs/` path.
|
||||
* Only such a ref has a remote-tracking twin (`origin/<ref>`) worth trying, and
|
||||
* only such a ref goes into a fix-line refspec; `origin/HEAD~1` would resolve
|
||||
* to a different commit than the `HEAD~1` the caller named.
|
||||
*/
|
||||
function isBranchName(ref: string): boolean {
|
||||
return !/^[0-9a-f]{7,64}$/.test(ref) && !/[~^]|@\{/.test(ref) && !/^(?:[A-Z_]*HEAD)$/.test(ref) && !ref.startsWith('refs/');
|
||||
}
|
||||
|
||||
function remoteNames(cwd: string): string[] {
|
||||
const remotes = (gitText(['remote'], cwd) ?? '').split('\n').filter(Boolean);
|
||||
return remotes.includes('origin') ? ['origin', ...remotes.filter((r) => r !== 'origin')] : remotes;
|
||||
}
|
||||
|
||||
/**
|
||||
* The remote's HEAD target → `main` → `master` (the gstack-diff-scope chain),
|
||||
* as a ref name; the remotes are walked in resolveBase's order, so the default
|
||||
* and the fix line come from the same remote.
|
||||
*/
|
||||
export function defaultBase(cwd: string): string {
|
||||
const head = gitText(['symbolic-ref', '-q', 'refs/remotes/origin/HEAD'], cwd);
|
||||
if (head) return head.replace(/^refs\/remotes\//, '');
|
||||
const remotes = remoteNames(cwd);
|
||||
for (const remote of remotes) {
|
||||
const head = gitText(['symbolic-ref', '-q', `refs/remotes/${remote}/HEAD`], cwd);
|
||||
if (head) return head.replace(/^refs\/remotes\//, '');
|
||||
}
|
||||
for (const name of ['main', 'master']) {
|
||||
if (resolvesToCommit(name, cwd) || resolvesToCommit(`origin/${name}`, cwd)) return name;
|
||||
if (resolvesToCommit(name, cwd) || remotes.some((remote) => resolvesToCommit(`${remote}/${name}`, cwd))) return name;
|
||||
}
|
||||
return 'main';
|
||||
}
|
||||
@@ -60,14 +83,50 @@ export function isShallow(cwd: string): boolean {
|
||||
return gitText(['rev-parse', '--is-shallow-repository'], cwd) === 'true';
|
||||
}
|
||||
|
||||
/** The concrete recovery for an unresolvable base: fetch it from the detected remote, unshallowing when needed. */
|
||||
/**
|
||||
* The base as a ref that resolves here and, when `head` is given, shares
|
||||
* history with it: the ref itself, else its remote-tracking branch
|
||||
* (`origin/main` for `main`). A clone that checked out only the feature
|
||||
* branch has no local `main`, and the merge-base against `origin/main` is the
|
||||
* same commit; a stale local `main` that shares no history with HEAD (a
|
||||
* fetched-over clone) yields to the twin too, so the fix line's fetch into the
|
||||
* tracking ref is a fix and not a loop.
|
||||
*/
|
||||
export function resolveBase(ref: string, cwd: string, head?: string): { ref: string; sha: string; local?: 'missing' | 'unrelated' } | null {
|
||||
const related = (sha: string) => !head || mergeBase(sha, head, cwd) !== null;
|
||||
const direct = resolvesToCommit(ref, cwd);
|
||||
if (direct && related(direct)) return { ref, sha: direct };
|
||||
if (!isBranchName(ref)) return direct ? { ref, sha: direct } : null;
|
||||
for (const remote of remoteNames(cwd)) {
|
||||
const tracking = `${remote}/${ref}`;
|
||||
const sha = resolvesToCommit(tracking, cwd);
|
||||
if (sha && related(sha)) return { ref: tracking, sha, local: direct ? 'unrelated' : 'missing' };
|
||||
}
|
||||
return direct ? { ref, sha: direct } : null;
|
||||
}
|
||||
|
||||
/** A word for a printed shell line: as is when it is plain, single-quoted otherwise (ref names may hold `;`, `$`, `(`, `{1,2}`). */
|
||||
function shellWord(s: string): string {
|
||||
return /^[A-Za-z0-9._\/@^~+:-]+$/.test(s) ? s : `'${s.replace(/'/g, "'\\''")}'`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The concrete recovery for an unresolvable base: fetch it from the detected
|
||||
* remote into its remote-tracking ref (which resolveBase then finds; a bare
|
||||
* `git fetch origin main` only updates FETCH_HEAD in a single-branch clone),
|
||||
* unshallowing when needed. The refspec is forced (`+`) so the line also
|
||||
* repairs a tracking ref that exists but shares no history with HEAD. A sha
|
||||
* or a relative ref (`HEAD~1`, `main^2`, `@{u}`) is not a refspec: that gets a
|
||||
* plain fetch (deepened when shallow).
|
||||
*/
|
||||
export function noBaseFix(ref: string, cwd: string): string {
|
||||
const remotes = (gitText(['remote'], cwd) ?? '').split('\n').filter(Boolean);
|
||||
const remotes = remoteNames(cwd);
|
||||
const m = /^([^/]+)\/(.+)$/.exec(ref);
|
||||
const remote = m && remotes.includes(m[1]) ? m[1] : remotes.includes('origin') ? 'origin' : remotes[0] ?? 'origin';
|
||||
const remote = m && remotes.includes(m[1]) ? m[1] : remotes[0] ?? 'origin';
|
||||
const branch = m && remotes.includes(m[1]) ? m[2] : ref;
|
||||
if (/^[0-9a-f]{7,64}\^?$/.test(ref) || ref.endsWith('^')) return isShallow(cwd) ? `git fetch --deepen=50 ${remote}` : `git fetch ${remote}`;
|
||||
return isShallow(cwd) ? `git fetch --unshallow ${remote} ${branch}` : `git fetch ${remote} ${branch}`;
|
||||
if (!isBranchName(ref)) return isShallow(cwd) ? `git fetch --deepen=50 ${shellWord(remote)}` : `git fetch ${shellWord(remote)}`;
|
||||
const refspec = shellWord(`+${branch}:refs/remotes/${remote}/${branch}`);
|
||||
return isShallow(cwd) ? `git fetch --unshallow ${shellWord(remote)} ${refspec}` : `git fetch ${shellWord(remote)} ${refspec}`;
|
||||
}
|
||||
|
||||
export function mergeBase(a: string, b: string, cwd: string): string | null {
|
||||
|
||||
+56
-3
@@ -16,6 +16,8 @@
|
||||
import { createHash } from "crypto";
|
||||
import { scan, type Finding, type RepoVisibility } from "./redact-engine";
|
||||
import { canonicalRemote } from "./remote-identity";
|
||||
import { parseChangelog } from "./changelog-check";
|
||||
import { versionWidth } from "./version-source";
|
||||
|
||||
export const OPS = ["pr-comment", "issue-comment", "reply", "pr-title", "pr-body", "pr-create", "issue-create"] as const;
|
||||
export type Op = (typeof OPS)[number];
|
||||
@@ -198,11 +200,61 @@ function visibility(env: PostEnv, req: Request, kind: HostKind, configBin: strin
|
||||
type Part = "title" | "body";
|
||||
interface Located { part: Part; finding: Finding }
|
||||
|
||||
function scanParts(parts: Array<[Part, string]>, vis: RepoVisibility, selfEmail: string | undefined) {
|
||||
/**
|
||||
* The repository's own four-part release versions: the root VERSION file's
|
||||
* value and every version in a root CHANGELOG.md release heading, read with
|
||||
* the same parser gstack-restamp uses (`## [1.2.3.4]`, `## v1.2.3.4`, an
|
||||
* optional `- date`; a heading that only mentions a quad is not one). A
|
||||
* four-part version is byte-for-byte a dotted quad, and `1.91.191.0` is a
|
||||
* public address, so "(1.91.191.0)" in a PR body raised pii.ip_public
|
||||
* (MEDIUM), which an unattended /ship cannot confirm. The engine's own version
|
||||
* exemption reads only the declaration right before the span, so prose that
|
||||
* merely names a release gets no help from it. These exact strings are
|
||||
* allowlisted for the scan wherever they appear (the allowlist is exact-string
|
||||
* and context-free, so `host: <released quad>` is exempt too, a trade the
|
||||
* repo's own release set is small enough to carry); any other dotted quad
|
||||
* (8.8.8.8, 203.0.113.7, a version this repo never released) still reports.
|
||||
* The files are read from the checked-out tree, so a branch that adds a
|
||||
* four-part heading widens the set for posts made from that checkout.
|
||||
*/
|
||||
function repoReleaseVersions(env: PostEnv): string[] {
|
||||
const top = env.run("git", ["rev-parse", "--show-toplevel"]);
|
||||
const root = top.status === 0 ? top.stdout.trim() : "";
|
||||
if (!root) return [];
|
||||
const read = (name: string) => { try { return env.readFile(`${root}/${name}`); } catch { return ""; } };
|
||||
const versions = new Set<string>();
|
||||
const current = read("VERSION").trim();
|
||||
if (versionWidth(current) === 4) versions.add(current);
|
||||
for (const s of parseChangelog(read("CHANGELOG.md")).sections) if (s.version && versionWidth(s.version) === 4) versions.add(s.version);
|
||||
return [...versions];
|
||||
}
|
||||
|
||||
/**
|
||||
* All-digit identifiers a ship receipt or a CI link names, which pii.phone.e164
|
||||
* reads as phone numbers: a 12-hex receipt sha that happens to be all digits
|
||||
* (`tree=704992822996`, `tree 704992822996`, and the receipt's `head=`,
|
||||
* `gated_tree=` and `predecessor=` keys) and an 11-digit GitHub Actions
|
||||
* run id as prose or link text (`run 19368712345`, `[run 19368712345](…/actions/runs/19368712345)`;
|
||||
* the engine already exempts the id inside the URL itself). Only the exact
|
||||
* tokens found in those positions are allowlisted, so a bare 11-digit number,
|
||||
* a 10-digit national number and an E.164 `+14155551234` still report.
|
||||
*/
|
||||
function receiptDigitTokens(texts: string[]): string[] {
|
||||
const tokens = new Set<string>();
|
||||
for (const text of texts) {
|
||||
for (const m of text.matchAll(/(?:^|[^\w])(?:tree|gated_tree|head|predecessor)[= ](\d{12})\b/g)) tokens.add(m[1]!);
|
||||
for (const m of text.matchAll(/\brun #?(\d{11})\b/g)) tokens.add(m[1]!);
|
||||
for (const m of text.matchAll(/\/actions\/runs\/(\d{11})\b/g)) tokens.add(m[1]!);
|
||||
}
|
||||
return [...tokens];
|
||||
}
|
||||
|
||||
function scanParts(parts: Array<[Part, string]>, vis: RepoVisibility, selfEmail: string | undefined, releaseVersions: string[]) {
|
||||
const found: Located[] = [];
|
||||
let oversize = false;
|
||||
const allowlist = [...releaseVersions, ...receiptDigitTokens(parts.map(([, text]) => text))];
|
||||
for (const [part, text] of parts) {
|
||||
const result = scan(text, { repoVisibility: vis, ...(selfEmail ? { selfEmail } : {}) });
|
||||
const result = scan(text, { repoVisibility: vis, allowlist, ...(selfEmail ? { selfEmail } : {}) });
|
||||
oversize ||= result.oversize;
|
||||
for (const finding of result.findings) found.push({ part, finding });
|
||||
}
|
||||
@@ -303,7 +355,8 @@ export function runPost(argv: string[], env: PostEnv, configBin = "gstack-config
|
||||
|
||||
const vis = visibility(env, req, remote.kind, configBin);
|
||||
const email = env.run("git", ["config", "user.email"]);
|
||||
const { found, oversize } = scanParts(parts, vis, email.status === 0 ? email.stdout.trim() || undefined : undefined);
|
||||
const hasQuad = parts.some(([, text]) => /\b\d{1,3}(?:\.\d{1,3}){3}\b/.test(text));
|
||||
const { found, oversize } = scanParts(parts, vis, email.status === 0 ? email.stdout.trim() || undefined : undefined, hasQuad ? repoReleaseVersions(env) : []);
|
||||
env.out(`REPO_VISIBILITY: ${vis}\n`);
|
||||
const line = (m: Located) => `RULE: ${m.finding.id} LINE: ${m.finding.line} PART: ${m.part}\n`;
|
||||
const high = found.filter(m => m.finding.severity === "HIGH");
|
||||
|
||||
@@ -141,7 +141,11 @@ export function verdictProblem(skill: string, gstackRoot: string, env: Env, bunP
|
||||
const file = findInstallVerdict(src, project, env);
|
||||
const v = readInstallVerdict(file);
|
||||
const problem = (detail: string): VerdictProblem => ({ detail, code: 'INSTALL_UNVERIFIED' });
|
||||
if (!v) return problem(`no install verdict at ${file} for ${src}`);
|
||||
if (!v && fs.existsSync(file)) return problem(`the install verdict at ${file} is unreadable or has an unsupported schema_version; rerun the installer`);
|
||||
if (!v) {
|
||||
return { ...problem(`no install verdict at ${file} for ${src}`),
|
||||
fix: `run the Capy skill installer (scripts/install.sh --for ${skill}) from ${project ?? 'this directory'}: the verdict is keyed by the project directory the installer runs in, so a run from the skill's own folder does not cover ${project ?? 'it'}; then launch with the GSTACK_LAUNCH line it prints` };
|
||||
}
|
||||
if (!v.ok) return problem(`the install verdict says: ${v.trailer}`);
|
||||
if (v.checkout !== src) return problem(`the install verdict is for checkout ${v.checkout}, not ${src}`);
|
||||
const head = spawnSync('git', ['-C', src, 'rev-parse', 'HEAD'], { encoding: 'utf8', timeout: 10_000 }).stdout?.trim() ?? '';
|
||||
|
||||
@@ -7,6 +7,9 @@
|
||||
* with no lane is `FAIL lanes — <file> has no test lane`, never a pass.
|
||||
* Markdown no test depends on is the docs/no-tests empty lane instead
|
||||
* (lib/pregate/docs-lane.ts): it does not fail, and unmapped code still does.
|
||||
* A release manifest whose diff against the merge base changes only version
|
||||
* values is waived as version-stamp-only (lib/pregate/stamp-only.ts) and named
|
||||
* on its own line; any other change to it still needs a lane.
|
||||
* A test that reads the file still selects that test.
|
||||
* Incidents: p0-4-pipe-hidden-failure (`false | tail` read as green),
|
||||
* p0-4-windows-lane-unrun (a Windows-only change merged with its lane unrun).
|
||||
@@ -328,6 +331,7 @@ function describe(sel: Selection): string[] {
|
||||
const lines = sel.noLane.map(f => `FAIL lanes — ${f} has no test lane`);
|
||||
if (sel.emptyLane.length) lines.push(`docs/no-tests (empty lane): ${sel.emptyLane.join(' ')}`);
|
||||
if (sel.inert.length) lines.push(`no-tests (assets and tool config): ${sel.inert.join(' ')}`);
|
||||
if (sel.stampOnly.length) lines.push(`version-stamp-only (no lane needed): ${sel.stampOnly.join(' ')} — only version values changed against the merge base; a manifest whose other content changed still needs a lane`);
|
||||
for (const r of sel.remote) lines.push(`requires-remote ${r.lane} (${r.platform}): ${r.tests.length} test(s) — ${r.tests.slice(0, 8).join(' ')}${r.tests.length > 8 ? ' …' : ''}`);
|
||||
lines.push(`Tiers ran: ${sel.tiersRan.join(' ') || '(none)'}; requires-remote: ${[...new Set(sel.remote.map(r => r.platform))].join(' ') || 'none'}`);
|
||||
return lines;
|
||||
@@ -414,10 +418,9 @@ async function run(ctx: PregateContext): Promise<CheckResult> {
|
||||
return { ...base, status: 'fail', code: 'PREGATE_NO_LANE', detail: `${sel.noLane.length} touched file(s) with no test lane: ${sel.noLane.join(' ')}`, fix: noLaneFix(ctx, sel.noLane) };
|
||||
}
|
||||
if (sel.tests.length === 0) {
|
||||
const inert = sel.inert.length ? `${sel.inert.length} asset or tool-config file(s)` : '';
|
||||
const detail = sel.emptyLane.length
|
||||
? `docs/no-tests: ${sel.emptyLane.length} documentation file(s)${inert ? ` and ${inert}` : ''}; nothing touched selects a test`
|
||||
: inert ? `no-tests: ${inert}; nothing touched selects a test`
|
||||
const classes = [sel.emptyLane.length ? `${sel.emptyLane.length} documentation file(s)` : '', sel.inert.length ? `${sel.inert.length} asset or tool-config file(s)` : '', sel.stampOnly.length ? `${sel.stampOnly.length} version-stamp-only manifest(s)` : ''].filter(Boolean);
|
||||
const label = sel.emptyLane.length ? 'docs/no-tests' : sel.inert.length ? 'no-tests' : 'version-stamp-only';
|
||||
const detail = classes.length ? `${label}: ${classes.join(' and ')}; nothing touched selects a test`
|
||||
: ctx.touched.length ? 'nothing touched selects a test' : 'no touched files';
|
||||
return { ...base, detail };
|
||||
}
|
||||
|
||||
@@ -76,6 +76,12 @@ export function isDocsEmptyLane(rel: string, extraGlobs: readonly string[] = [])
|
||||
return isBuiltinDocsPath(norm);
|
||||
}
|
||||
|
||||
/** True under a test directory or a protected tree (snapshots, goldens, baselines, fixtures, CI workflows and actions, `.gstack/`): never an empty lane, never a version stamp. */
|
||||
export function isProtectedTree(rel: string): boolean {
|
||||
const norm = rel.replace(/\\/g, '/').replace(/^\.\//, '');
|
||||
return isTestTree(norm) || PROTECTED_TREE.test(norm);
|
||||
}
|
||||
|
||||
/** Tool configuration on the allowlist: a named dotfile at any depth, or a file under an editor or design-tool dot-directory. */
|
||||
function isToolConfig(rel: string): boolean {
|
||||
return INERT_DOTFILE.test(rel.split('/').pop() ?? rel) || INERT_DOT_DIR.test(rel);
|
||||
|
||||
@@ -7,12 +7,16 @@
|
||||
* runner-owned selection manifest `.gstack/pregate.json` declares for a
|
||||
* planner-computed matrix. Every uncertainty widens the selection.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { GENERATED_PATH } from '../regen';
|
||||
import { runBashSync } from '../script-invocation';
|
||||
import { matchesAny } from '../ship-policy';
|
||||
import { PREGATE_CONFIG_PATH } from './config';
|
||||
import { isDocsEmptyLane, isInertNoTests } from './docs-lane';
|
||||
import { isDocsEmptyLane, isInertNoTests, isProtectedTree } from './docs-lane';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { buildGraph, isTestFile, listRepoFiles, testsFor, type Graph } from './import-graph';
|
||||
import { stampFileKind, versionStampOnly } from './stamp-only';
|
||||
import type { PregateContext, RemoteObligation } from './types';
|
||||
import type { Lane } from './workflows';
|
||||
|
||||
@@ -23,6 +27,8 @@ export interface Selection {
|
||||
emptyLane: string[];
|
||||
/** Assets and dotfile tool config with no import-graph test: files no runner loads, also an empty lane. */
|
||||
inert: string[];
|
||||
/** Release manifests whose diff against the merge base changes only version values (lib/pregate/stamp-only.ts): waived, no lane needed. */
|
||||
stampOnly: string[];
|
||||
/** Every test the touched files select, sorted. */
|
||||
tests: string[];
|
||||
/** Tests that run on this machine. */
|
||||
@@ -70,6 +76,27 @@ export function laneFileSet(ctx: PregateContext, lane: Lane): string[] | '?' | '
|
||||
return files.size ? [...files] : 'all';
|
||||
}
|
||||
|
||||
const STAMP_FILE_CAP = 1024 * 1024;
|
||||
/** Strict UTF-8 text of `bytes`, BOM kept as content, or null when invalid (two different invalid bytes would otherwise both decode to U+FFFD and compare equal) or over the cap. */
|
||||
function strictText(bytes: Buffer | null | undefined): string | null {
|
||||
if (!bytes || bytes.length > STAMP_FILE_CAP) return null;
|
||||
try { return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes); } catch { return null; }
|
||||
}
|
||||
/** The file's text at the merge base when it is a regular blob there of mode `mode` (null when absent, a symlink or submodule, another mode, unreadable, invalid UTF-8 or over 1 MiB). */
|
||||
function readAtMergeBase(ctx: PregateContext, rel: string, mode: string): string | null {
|
||||
const ls = spawnSync('git', ['ls-tree', ctx.mergeBase, '--', rel], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: 60_000, env: ctx.env });
|
||||
if (ls.status !== 0 || !(ls.stdout ?? '').startsWith(`${mode} blob `)) return null;
|
||||
const r = spawnSync('git', ['show', `${ctx.mergeBase}:${rel}`], { cwd: ctx.repoRoot, timeout: 60_000, maxBuffer: STAMP_FILE_CAP + 1, env: ctx.env });
|
||||
return r.status === 0 ? strictText(r.stdout as Buffer) : null;
|
||||
}
|
||||
/** The working-tree file's text and git mode under the same rules (a regular file only; a symlink or a mode change is not a stamp). */
|
||||
function readWorkingTree(repoRoot: string, rel: string): { text: string | null; mode: string } {
|
||||
try {
|
||||
const abs = path.join(repoRoot, rel); const st = fs.lstatSync(abs);
|
||||
return { text: st.isFile() && st.size <= STAMP_FILE_CAP ? strictText(fs.readFileSync(abs)) : null, mode: st.mode & 0o111 ? '100755' : '100644' };
|
||||
} catch { return { text: null, mode: '100644' }; }
|
||||
}
|
||||
|
||||
/** Tracked files matching a registry entry's output globs. */
|
||||
function outputsOf(all: readonly string[], globs: readonly string[]): string[] { return all.filter(f => matchesAny(globs, f)); }
|
||||
|
||||
@@ -83,7 +110,9 @@ export function selectLanes(ctx: PregateContext): Selection {
|
||||
const noLane: string[] = [];
|
||||
const emptyLane: string[] = [];
|
||||
const inert: string[] = [];
|
||||
const stampOnly: string[] = [];
|
||||
const seeds = new Map<string, string>();
|
||||
const touchedByPath = new Map(touched.map(t => [t.path, t]));
|
||||
for (const t of touched) { seeds.set(t.path, t.status === 'D' ? 'deleted' : 'touched'); if (t.renamedFrom) seeds.set(t.renamedFrom, `renamed to ${t.path}`); }
|
||||
for (const e of ctx.registry.entries) {
|
||||
if (!touched.some(t => matchesAny(e.inputs, t.path) || matchesAny(e.outputs, t.path))) continue;
|
||||
@@ -118,6 +147,18 @@ export function selectLanes(ctx: PregateContext): Selection {
|
||||
provenance.push(`${file}: no-tests (asset or tool config no runner loads)`);
|
||||
continue;
|
||||
}
|
||||
// A release manifest the branch only re-stamped: nothing a test could observe changed, so no lane is owed (gbrain #7625). Never under a test, fixture, golden or pin tree: a test may compare against that JSON by a path the graph cannot see.
|
||||
const entry = touchedByPath.get(file);
|
||||
if (entry && entry.status === 'M' && !entry.renamedFrom && stampFileKind(file) && !isProtectedTree(file)) {
|
||||
const after = readWorkingTree(ctx.repoRoot, file);
|
||||
const verdict = versionStampOnly(file, after.text === null ? null : readAtMergeBase(ctx, file, after.mode), after.text);
|
||||
if (verdict.stampOnly) {
|
||||
stampOnly.push(file);
|
||||
provenance.push(`${file}: version-stamp-only against ${ctx.mergeBase.slice(0, 12)} (${verdict.changed.join('; ')}); no lane needed`);
|
||||
continue;
|
||||
}
|
||||
provenance.push(`${file}: not version-stamp-only (${verdict.reason})`);
|
||||
}
|
||||
noLane.push(file);
|
||||
}
|
||||
const laneSets: Record<string, string[] | '?'> = {};
|
||||
@@ -141,7 +182,7 @@ export function selectLanes(ctx: PregateContext): Selection {
|
||||
if (anyLocalAll || ctx.lanes.filter(l => l.runsTests && !l.manual && l.tier !== 'platform').length === 0) for (const t of selected) local.add(t);
|
||||
for (const t of selected) if (!local.has(t) && !remote.some(r => r.tests.includes(t))) local.add(t);
|
||||
if (local.size && tiers.size === 0) tiers.add('unit');
|
||||
return { noLane: noLane.sort(), emptyLane: emptyLane.sort(), inert: inert.sort(), tests: selected, localTests: [...local].sort(), remote, tiersRan: [...tiers].sort(), laneSets, provenance };
|
||||
return { noLane: noLane.sort(), emptyLane: emptyLane.sort(), inert: inert.sort(), stampOnly: stampOnly.sort(), tests: selected, localTests: [...local].sort(), remote, tiersRan: [...tiers].sort(), laneSets, provenance };
|
||||
}
|
||||
|
||||
export function isTest(file: string): boolean { return isTestFile(file); }
|
||||
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* pregate version-stamp-only — a release manifest whose diff against the base
|
||||
* changes nothing but its version values needs no test lane.
|
||||
*
|
||||
* gbrain's `release:restamp` rewrites six plugin manifests, its VERSION file
|
||||
* and package.json; its base's `.gstack/pregate.json` declares no lane for
|
||||
* them, and pregate reads policy from the base, so every stamp PR failed
|
||||
* `lanes` with `PREGATE_NO_LANE` and the PR could not fix itself (gbrain #7625,
|
||||
* #7606). A stamp changes no behaviour a test could observe, so `lanes` waives
|
||||
* such a file, naming it and why, when `versionStampOnly` says the before and
|
||||
* after texts differ only as a stamp does.
|
||||
*
|
||||
* The rule is deliberately narrow. Three file kinds, each with its own exact
|
||||
* definition; anything else, a parse failure, a file absent on either side,
|
||||
* or an unchanged file is not stamp-only, and the caller falls through to
|
||||
* `PREGATE_NO_LANE` as before:
|
||||
*
|
||||
* - JSON (`*.json`): both texts parse to objects of the same shape (the same
|
||||
* own keys in the same order in every object, the same length in every
|
||||
* array, the same type at every leaf); the only differing leaf is the
|
||||
* document's top-level `version`, a bare version string on both sides
|
||||
* (`VERSION_TOKEN`: `1.2.3`, `v1.2.3.4`, `1.2.3-rc.1`); and the two texts
|
||||
* are line-for-line identical once that one changed line has its old quoted
|
||||
* version replaced by the new one. Nothing nested qualifies, so a dependency
|
||||
* pin or range, a lockfile entry, a catalog, a devcontainer feature option
|
||||
* or a package named `version` is never a stamp; nor is a changed
|
||||
* description, a new, removed or reordered key, a floor such as
|
||||
* `minHostVersion` or `schemaVersion`, a reflow, a document nested too deep
|
||||
* to walk, or any change the parser would erase (a big integer, a
|
||||
* duplicate key).
|
||||
* - plist (`*.plist`): the same number of lines; every differing line pair is
|
||||
* a `<string>` holding a bare version on both sides, and the line before it
|
||||
* is `<key>CFBundleShortVersionString</key>` or `<key>CFBundleVersion</key>`.
|
||||
* `MinimumOSVersion` and every other key are not stamps.
|
||||
* - VERSION files (basename `VERSION` or `VERSION.txt`, any case): the whole
|
||||
* content is one bare version, with at most a trailing LF, on both sides
|
||||
* (a BOM, a space, a CR or a second line is not).
|
||||
*
|
||||
* Texts are compared as they are: a line-ending change, a BOM or any other
|
||||
* byte the parser would hide is content, and is not a stamp.
|
||||
*
|
||||
* Applied by lib/pregate/lane-selection.ts only after the import graph, the
|
||||
* declarations, the regen registry and the two empty-lane classes found
|
||||
* nothing, so a manifest a test reads still selects that test; only to a
|
||||
* modified, unrenamed file; and never under a test, fixture, golden, snapshot,
|
||||
* `.gstack/` or CI-workflow tree, whose JSON a test may compare against by a
|
||||
* path the import graph cannot see. The verdict is per file: the other touched files
|
||||
* keep their own lanes or their own `PREGATE_NO_LANE`.
|
||||
*/
|
||||
|
||||
/** A bare version: optional `v`, two to four dot-separated numbers, an optional pre-release or build suffix. No range operators. */
|
||||
export const VERSION_TOKEN = /^v?\d+(?:\.\d+){1,3}(?:[-+][0-9A-Za-z.-]+)?$/;
|
||||
/** The one JSON key a stamp may change, at the document root only. */
|
||||
const JSON_VERSION_KEY = 'version';
|
||||
/** The plist keys a stamp may change. */
|
||||
const PLIST_VERSION_KEYS = new Set(['CFBundleShortVersionString', 'CFBundleVersion']);
|
||||
const VERSION_FILE = /^VERSION(?:\.txt)?$/i;
|
||||
const PLIST_STRING = /^\s*<string>([^<]*)<\/string>\s*$/;
|
||||
const PLIST_KEY = /^\s*<key>([^<]*)<\/key>\s*$/;
|
||||
|
||||
export type StampVerdict = { stampOnly: true; changed: string[] } | { stampOnly: false; reason: string };
|
||||
|
||||
const not = (reason: string): StampVerdict => ({ stampOnly: false, reason });
|
||||
|
||||
/** Which definition applies to `rel`; null when none does. */
|
||||
export function stampFileKind(rel: string): 'json' | 'plist' | 'version' | null {
|
||||
const base = rel.replace(/\\/g, '/').split('/').pop() ?? rel;
|
||||
if (VERSION_FILE.test(base)) return 'version';
|
||||
if (/\.json$/i.test(base)) return 'json';
|
||||
if (/\.plist$/i.test(base)) return 'plist';
|
||||
return null;
|
||||
}
|
||||
|
||||
type Json = null | boolean | number | string | Json[] | { [k: string]: Json };
|
||||
interface VersionChange { path: string; from: string; to: string }
|
||||
|
||||
function kindOf(v: Json): string {
|
||||
if (v === null) return 'null';
|
||||
if (Array.isArray(v)) return 'array';
|
||||
return typeof v;
|
||||
}
|
||||
|
||||
const isObject = (v: Json): v is { [k: string]: Json } => v !== null && typeof v === 'object' && !Array.isArray(v);
|
||||
|
||||
/**
|
||||
* Walks `a` and `b` together. `key` is the own key `a` sits under (null at the
|
||||
* root and inside arrays), `at` the display path, `depth` how many containers
|
||||
* enclose it (1 for a root object's own values). Returns the first structural
|
||||
* difference as a reason, or null after recording the version change.
|
||||
*/
|
||||
function jsonDiff(a: Json, b: Json, key: string | null, at: string, depth: number, changed: VersionChange[]): string | null {
|
||||
const here = at || '$';
|
||||
if (kindOf(a) !== kindOf(b)) return `${here} changed type (${kindOf(a)} → ${kindOf(b)})`;
|
||||
if (Array.isArray(a) && Array.isArray(b)) {
|
||||
if (a.length !== b.length) return `${here} changed length (${a.length} → ${b.length})`;
|
||||
for (let i = 0; i < a.length; i++) { const r = jsonDiff(a[i]!, b[i]!, null, `${at}[${i}]`, depth + 1, changed); if (r) return r; }
|
||||
return null;
|
||||
}
|
||||
if (isObject(a) && isObject(b)) {
|
||||
const ka = Object.keys(a); const kb = Object.keys(b);
|
||||
for (const k of ka) if (!Object.hasOwn(b, k)) return `${at}.${k} removed`;
|
||||
for (const k of kb) if (!Object.hasOwn(a, k)) return `${at}.${k} added`;
|
||||
for (let i = 0; i < ka.length; i++) if (ka[i] !== kb[i]) return `${here} keys reordered (${ka[i]} → ${kb[i]} at position ${i})`;
|
||||
for (const k of ka) { const r = jsonDiff(a[k]!, b[k]!, k, `${at}.${k}`, depth + 1, changed); if (r) return r; }
|
||||
return null;
|
||||
}
|
||||
if (a === b) return null;
|
||||
if (typeof a !== 'string' || typeof b !== 'string') return `${here} changed and is not a version string`;
|
||||
if (key !== JSON_VERSION_KEY) return `${here} changed and its key is not \`${JSON_VERSION_KEY}\``;
|
||||
if (depth !== 1) return `${here} changed and is not the top-level \`${JSON_VERSION_KEY}\``;
|
||||
if (!VERSION_TOKEN.test(a) || !VERSION_TOKEN.test(b)) return `${here} changed and a value is not a bare version (${JSON.stringify(a)} → ${JSON.stringify(b)})`;
|
||||
changed.push({ path: at.replace(/^\./, ''), from: a, to: b });
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The texts must differ on exactly one line per recorded version change, each
|
||||
* such line being the old line with that change's old quoted version replaced
|
||||
* by the new one, each change consumed once. Rejects what the parser erased:
|
||||
* reflow, a shadowed duplicate key, number spellings, escape spellings.
|
||||
*/
|
||||
function textDiffersOnlyByVersions(a: string, b: string, changed: readonly VersionChange[]): string | null {
|
||||
const la = a.split('\n'); const lb = b.split('\n');
|
||||
if (la.length !== lb.length) return `line count changed (${la.length} → ${lb.length})`;
|
||||
const unused = [...changed];
|
||||
for (let i = 0; i < la.length; i++) {
|
||||
if (la[i] === lb[i]) continue;
|
||||
const k = unused.findIndex(c => la[i]!.includes(JSON.stringify(c.from)) && la[i]!.replace(JSON.stringify(c.from), JSON.stringify(c.to)) === lb[i]);
|
||||
if (k < 0) return `line ${i + 1} changed beyond a version value`;
|
||||
unused.splice(k, 1);
|
||||
}
|
||||
return unused.length ? `${unused[0]!.path} changed in the document but on no line of its own` : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `rel` changed from `before` to `after` as a version stamp and nothing
|
||||
* else, per the file-kind definitions above. `before` is the file's text at the
|
||||
* merge base, `after` its working-tree text; null on either side (absent,
|
||||
* unreadable, not valid UTF-8, or over the caller's size cap) is never a stamp.
|
||||
*/
|
||||
export function versionStampOnly(rel: string, before: string | null, after: string | null): StampVerdict {
|
||||
const kind = stampFileKind(rel);
|
||||
if (!kind) return not('not a JSON, plist or VERSION file');
|
||||
if (after === null) return not('absent or unreadable in the working tree');
|
||||
if (before === null) return not('absent or unreadable at the merge base');
|
||||
const a = before; const b = after;
|
||||
if (a === b) return not('unchanged');
|
||||
if (kind === 'version') {
|
||||
const x = a.replace(/\n$/, ''); const y = b.replace(/\n$/, '');
|
||||
if (!VERSION_TOKEN.test(x) || !VERSION_TOKEN.test(y)) return not('content is not one bare version on both sides');
|
||||
if (x === y) return not('no version changed (trailing newline only)');
|
||||
return { stampOnly: true, changed: [`${x} → ${y}`] };
|
||||
}
|
||||
if (kind === 'plist') {
|
||||
const la = a.split('\n'); const lb = b.split('\n');
|
||||
if (la.length !== lb.length) return not(`line count changed (${la.length} → ${lb.length})`);
|
||||
const changed: string[] = [];
|
||||
for (let i = 0; i < la.length; i++) {
|
||||
if (la[i] === lb[i]) continue;
|
||||
const ma = PLIST_STRING.exec(la[i]!); const mb = PLIST_STRING.exec(lb[i]!);
|
||||
if (!ma || !mb) return not(`line ${i + 1} changed and is not a <string> value`);
|
||||
if (!VERSION_TOKEN.test(ma[1]!) || !VERSION_TOKEN.test(mb[1]!)) return not(`line ${i + 1} changed and a value is not a bare version`);
|
||||
const key = i > 0 ? PLIST_KEY.exec(la[i - 1]!)?.[1] : undefined;
|
||||
if (key === undefined || !PLIST_VERSION_KEYS.has(key)) return not(`line ${i + 1} changed under ${key === undefined ? 'no <key>' : `<key>${key}</key>`}, not a bundle version key`);
|
||||
changed.push(`${key} ${ma[1]} → ${mb[1]}`);
|
||||
}
|
||||
return changed.length ? { stampOnly: true, changed } : not('no line changed');
|
||||
}
|
||||
let ja: Json; let jb: Json;
|
||||
try { ja = JSON.parse(a); jb = JSON.parse(b); } catch (e) { return not(`not parseable JSON on both sides (${(e as Error).message})`); }
|
||||
if (!isObject(ja) || !isObject(jb)) return not('not a JSON object on both sides');
|
||||
const changed: VersionChange[] = [];
|
||||
let reason: string | null;
|
||||
try { reason = jsonDiff(ja, jb, null, '', 0, changed); } catch (e) { return not(`could not compare (${e instanceof RangeError ? 'nested too deep' : (e as Error).message})`); }
|
||||
reason ??= changed.length ? textDiffersOnlyByVersions(a, b, changed) : 'no value changed (whitespace or spelling only)';
|
||||
if (reason) return not(reason);
|
||||
return { stampOnly: true, changed: changed.map(c => `${c.path} ${c.from} → ${c.to}`) };
|
||||
}
|
||||
+89
-1
@@ -1,7 +1,9 @@
|
||||
/**
|
||||
* redact-engine — pure scanning + auto-redaction over the shared taxonomy.
|
||||
*
|
||||
* No I/O. Deterministic. The CLI shim (`bin/gstack-redact`), the pre-push hook
|
||||
* No I/O. Deterministic given its one ambient input: `scan()` reads the
|
||||
* credential-named variables of `process.env` by value unless `processEnv` is
|
||||
* passed (GSTA-104). The CLI shim (`bin/gstack-redact`), the pre-push hook
|
||||
* (`bin/gstack-redact-prepush`), and tests all import from here.
|
||||
*
|
||||
* Key behaviors (locked in /plan-eng-review + two Codex passes):
|
||||
@@ -75,6 +77,17 @@ export interface ScanOptions {
|
||||
* literal: in a TypeScript/JSX file `session: SessionState,` is a type.
|
||||
*/
|
||||
sourcePath?: string;
|
||||
/**
|
||||
* The environment whose credential-named variables (a name whose last
|
||||
* `_`-separated word is KEY, TOKEN, SECRET, PASSWORD, PASSWD, PASS, PAT or
|
||||
* APIKEY, or that ends in _PWD, any case, carrying no PUBLIC word; see
|
||||
* `credentialEnvValues`) are scanned for BY VALUE: wherever one of those
|
||||
* values appears in the text it is an `env.value` HIGH finding, whatever its
|
||||
* shape, so a value that reached the text (a dumped environment in a tool
|
||||
* output, GSTA-96) never reaches a PR body, receipt, report or Drive file.
|
||||
* Defaults to `process.env`; pass `{}` to scan without one.
|
||||
*/
|
||||
processEnv?: Record<string, string | undefined>;
|
||||
}
|
||||
|
||||
export interface ScanResult {
|
||||
@@ -460,6 +473,46 @@ function scanInternal(input: string, opts: ScanOptions, spans?: Map<Finding, Ori
|
||||
}
|
||||
}
|
||||
|
||||
// Environment values, by exact value: a credential that is already in the
|
||||
// process environment needs no shape to be recognized.
|
||||
for (const { name, value: raw } of credentialEnvValues(opts.processEnv ?? process.env)) {
|
||||
// The text was normalized before matching; the value takes the same form,
|
||||
// and the allowlist and placeholder rules see both forms.
|
||||
const value = normalizeOriginal(raw).normalized;
|
||||
if (allow.has(raw) || allow.has(value) || value.length < MIN_ENV_VALUE_LENGTH || isPlaceholderSpan(value)) continue;
|
||||
let from = 0;
|
||||
let normOffset: number;
|
||||
while ((normOffset = normalized.indexOf(value, from)) !== -1) {
|
||||
from = normOffset + value.length;
|
||||
const origOffset = map[Math.min(normOffset, map.length - 1)] ?? 0;
|
||||
// Keyed on the span, not the offset alone: when one credential's value is
|
||||
// a prefix of another's, both findings exist and their spans coalesce.
|
||||
const key = `env.value:${origOffset}:${value.length}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
starts ??= lineStarts(input);
|
||||
const { line, col } = lineColAt(starts, input, origOffset);
|
||||
const finding: Finding = {
|
||||
id: "env.value",
|
||||
tier: "HIGH",
|
||||
severity: "HIGH",
|
||||
category: "secret",
|
||||
description: `Value of the ${name} environment variable`,
|
||||
line,
|
||||
col,
|
||||
// A live credential: the name already says what leaked; show no bytes.
|
||||
preview: "********",
|
||||
autoRedactable: false,
|
||||
repoVisibility,
|
||||
};
|
||||
findings.push(finding);
|
||||
if (spans) {
|
||||
const end = spanEnds?.[normOffset + value.length - 1];
|
||||
if (end !== undefined && end > origOffset) spans.set(finding, { start: origOffset, end });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stable order: by line, then col, then id.
|
||||
findings.sort((a, b) => a.line - b.line || a.col - b.col || a.id.localeCompare(b.id));
|
||||
|
||||
@@ -469,6 +522,41 @@ function scanInternal(input: string, opts: ScanOptions, spans?: Map<Finding, Ori
|
||||
return { findings, counts, repoVisibility, oversize: false };
|
||||
}
|
||||
|
||||
const CREDENTIAL_ENV_NAME = /(^|_)(KEY|TOKEN|SECRET|PASSWORD|PASSWD|PASS|PAT|APIKEY)$|_PWD$/i;
|
||||
/** Framework-public keys (NEXT_PUBLIC_*_KEY, VITE_PUBLIC_*, *_PUBLIC_KEY) are committed on purpose. */
|
||||
const PUBLIC_ENV_NAME = /(^|_)PUBLIC(_|$)/i;
|
||||
/** gstack's own credential-named variables whose values are not secrets (bin/gstack-brain-lock.sh sets the lock token). */
|
||||
export const NON_SECRET_ENV_NAMES: ReadonlySet<string> = new Set(["GSTACK_BRAIN_LOCK_TOKEN"]);
|
||||
const MIN_ENV_VALUE_LENGTH = 8;
|
||||
|
||||
/**
|
||||
* The (name, value) pairs of `env` worth scanning for by value: credential-named
|
||||
* variables (the last `_`-separated word is KEY, TOKEN, SECRET, PASSWORD, PASSWD,
|
||||
* PASS, PAT or APIKEY, or the name ends in _PWD; names carrying a PUBLIC word and
|
||||
* gstack's own non-secret tokens are exempt) whose trimmed value is long enough to
|
||||
* be a credential and is not a placeholder. A short or placeholder value would
|
||||
* match ordinary prose. There is no path exemption on purpose: a shape rule
|
||||
* exempted base64 secrets that start with `/`, and an existence check let the
|
||||
* scanned agent exempt a value by creating a path of that name; a path-valued
|
||||
* SSL_KEY that must appear in a text goes through `allowlist` where a caller
|
||||
* passes one (`gstack-redact`, the pre-push hook), else the variable is set
|
||||
* empty for that one command. The value is
|
||||
* trimmed, so a key read from a file with a trailing newline still matches its
|
||||
* leaked form; it is matched against the normalized text in its own normalized
|
||||
* form, so a value NFKC rewrites still finds its leaked bytes.
|
||||
*/
|
||||
export function credentialEnvValues(env: Record<string, string | undefined>): Array<{ name: string; value: string }> {
|
||||
const out: Array<{ name: string; value: string }> = [];
|
||||
for (const [name, raw] of Object.entries(env)) {
|
||||
if (!CREDENTIAL_ENV_NAME.test(name) || PUBLIC_ENV_NAME.test(name) || NON_SECRET_ENV_NAMES.has(name)) continue;
|
||||
const value = raw?.trim() ?? "";
|
||||
if (value.length < MIN_ENV_VALUE_LENGTH) continue;
|
||||
if (isPlaceholderSpan(value)) continue;
|
||||
out.push({ name, value });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function withFlags(flags: string): string {
|
||||
let f = flags;
|
||||
if (!f.includes("g")) f += "g";
|
||||
|
||||
+13
-3
@@ -380,9 +380,19 @@ function runStages(o: RestampOptions, plan: Plan, staging: string, since: string
|
||||
const r = reheadChangelog(fs.readFileSync(clPath, 'utf8'), { from, to: plan.newVersion, date, file: clRel! });
|
||||
if (r.error) { cl.status = 'failed'; cl.detail = r.error.message; throw new Refusal(r.error.code, `${clRel}: ${r.error.message}; nothing written to the branch`); }
|
||||
if (r.moved) fs.writeFileSync(clPath, r.text);
|
||||
const check = checkChangelog(fs.readFileSync(clPath, 'utf8'), plan.newVersion, clRel!);
|
||||
if (check.length) { cl.status = 'failed'; cl.detail = check[0]!.message; throw new Refusal(check[0]!.code, `${check[0]!.path}: ${check[0]!.message}; nothing written to the branch`); }
|
||||
cl.status = 'done'; cl.detail = r.moved ? `${r.from} -> ${r.to}` : `already headed ${plan.newVersion}`;
|
||||
// Inherited duplicates are allowed at the larger count of two base readings: the base tip and
|
||||
// `since`, the commit the branch's own lines are counted from (its fork point, or the predecessor's
|
||||
// head in a gate-ahead stamp), so neither a duplicate main collapsed after the fork nor one it
|
||||
// gained since refuses the PR. A reading the base cannot show (the PR adds the file) is named.
|
||||
const readings = [...new Set([plan.baseSha, since])].map(sha => ({ sha, cl: git(o.repoRoot, ['show', `${sha}:${clRel}`]) }));
|
||||
const readable = readings.filter(x => x.cl.ok);
|
||||
const unreadable = readings.filter(x => !x.cl.ok).map(x => `${x.sha.slice(0, 12)}:${clRel}`);
|
||||
const baseNote = unreadable.length === 0 ? '' : readable.length === 0
|
||||
? `; base ${clRel} unreadable at ${unreadable.join(', ')}, so every duplicate heading counts`
|
||||
: `; base ${clRel} unreadable at ${unreadable.join(', ')}, inherited headings counted at ${readable.map(x => x.sha.slice(0, 12)).join(', ')} only`;
|
||||
const check = checkChangelog(fs.readFileSync(clPath, 'utf8'), plan.newVersion, clRel!, readable.map(x => x.cl.out + '\n'));
|
||||
if (check.length) { cl.status = 'failed'; cl.detail = check[0]!.message + baseNote; throw new Refusal(check[0]!.code, `${check[0]!.path}: ${check[0]!.message}${baseNote}; nothing written to the branch`); }
|
||||
cl.status = 'done'; cl.detail = (r.moved ? `${r.from} -> ${r.to}` : `already headed ${plan.newVersion}`) + baseNote;
|
||||
} else { cl.status = 'skipped'; cl.detail = clRel ? `${clRel} absent` : 'changelog: null'; }
|
||||
injected('changelog');
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* Result codes for gstack's Capy adaptations: the per-thread approval record
|
||||
* (gstack-approvals), the Captain brief (gstack-owner-brief render --captain)
|
||||
* and PR screenshots (gstack-pr-shots). Kept beside lib/result-codes.ts (which
|
||||
* (gstack-approvals), the Captain brief (gstack-owner-brief render --captain),
|
||||
* PR screenshots (gstack-pr-shots) and the installer's launch file. Kept beside lib/result-codes.ts (which
|
||||
* spreads this table into RESULT_CODES) so the owner module stays within the
|
||||
* module-size ratchet; the same rules apply: one stable docs/troubleshooting.md
|
||||
* anchor, a summary and a fix clause each.
|
||||
@@ -64,4 +64,9 @@ export const CAPY_RESULT_CODES = {
|
||||
summary: 'a browser was available but rendering shot.html failed',
|
||||
fix: 'read the renderer transcript the line names; screenshot shot.html with your own browser at 880px width, or use the text fallback in text.md',
|
||||
},
|
||||
LAUNCH_FILE_UNWRITABLE: {
|
||||
anchor: 'capy-install-launch-file-unwritable',
|
||||
summary: 'the install passed but its launch file (the sourceable GSTACK_LAUNCH_FILE beside the verdict) could not be written, so the run is a fail: no caller is handed only the launch line to re-type',
|
||||
fix: 'make the verdict directory under the state root writable, or set GSTACK_STATE_ROOT to a writable directory, then re-run bin/gstack-capy-install',
|
||||
},
|
||||
} satisfies Record<string, ResultCode>;
|
||||
@@ -132,6 +132,22 @@ export const RELEASE_RESULT_CODES = {
|
||||
summary: 'a train stopped partway through publication, so main holds a prefix no gate covered on its own; ordinary trains are refused until a repair lands',
|
||||
fix: 'admit one repair PR at its exact head (train build --repair --prs <n>@<sha>); the freeze clears when verify finds main’s tree after that merge in a green record',
|
||||
},
|
||||
// CREW rule 22 (Garry, 2026-10-25): /ship runs before every READY; the queue refuses a PR whose body has no current receipt.
|
||||
QUEUE_RECEIPT_MISSING: {
|
||||
anchor: 'queue-receipt-missing',
|
||||
summary: 'the PR body carries no valid /ship receipt, so the PR was never shipped through gstack and cannot be READY’d, admitted to a train or merged',
|
||||
fix: 'run /ship on the branch (unattended it writes the receipt into an open PR at HEAD with gstack-ship-receipt write --pr <n> --body …, else leaves it in .gstack/tmp/ship-receipt.md for the parent to post), then gstack-ship-queue ready again; there is no flag to skip this',
|
||||
},
|
||||
QUEUE_RECEIPT_STALE: {
|
||||
anchor: 'queue-receipt-stale',
|
||||
summary: 'the PR body’s /ship receipt is for another head than the one being READY’d or merged (a push after /ship, or a receipt copied from another PR)',
|
||||
fix: 'rerun /ship on the current head so the receipt names it, then gstack-ship-queue ready again; a slot’s stamp commit keeps the receipt written for its pinned head',
|
||||
},
|
||||
QUEUE_SCREEN_STALE_REF: {
|
||||
anchor: 'queue-screen-stale-ref',
|
||||
summary: 'GitHub’s refs/pull/<n>/merge is missing or still merges an older head than the PR’s: the PR conflicts with the base (a CHANGELOG-only conflict counts), so a screen dispatched on it would plan no cases and report success',
|
||||
fix: 'resolve the conflict with the base branch (or wait for GitHub to recompute the merge ref), push, then gstack-ship-queue ready --pr <n> --screen again; a lane never READYs on a screen that planned nothing',
|
||||
},
|
||||
READY_SCREEN_CHOICE_REQUIRED: {
|
||||
anchor: 'ready-screen-choice-required',
|
||||
summary: 'the base policy turns trains on, so READY needs an explicit screen choice: --screen (pay for a diff-relevant screen now) or --no-screen',
|
||||
|
||||
+3
-3
@@ -226,8 +226,8 @@ export const RESULT_CODES = {
|
||||
},
|
||||
CHANGELOG_DUPLICATE_HEADING: {
|
||||
anchor: 'changelog-duplicate-heading',
|
||||
summary: 'two CHANGELOG sections carry the same version heading',
|
||||
fix: 'merge the duplicate sections into one entry under the new version',
|
||||
summary: 'two CHANGELOG sections carry the same version heading, beyond any duplicate the base branch already carries',
|
||||
fix: 'merge the duplicate sections the PR introduced into one entry under the new version',
|
||||
},
|
||||
CHANGELOG_TOP_MISMATCH: {
|
||||
anchor: 'changelog-top-mismatch',
|
||||
@@ -385,7 +385,7 @@ export const RESULT_CODES = {
|
||||
PREGATE_NO_LANE: {
|
||||
anchor: 'pregate-no-lane',
|
||||
summary: 'a touched source file maps to no test lane through imports, repo-path literals, the bin/<name> convention or a declaration',
|
||||
fix: 'add a test that names the file, or declare its lane (or an explicit empty lane) under `dependencies` in .gstack/pregate.json. Markdown no test depends on is already a docs/no-tests empty lane; other documentation goes in `empty_lanes`',
|
||||
fix: 'add a test that names the file, or declare its lane (or an explicit empty lane) under `dependencies` in .gstack/pregate.json. Markdown no test depends on is already a docs/no-tests empty lane; other documentation goes in `empty_lanes`; a release manifest whose only change against the merge base is its version values is waived as version-stamp-only',
|
||||
},
|
||||
PREGATE_LANE_FAILED: {
|
||||
anchor: 'pregate-lane-failed',
|
||||
|
||||
@@ -5,11 +5,12 @@
|
||||
* full-gate check run. Dispatch itself goes through scripts/eval-bg.ts.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import type { Exec, RunSummary } from '../scripts/eval-bg';
|
||||
import type { QueueContext } from './ship-queue';
|
||||
|
||||
export interface PrInfo { headRefOid: string; isCrossRepository: boolean; state: string; headRefName: string }
|
||||
export interface PrInfo { headRefOid: string; isCrossRepository: boolean; state: string; headRefName: string; body: string }
|
||||
export interface RunInfo { databaseId: number; status: string; conclusion: string | null; url: string; headSha: string; event: string; displayTitle: string }
|
||||
export interface CheckInfo { status: string; conclusion: string | null }
|
||||
|
||||
@@ -33,10 +34,10 @@ export function repoSlug(ctx: QueueContext): string | null {
|
||||
}
|
||||
|
||||
export function prInfo(ctx: QueueContext, pr: string): PrInfo | { error: string } {
|
||||
const r = ctx.gh(['pr', 'view', pr, '--json', 'headRefOid,isCrossRepository,state,headRefName']);
|
||||
const r = ctx.gh(['pr', 'view', pr, '--json', 'headRefOid,isCrossRepository,state,headRefName,body']);
|
||||
const p = json<Partial<PrInfo>>(r);
|
||||
if (!p?.headRefOid) return { error: `cannot read PR #${pr} through gh (${(r.err ?? r.out).trim().slice(0, 120) || 'no headRefOid'})` };
|
||||
return { headRefOid: p.headRefOid, isCrossRepository: p.isCrossRepository !== false, state: p.state ?? 'OPEN', headRefName: p.headRefName ?? '' };
|
||||
return { headRefOid: p.headRefOid, isCrossRepository: p.isCrossRepository !== false, state: p.state ?? 'OPEN', headRefName: p.headRefName ?? '', body: p.body ?? '' };
|
||||
}
|
||||
|
||||
const RUN_FIELDS = 'databaseId,status,conclusion,url,headSha,event,displayTitle';
|
||||
@@ -56,18 +57,49 @@ export function asRun(summary: RunSummary): { id: number; url: string } {
|
||||
return { id: summary.databaseId, url: summary.url };
|
||||
}
|
||||
|
||||
/** One JSON artifact of a run (the first .json file it holds); null when it is missing or unreadable. */
|
||||
export function downloadJson(ctx: QueueContext, slug: string, run: number, name: string, scratch: string): unknown {
|
||||
/** One JSON artifact of a run (the named file, else the first .json file it holds); null when it is missing or unreadable. */
|
||||
export function downloadJson(ctx: QueueContext, slug: string, run: number, name: string, scratch: string, file?: string): unknown {
|
||||
const dir = path.join(scratch, `${run}-${name}`);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
try {
|
||||
if (!ctx.gh(['run', 'download', String(run), '--repo', slug, '-n', name, '-D', dir]).ok) return null;
|
||||
const file = fs.readdirSync(dir).find(f => f.endsWith('.json'));
|
||||
return file ? JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')) : null;
|
||||
const pick = file ?? fs.readdirSync(dir).find(f => f.endsWith('.json'));
|
||||
return pick && fs.existsSync(path.join(dir, pick)) ? JSON.parse(fs.readFileSync(path.join(dir, pick), 'utf8')) : null;
|
||||
} catch { return null; } finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
/**
|
||||
* A finished screen run that planned no cases is not a screen: when the merge
|
||||
* ref it was dispatched for was superseded, evals.yml skips plan-slices and
|
||||
* concludes success (GSTA-95 on #3431). A screen run uploads no train record,
|
||||
* so the evidence is the plan itself: no `paid-plan` artifact, or a manifest
|
||||
* with nothing planned. `hollow` carries the reason; `live` is a screen that
|
||||
* planned cases or is still running; `unknown` names what could not be read
|
||||
* (the caller decides, and says, how it counts).
|
||||
*/
|
||||
export type ScreenPlan = { kind: 'hollow'; reason: string } | { kind: 'live' } | { kind: 'unknown'; reason: string };
|
||||
export function hollowScreen(ctx: QueueContext, slug: string, run: number): ScreenPlan {
|
||||
if (!Number.isSafeInteger(run) || run <= 0) return { kind: 'unknown', reason: `screen run id ${String(run).slice(0, 40)} is not a run number` };
|
||||
const view = runView(ctx, slug, run);
|
||||
if (!view) return { kind: 'unknown', reason: `run ${run} is unreadable` };
|
||||
if (view.status !== 'completed') return { kind: 'live' };
|
||||
const list = json<{ artifacts?: Array<{ name?: string; expired?: boolean }> }>(ctx.gh(['api', `repos/${slug}/actions/runs/${run}/artifacts?name=${PLAN_ARTIFACT}&per_page=10`]));
|
||||
if (!Array.isArray(list?.artifacts)) return { kind: 'unknown', reason: `run ${run}'s artifact list is unreadable` };
|
||||
const plan = list.artifacts.find(a => a.name === PLAN_ARTIFACT);
|
||||
if (!plan) return { kind: 'hollow', reason: `run ${run} planned no cases (no ${PLAN_ARTIFACT} artifact; concluded ${view.conclusion ?? 'unknown'})` };
|
||||
if (plan.expired) return { kind: 'unknown', reason: `run ${run}'s ${PLAN_ARTIFACT} artifact has expired` };
|
||||
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-screen-'));
|
||||
let manifest: { entries?: Array<{ status?: unknown }> } | null;
|
||||
try { manifest = downloadJson(ctx, slug, run, PLAN_ARTIFACT, scratch, 'manifest.json') as typeof manifest; } finally { fs.rmSync(scratch, { recursive: true, force: true }); }
|
||||
if (!Array.isArray(manifest?.entries)) return { kind: 'unknown', reason: `run ${run}'s ${PLAN_ARTIFACT} manifest.json is unreadable` };
|
||||
const planned = manifest.entries.filter(e => e?.status === 'planned').length;
|
||||
return planned === 0 ? { kind: 'hollow', reason: `run ${run} planned no cases (an empty plan manifest; concluded ${view.conclusion ?? 'unknown'})` } : { kind: 'live' };
|
||||
}
|
||||
|
||||
/** The plan manifest every evals.yml run that planned anything uploads (plan-slices); a superseded screen never reaches that job. */
|
||||
const PLAN_ARTIFACT = 'paid-plan';
|
||||
|
||||
/** The named check run on a commit (the newest when several); null when absent or unreadable. */
|
||||
export function checkRun(ctx: QueueContext, sha: string, name: string): CheckInfo | null {
|
||||
const r = json<{ check_runs?: Array<{ name?: string; status?: string; conclusion?: string | null }> }>(
|
||||
|
||||
@@ -16,7 +16,7 @@ import { EXIT } from './headless-artifacts';
|
||||
import { ready, type QueueContext } from './ship-queue';
|
||||
import { classifyTrainHead } from './version-gate-train';
|
||||
import { git } from './ship-queue-gate';
|
||||
import { execFor, repoSlug } from './ship-queue-train-github';
|
||||
import { execFor, hollowScreen, repoSlug } from './ship-queue-train-github';
|
||||
import { loadReady, nowIso, persist, readyPath, type ReadyReceipt, type ScreenChoice } from './ship-queue-train-record';
|
||||
import { outOk, outPending, outRefused, trainEnv, type TrainEnv, type TrainOut } from './ship-queue-train';
|
||||
|
||||
@@ -85,10 +85,12 @@ async function dispatchScreen(env: TrainEnv, o: ReadyOptions, mergeSha: string,
|
||||
if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, [], `${Q} ready --pr ${o.pr} --screen`, 'none');
|
||||
const exec = execFor(env.ctx);
|
||||
const existing = findDuplicateRun(listDispatchRuns(dispatch.workflow, exec, slug) ?? [], dedupeKey('pr', mergeSha, baseSha, scope));
|
||||
if (existing) return { choice: { choice: 'screen', policy, nonce: /eval-bg (\S+?)\]/.exec(existing.displayTitle)?.[1] ?? nonce, run: existing.databaseId, merge_sha: mergeSha }, line: `SCREEN_DISPATCH: following run ${existing.databaseId} (same merge SHA, base and policy; no duplicate)` };
|
||||
const plan = existing ? hollowScreen(env.ctx, slug, existing.databaseId) : null;
|
||||
const hollow = plan?.kind === 'hollow' ? plan.reason : null;
|
||||
if (existing && !hollow) return { choice: { choice: 'screen', policy, nonce: /eval-bg (\S+?)\]/.exec(existing.displayTitle)?.[1] ?? nonce, run: existing.databaseId, merge_sha: mergeSha }, line: `SCREEN_DISPATCH: following run ${existing.databaseId} (same merge SHA, base and policy; no duplicate)` };
|
||||
const found = dispatchWorkflow(dispatch, env.ctx.base, exec, slug) ? await awaitNonceRun(dispatch.workflow, nonce, exec, slug) : null;
|
||||
if (!found) return outPending({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `no ${dispatch.workflow} run named ${nonce} appeared after the screen dispatch` }, [], `${Q} ready --pr ${o.pr} --screen${o.broad ? ' --broad' : ''} (adopts the run by its key once it appears)`, 'none');
|
||||
return { choice: { choice: 'screen', policy, nonce, run: found.databaseId, merge_sha: mergeSha }, line: `SCREEN_DISPATCH: ${dispatch.workflow} run ${found.databaseId} ${found.url} nonce ${nonce}` };
|
||||
return { choice: { choice: 'screen', policy, nonce, run: found.databaseId, merge_sha: mergeSha }, line: `SCREEN_DISPATCH: ${dispatch.workflow} run ${found.databaseId} ${found.url} nonce ${nonce}${hollow ? ` (not following ${hollow}: a screen that planned nothing is not a screen)` : ''}` };
|
||||
}
|
||||
|
||||
export async function trainReady(ctx: QueueContext, o: ReadyOptions): Promise<TrainOut> {
|
||||
@@ -113,8 +115,14 @@ export async function trainReady(ctx: QueueContext, o: ReadyOptions): Promise<Tr
|
||||
let choice: ScreenChoice = { choice: 'no-screen', policy: null, nonce: null, run: null, merge_sha: null };
|
||||
if (o.screen === 'screen') {
|
||||
const ls = git(ctx.repoRoot, ['ls-remote', env.remote, `refs/pull/${o.pr}/merge`]);
|
||||
const mergeSha = ls.ok ? ls.out.split(/\s/)[0] ?? '' : '';
|
||||
if (!/^[0-9a-f]{40,64}$/.test(mergeSha)) return outRefused({ code: 'QUEUE_NOT_READY', message: `${env.remote} has no refs/pull/${o.pr}/merge (the PR conflicts with ${ctx.base}, or GitHub has not computed it yet)` }, lines, `merge ${ctx.base} into the branch, push, then ${Q} ready --pr ${o.pr} --screen`, 'none');
|
||||
if (!ls.ok) return outRefused({ code: 'QUEUE_SCREEN_STALE_REF', message: `could not list ${env.remote}'s refs: ${(ls.err || ls.out).trim().replace(/\s+/g, ' ').slice(0, 200) || 'git ls-remote failed'}` }, lines, `check the remote, then ${Q} ready --pr ${o.pr} --screen`, 'none');
|
||||
const mergeSha = ls.out.split(/\s/)[0] ?? '';
|
||||
const again = `resolve the conflict with ${ctx.base} (or wait for GitHub to recompute refs/pull/${o.pr}/merge), push, then ${Q} ready --pr ${o.pr} --screen; a screen dispatched on a stale merge ref plans no cases`;
|
||||
if (!/^[0-9a-f]{40,64}$/.test(mergeSha)) return outRefused({ code: 'QUEUE_SCREEN_STALE_REF', message: `${env.remote} has no refs/pull/${o.pr}/merge (the PR conflicts with ${ctx.base}, or GitHub has not computed it yet)` }, lines, again, 'none');
|
||||
const fetched = git(ctx.repoRoot, ['fetch', '-q', env.remote, `refs/pull/${o.pr}/merge`]);
|
||||
if (!fetched.ok) return outRefused({ code: 'QUEUE_SCREEN_STALE_REF', message: `could not fetch refs/pull/${o.pr}/merge from ${env.remote}: ${(fetched.err || fetched.out).trim().replace(/\s+/g, ' ').slice(0, 200) || 'git fetch failed'}` }, lines, `check the remote, then ${Q} ready --pr ${o.pr} --screen`, 'none');
|
||||
const mergeParent = git(ctx.repoRoot, ['rev-parse', '--verify', '--quiet', `${mergeSha}^2`]).out;
|
||||
if (mergeParent !== head) return outRefused({ code: 'QUEUE_SCREEN_STALE_REF', message: `${env.remote}'s refs/pull/${o.pr}/merge (${mergeSha.slice(0, 12)}) merges ${mergeParent ? mergeParent.slice(0, 12) : 'no PR head'}, not the PR head ${head.slice(0, 12)}: GitHub has not recomputed it, which is what a conflicting PR looks like (CHANGELOG-only conflicts included)` }, lines, again, 'none');
|
||||
const d = await dispatchScreen(env, o, mergeSha, baseSha);
|
||||
if ('exit' in d) return { ...d, lines: [...lines, ...d.lines] };
|
||||
choice = d.choice;
|
||||
|
||||
@@ -17,7 +17,8 @@ import { randomBytes } from 'node:crypto';
|
||||
import { DISPATCH_INPUTS, awaitNonceRun, dispatchWorkflow, makeNonce } from '../scripts/eval-bg';
|
||||
import type { QueueContext } from './ship-queue';
|
||||
import { git } from './ship-queue-gate';
|
||||
import { checkRun, execFor, repoSlug } from './ship-queue-train-github';
|
||||
import { bindReceipt } from './ship-receipt';
|
||||
import { checkRun, execFor, prInfo, repoSlug } from './ship-queue-train-github';
|
||||
import { activeFreeze, attemptsOf, nowIso, persist, trainFile, type FreezeRecord, type TrainRecord } from './ship-queue-train-record';
|
||||
import {
|
||||
effect, fetchBase, fetchTrainBranch, findTrain, frozenLines, nextFor, outFailed, outOk, outRefused, pinList, prsArgs, save, trainEnv, type TrainEnv, type TrainOut,
|
||||
@@ -132,7 +133,11 @@ export async function verifyTrain(ctx: QueueContext, o: { pr: string; train: str
|
||||
const pinned = head === t.slots[i]!.head ? { same: true, detail: 'the pinned head' } : sameModuloStamps(env, t.slots[i]!.head, head);
|
||||
const stampOnly = pinned.same ? pinned : mergedIn(env, t, i, head, main);
|
||||
if (!stampOnly.same) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr} moved past its pinned head ${t.slots[i]!.head.slice(0, 12)} and is not T${after.index} modulo stamps (${head.slice(0, 12)}: ${stampOnly.detail}; against the pin it ${pinned.detail})`, main, lines);
|
||||
return outOk([...lines, `main: ${main} = T${before.index} (${pre.detail})`, `head: ${head} (${stampOnly.detail})`,
|
||||
const info = prInfo(ctx, o.pr);
|
||||
if ('error' in info) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: info.error }, lines, `${Q} verify --pr ${o.pr} --train ${t.nonce}`, run);
|
||||
const shipped = bindReceipt(info.body, o.pr, [t.slots[i]!.head, head]);
|
||||
if (!shipped.ok) return outRefused({ code: shipped.code, message: shipped.message }, [...lines, shipped.line], `#${o.pr}'s lane runs /ship and READYs again; then ${Q} train build --prs <pins>`, run);
|
||||
return outOk([...lines, `main: ${main} = T${before.index} (${pre.detail})`, `head: ${head} (${stampOnly.detail})`, `${shipped.line} (binds ${shipped.head === t.slots[i]!.head ? 'the pinned head' : 'the current head'})`,
|
||||
`VERIFY_TRAIN: #${o.pr} may merge as slot ${i + 1} of train ${t.nonce} (green run ${gate.run?.id ?? '-'}, ${env.train.full_gate_check} success)`, `MERGE_CONDITION: sha=${head}`],
|
||||
`merge #${o.pr} with expected head ${head.slice(0, 12)} (gstack-restamp stamps at merge), then ${Q} verify --pr ${o.pr} --train ${t.nonce}`, run);
|
||||
}
|
||||
|
||||
+14
-3
@@ -21,8 +21,9 @@ import type { ResultStatus } from './headless-artifacts';
|
||||
import { versionAtRef, versionFiles } from './restamp';
|
||||
import type { LoadedPolicy, ShipPolicy, TrainPolicy } from './ship-policy';
|
||||
import { loadBasePolicy, type QueueContext } from './ship-queue';
|
||||
import { bindReceipt } from './ship-receipt';
|
||||
import { git, sha256, withIndex, type QueueFailure } from './ship-queue-gate';
|
||||
import { asRun, execFor, newestBranchRun, prInfo, remoteBranchSha, repoSlug, runView, checkRun } from './ship-queue-train-github';
|
||||
import { asRun, execFor, hollowScreen, newestBranchRun, prInfo, remoteBranchSha, repoSlug, runView, checkRun } from './ship-queue-train-github';
|
||||
import {
|
||||
TERMINAL_STATES, activeFreeze, attemptsOf, listTrains, loadBisect, loadReady, loadTrain, nowIso, persist, rebuildFromNotes, repoIdentity, trainFile,
|
||||
type AttemptPurpose, type AttemptRecord, type TrainRecord, type TrainSlot,
|
||||
@@ -284,6 +285,8 @@ function admitSlots(env: TrainEnv, pins: Array<{ pr: string; sha: string }>): {
|
||||
const f = git(ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, ...specs]);
|
||||
if (!f.ok) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `git fetch of the PR heads failed: ${f.err.slice(0, 160)}` }, [], 'none', 'none');
|
||||
const slots: TrainSlot[] = [];
|
||||
const lines: string[] = [];
|
||||
let slug: string | null | undefined;
|
||||
for (const p of pins) {
|
||||
const info = prInfo(ctx, p.pr);
|
||||
if ('error' in info) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: info.error }, [], 'none', 'none');
|
||||
@@ -293,11 +296,18 @@ function admitSlots(env: TrainEnv, pins: Array<{ pr: string; sha: string }>): {
|
||||
if (!info.headRefOid.startsWith(p.sha) || fetched !== info.headRefOid) {
|
||||
return outRefused({ code: 'QUEUE_TRAIN_MOVED', message: `PR #${p.pr} moved: pinned ${p.sha.slice(0, 12)}, head is now ${info.headRefOid.slice(0, 12)}` }, [], `ask #${p.pr}'s lane for READY on ${info.headRefOid.slice(0, 12)}, then ${Q} train build --prs <pins>`, 'none');
|
||||
}
|
||||
const shipped = bindReceipt(info.body, p.pr, [info.headRefOid]);
|
||||
if (!shipped.ok) return outRefused({ code: shipped.code, message: shipped.message }, [shipped.line], `#${p.pr}'s lane runs /ship, then ${Q} ready --pr ${p.pr} --screen|--no-screen; then ${Q} train build --prs <pins>`, 'none');
|
||||
const ready = loadReady(env.repo.keys, p.pr, info.headRefOid);
|
||||
if (!ready) return outRefused({ code: 'QUEUE_NOT_READY', message: `no READY receipt for #${p.pr} at ${info.headRefOid.slice(0, 12)} (local state or ${'refs/notes/gstack-trains'})` }, [], `the lane runs ${Q} ready --pr ${p.pr} --screen|--no-screen`, 'none');
|
||||
slots.push({ pr: p.pr, head: info.headRefOid, ready_at: ready.ready_at, screen: ready.screen });
|
||||
let screen = ready.screen;
|
||||
if (screen.choice === 'screen' && screen.run && slug === undefined) slug = repoSlug(ctx);
|
||||
const plan = screen.choice === 'screen' && screen.run ? (slug ? hollowScreen(ctx, slug, screen.run) : { kind: 'unknown' as const, reason: 'the repository slug is unreadable' }) : null;
|
||||
if (plan?.kind === 'hollow') { lines.push(`SCREEN: #${p.pr} ${plan.reason}; counted as not-run, the slot rides unscreened`); screen = { ...screen, choice: 'no-screen' }; }
|
||||
else if (plan?.kind === 'unknown') lines.push(`SCREEN: #${p.pr} ${plan.reason}; counted as screened, unverified`);
|
||||
slots.push({ pr: p.pr, head: info.headRefOid, ready_at: ready.ready_at, screen });
|
||||
}
|
||||
return { slots, lines: [] };
|
||||
return { slots, lines };
|
||||
}
|
||||
|
||||
/** Start a new train from --prs, or find the live train with the same pins and base. */
|
||||
@@ -315,6 +325,7 @@ async function newTrain(env: TrainEnv, o: BuildOptions): Promise<{ t: TrainRecor
|
||||
if ('exit' in admitted) return admitted;
|
||||
const order = orderSlots(admitted.slots, o.repair ? 1 : env.train.max_size, explicit);
|
||||
const lines = [
|
||||
...admitted.lines,
|
||||
`ORDER: ${order.train.map((s, i) => `${i + 1}:#${s.pr}(${s.screen.choice === 'screen' ? 'screened' : 'unscreened'})`).join(' ')}${explicit ? ' (explicit --order)' : ''}`,
|
||||
...(order.held.length ? [`HELD: ${order.held.map(s => '#' + s.pr).join(',')} (${order.split ? 'unscreened PRs form their own train' : `over the train size ${env.train.max_size}`})`] : []),
|
||||
];
|
||||
|
||||
+88
-13
@@ -5,8 +5,9 @@
|
||||
* so every step ends in a line a coordinator can act on:
|
||||
*
|
||||
* ready READY #<pr> <head> STAMP_VERSION=<X|unstamped> plus head, base,
|
||||
* CI at that exact head, the PREGATE line and the gate state;
|
||||
* refused when the tree is not the pushed PR head.
|
||||
* CI at that exact head, the PREGATE line, the gate state and the
|
||||
* /ship receipt; refused when the tree is not the pushed PR head
|
||||
* or the PR body carries no receipt for it (QUEUE_RECEIPT_*).
|
||||
* stamp preflight (gated head equals HEAD modulo stamps, base, policy,
|
||||
* gate record under full_e2e), restamp, commit, tree receipt,
|
||||
* each stage journaled; QUEUE_STAMP: <pr> <version> head= gated=
|
||||
@@ -23,7 +24,7 @@ import * as path from 'node:path';
|
||||
import { readPregateFile, verifyPregate, PREGATE_OUT_REL } from './pregate/index';
|
||||
import { ensureTmpExcluded, restamp, versionAtRef } from './restamp';
|
||||
import type { ResultCodeName } from './result-codes';
|
||||
import { extractReceipt, receiptSummaryLine } from './ship-receipt';
|
||||
import { bindReceipt, extractReceipt } from './ship-receipt';
|
||||
import { isUnavailable, loadPolicyFromBase, type LoadedPolicy } from './ship-policy';
|
||||
import {
|
||||
QUEUE_DIR_REL, applyStampPatch, commitForTree, currentAttempt, git, listAttempts, recomputeMergeTree, sha256, writeJsonAtomic,
|
||||
@@ -59,6 +60,37 @@ function treeReceipt(ctx: QueueContext, policy: LoadedPolicy, gated: string, cur
|
||||
return 'error' in r ? { code: 'QUEUE_PROOF_MISSING', message: r.error } : r;
|
||||
}
|
||||
|
||||
/**
|
||||
* The heads a READY accepts a /ship receipt for: HEAD, plus the receipt's own
|
||||
* head when HEAD is that commit with only stamp files changed on top (the
|
||||
* slot's stamp commit, docs/ship-policy.md#ship-before-ready), or, from slot
|
||||
* 2 on, the stamp over a merge of the base into it: /ship ran on the pin, the
|
||||
* stamp changes nothing it checked, and verify --train binds the same pair
|
||||
* against the train's own prefix. A merge with no stamp on top is a lane's
|
||||
* own push (rule 14), so it reruns /ship like any other head. Any other
|
||||
* receipt head stays QUEUE_RECEIPT_STALE.
|
||||
*/
|
||||
function acceptedReceiptHeads(ctx: QueueContext, policy: LoadedPolicy, head: string, body: string): string[] {
|
||||
const ex = extractReceipt(body);
|
||||
const r = ex.receipt;
|
||||
if (!r || ex.errors.length || typeof r.head !== 'string' || !/^[0-9a-f]{12,64}$/i.test(r.head) || head.startsWith(r.head)) return [head];
|
||||
const root = ctx.repoRoot;
|
||||
const full = git(root, ['rev-parse', '--verify', '--quiet', `${r.head}^{commit}`]);
|
||||
if (!full.ok || !git(root, ['merge-base', '--is-ancestor', full.out, head]).ok) return [head];
|
||||
const stampOnly = (from: string, to: string) => { const t = from === to ? null : treeReceipt(ctx, policy, from, to); return !t || (!('message' in t) && t.tree === 'same-modulo-stamps'); };
|
||||
if (stampOnly(full.out, head)) return [full.out, head];
|
||||
const baseSha = git(root, ['rev-parse', `origin/${ctx.base}`]).out;
|
||||
const onBase = (sha: string) => !!sha && git(root, ['merge-base', '--is-ancestor', sha, baseSha]).ok;
|
||||
const merge = git(root, ['rev-parse', '--verify', '--quiet', `${head}^1`]).out;
|
||||
const stamp = merge && merge !== head ? treeReceipt(ctx, policy, merge, head) : null;
|
||||
if (!stamp || 'message' in stamp || stamp.tree !== 'same-modulo-stamps' || stamp.files.length === 0) return [head];
|
||||
const parents = git(root, ['rev-list', '--parents', '-n', '1', merge]).out.split(' ').slice(1);
|
||||
if (parents.length !== 2) return [head];
|
||||
const pinSide = onBase(parents[0]!) ? parents[1]! : onBase(parents[1]!) ? parents[0]! : '';
|
||||
if (pinSide && git(root, ['merge-base', '--is-ancestor', full.out, pinSide]).ok && stampOnly(full.out, pinSide)) return [full.out, head];
|
||||
return [head];
|
||||
}
|
||||
|
||||
const CI_PAGE = 100;
|
||||
const CI_MAX_PAGES = 10;
|
||||
|
||||
@@ -80,21 +112,61 @@ function readAllPages(ctx: QueueContext, endpoint: string, key: 'check_runs' | '
|
||||
return { error: `${key}: more than ${CI_PAGE * CI_MAX_PAGES} entries, not all read` };
|
||||
}
|
||||
|
||||
/** CI at one exact SHA: every check run and every commit-status context; unreadable or incomplete is unknown, never green. */
|
||||
function ciAtHead(ctx: QueueContext, sha: string): { line: string; green: boolean } {
|
||||
/**
|
||||
* The latest run per check, the way GitHub's rollup counts: a check that was
|
||||
* re-run or superseded (a cancelled run beside a later pass of the same name on
|
||||
* the same commit, from a second workflow run) is judged by its newest run
|
||||
* alone. A check is one app's check name, so two apps that both report "build"
|
||||
* stay separate. Newest is the latest completed_at, then the highest id; a run
|
||||
* still in progress is newer than any finished one. A run with no name, or a
|
||||
* completed run with no readable completed_at, is never collapsed and counts on
|
||||
* its own (fail closed).
|
||||
*/
|
||||
export function latestRunPerName<T extends { name?: unknown; id?: unknown; status?: unknown; completed_at?: unknown; app?: unknown }>(runs: T[]): T[] {
|
||||
const latest = new Map<string, T>();
|
||||
const standalone: T[] = [];
|
||||
const rank = (r: T) => (r.status !== 'completed' ? Infinity : typeof r.completed_at === 'string' ? Date.parse(r.completed_at) : NaN);
|
||||
const id = (r: T) => (typeof r.id === 'number' ? r.id : Number(r.id) || 0);
|
||||
for (const r of runs) {
|
||||
const app = (r.app as { id?: unknown } | null | undefined)?.id;
|
||||
if (typeof r.name !== 'string' || !r.name || Number.isNaN(rank(r))) { standalone.push(r); continue; }
|
||||
const key = `${typeof app === 'number' || typeof app === 'string' ? app : ''}\0${r.name}`;
|
||||
const have = latest.get(key);
|
||||
if (!have || rank(r) > rank(have) || (rank(r) === rank(have) && id(r) > id(have))) latest.set(key, r);
|
||||
}
|
||||
return [...latest.values(), ...standalone];
|
||||
}
|
||||
|
||||
/** CI at one exact SHA: the latest check run per name (as GitHub's rollup counts) and every commit-status context; unreadable or incomplete is unknown, never green. */
|
||||
function ciAtHead(ctx: QueueContext, sha: string): { line: string; green: boolean; none: boolean } {
|
||||
const at = sha.slice(0, 12);
|
||||
const runs = readAllPages(ctx, `repos/{owner}/{repo}/commits/${sha}/check-runs`, 'check_runs');
|
||||
if ('error' in runs) return { line: `unknown at ${at} (${runs.error})`, green: false };
|
||||
if ('error' in runs) return { line: `unknown at ${at} (${runs.error})`, green: false, none: false };
|
||||
const statuses = readAllPages(ctx, `repos/{owner}/{repo}/commits/${sha}/status`, 'statuses');
|
||||
if ('error' in statuses) return { line: `unknown at ${at} (${statuses.error})`, green: false };
|
||||
if ('error' in statuses) return { line: `unknown at ${at} (${statuses.error})`, green: false, none: false };
|
||||
const verdicts = [
|
||||
...runs.items.map(c => c.status !== 'completed' ? 'pending' : c.conclusion === 'success' ? 'passed' : c.conclusion === 'skipped' || c.conclusion === 'neutral' ? 'skipped' : 'failed'),
|
||||
...latestRunPerName(runs.items).map(c => c.status !== 'completed' ? 'pending' : c.conclusion === 'success' ? 'passed' : c.conclusion === 'skipped' || c.conclusion === 'neutral' ? 'skipped' : 'failed'),
|
||||
...statuses.items.map(s => s.state === 'success' ? 'passed' : s.state === 'pending' ? 'pending' : 'failed'),
|
||||
];
|
||||
const count = (v: string) => verdicts.filter(x => x === v).length;
|
||||
const [passed, failed, pending, skipped] = [count('passed'), count('failed'), count('pending'), count('skipped')];
|
||||
const green = verdicts.length > 0 && pending === 0 && failed === 0;
|
||||
return { line: `${green ? 'green' : pending ? 'pending' : verdicts.length ? 'red' : 'none'} at ${at} (passed=${passed} failed=${failed} pending=${pending} skipped=${skipped})`, green };
|
||||
return { line: `${green ? 'green' : pending ? 'pending' : verdicts.length ? 'red' : 'none'} at ${at} (passed=${passed} failed=${failed} pending=${pending} skipped=${skipped})`, green, none: verdicts.length === 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* A PR that conflicts with the base (CHANGELOG-only conflicts included, which
|
||||
* the train resolves) has no merge ref, so GitHub runs no pull_request CI on
|
||||
* its head. The lane dispatches the free workflows on the branch by hand; their
|
||||
* check runs attach to the commit, so ready reads them like any other.
|
||||
*/
|
||||
function conflictHint(mergeable: unknown, ci: { green: boolean; none: boolean }, branch: string | undefined, fork: boolean, head: string): string | null {
|
||||
if (mergeable !== 'CONFLICTING') return null;
|
||||
const ref = branch && /^[\w.][\w./-]*$/.test(branch) ? `--ref ${branch}` : '--ref <branch>';
|
||||
if (ci.none && fork) return `ci-hint: PR conflicts with the base (no merge ref, so pull_request CI did not run on ${head.slice(0, 12)}) and its head is on a fork, so the free workflows cannot be dispatched here; the coordinator decides (advisory)`;
|
||||
if (ci.none) return `ci-hint: PR conflicts with the base (no merge ref, so pull_request CI did not run on ${head.slice(0, 12)}); dispatch the free workflows on the branch (gh workflow run free-tests.yml ${ref} && gh workflow run windows-free-tests.yml ${ref}), clear the pregate lanes with their run URLs (gstack-pregate clear --remote <lane> --run-url <run>), then rerun ready (advisory)`;
|
||||
if (ci.green) return `ci-hint: PR conflicts with the base (no merge ref); the runs at ${head.slice(0, 12)} were not pull_request runs (dispatched on the branch, or push-triggered), and they count (advisory)`;
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Under full_e2e: a passing gate bound to this head (the stamp journal, a gate-ahead attempt on it) or the lane's own run on it. */
|
||||
@@ -120,8 +192,8 @@ export function ready(ctx: QueueContext, o: { pr: string; pregate?: string; gate
|
||||
const head = git(ctx.repoRoot, ['rev-parse', 'HEAD']).out;
|
||||
const dirty = git(ctx.repoRoot, ['status', '--porcelain', '--untracked-files=no']).out;
|
||||
if (dirty) return fail({ code: 'QUEUE_NOT_READY', message: `uncommitted tracked changes: ${dirty.split('\n').map(l => l.slice(3)).slice(0, 8).join(', ')}` }, [], run);
|
||||
const view = ctx.gh(['pr', 'view', o.pr, '--json', 'headRefOid,url,body']);
|
||||
let info: { headRefOid?: string; url?: string; body?: string } = {};
|
||||
const view = ctx.gh(['pr', 'view', o.pr, '--json', 'headRefOid,url,body,mergeable,headRefName,isCrossRepository']);
|
||||
let info: { headRefOid?: string; url?: string; body?: string; mergeable?: string; headRefName?: string; isCrossRepository?: boolean } = {};
|
||||
try { info = view.ok ? JSON.parse(view.out) : {}; } catch {}
|
||||
if (!info.headRefOid) return fail({ code: 'QUEUE_NOT_READY', message: `cannot read PR #${o.pr}'s head through gh (${(view.err ?? view.out).trim().slice(0, 120)})` }, [], run);
|
||||
if (info.headRefOid !== head) return fail({ code: 'QUEUE_NOT_READY', message: `HEAD ${head.slice(0, 12)} is not PR #${o.pr}'s pushed head ${info.headRefOid.slice(0, 12)}` }, [], run);
|
||||
@@ -141,8 +213,9 @@ export function ready(ctx: QueueContext, o: { pr: string; pregate?: string; gate
|
||||
pregateLine = `PREGATE: ${read.file.verdict} tree=${read.file.tree.slice(0, 12)} head=${(read.file.head ?? '').slice(0, 12)} checks=${read.file.checks.length}${v.ok ? '' : ` — ${v.message} (${v.code})`}`;
|
||||
} else pregateLine = `PREGATE: unreadable (${read.errors[0]?.message ?? 'invalid'})`;
|
||||
}
|
||||
const receipt = extractReceipt(info.body ?? '');
|
||||
const receipt = bindReceipt(info.body ?? '', o.pr, acceptedReceiptHeads(ctx, policy, head, info.body ?? ''));
|
||||
const attempt = currentAttempt(ctx.repoRoot, o.pr);
|
||||
const hint = conflictHint(info.mergeable, ci, info.headRefName, info.isCrossRepository === true, head);
|
||||
const lines = [
|
||||
`READY #${o.pr} ${head} STAMP_VERSION=${version}`,
|
||||
`pr: ${info.url ?? `#${o.pr}`}`,
|
||||
@@ -150,15 +223,17 @@ export function ready(ctx: QueueContext, o: { pr: string; pregate?: string; gate
|
||||
`version: ${version}`,
|
||||
`base: origin/${ctx.base}@${baseSha}`,
|
||||
`ci: ${ci.line}`,
|
||||
...(hint ? [hint] : []),
|
||||
pregateLine,
|
||||
gateLine(policy, attempt),
|
||||
receipt.found && receipt.receipt && !receipt.errors.length ? receiptSummaryLine(receipt.receipt) : `SHIP_RECEIPT: ${receipt.found ? 'invalid' : 'none'} in the PR body`,
|
||||
receipt.line,
|
||||
];
|
||||
const gate = policy.policy.full_e2e ? gateEvidence(ctx, o.pr, head, attempt, o.gateExit, o.gateLog) : null;
|
||||
if (gate) lines.push(gate.line);
|
||||
if (!ci.green) return fail({ code: 'QUEUE_NOT_READY', message: `CI is not green at ${head.slice(0, 12)}: ${ci.line}` }, lines, run);
|
||||
if (!pregateOk) return fail({ code: 'QUEUE_NOT_READY', message: `the pregate result is not a current pass: ${pregateLine}` }, lines, run);
|
||||
if (gate && !gate.ok) return fail({ code: 'QUEUE_GATE_MISSING', message: `full_e2e is set and no passing gate is bound to ${head.slice(0, 12)}: ${gate.line}` }, lines, run);
|
||||
if (!receipt.ok) return fail({ code: receipt.code, message: receipt.message }, lines, run);
|
||||
return { ok: true, lines, run };
|
||||
}
|
||||
|
||||
|
||||
+37
-1
@@ -49,6 +49,39 @@ export function extractReceipt(body: string): { found: boolean; receipt?: ShipRe
|
||||
}
|
||||
}
|
||||
|
||||
export type ReceiptBindingCode = 'QUEUE_RECEIPT_MISSING' | 'QUEUE_RECEIPT_STALE';
|
||||
export type ReceiptBinding =
|
||||
| { ok: true; receipt: ShipReceipt; head: string; line: string }
|
||||
| { ok: false; code: ReceiptBindingCode; message: string; line: string };
|
||||
|
||||
/**
|
||||
* The /ship receipt a PR body must carry before READY, train build or a train
|
||||
* merge (CREW rule 22; docs/ship-policy.md#ship-before-ready): a valid block
|
||||
* whose `head` is one of the accepted heads. `accepted` is the head being
|
||||
* READY'd, or the slot's pinned head beside the current head when the current
|
||||
* head is the pin's stamp commit (the stamp changes only stamp files, so the
|
||||
* receipt written for the pin stays the binding one). `line` is the
|
||||
* informational `SHIP_RECEIPT:` line the READY block prints either way.
|
||||
*/
|
||||
export function bindReceipt(body: string, pr: string | undefined, accepted: readonly string[]): ReceiptBinding {
|
||||
const subject = pr === undefined ? "the receipt's" : `PR #${pr}'s`;
|
||||
const ex = extractReceipt(body);
|
||||
if (!ex.found) return { ok: false, code: 'QUEUE_RECEIPT_MISSING', message: `${subject} body carries no ship receipt`, line: 'SHIP_RECEIPT: none in the PR body' };
|
||||
if (!ex.receipt || ex.errors.length) {
|
||||
const why = ex.errors[0] ? oneLine(`${ex.errors[0].message} at ${ex.errors[0].path}`).slice(0, 200) : 'invalid';
|
||||
return { ok: false, code: 'QUEUE_RECEIPT_MISSING', message: `${subject} ship receipt is invalid (${why})`, line: 'SHIP_RECEIPT: invalid in the PR body' };
|
||||
}
|
||||
const r = ex.receipt;
|
||||
const line = receiptSummaryLine(r);
|
||||
const head = accepted.find(h => h === r.head || (r.head.length >= 12 && h.startsWith(r.head)));
|
||||
if (!head) {
|
||||
const want = accepted.map(h => h.slice(0, 12)).join(' or ');
|
||||
return { ok: false, code: 'QUEUE_RECEIPT_STALE', message: `${subject} ship receipt is for ${oneLine(r.head.slice(0, 12))}, not the head ${want} being merged`, line };
|
||||
}
|
||||
if (pr !== undefined && r.pr !== undefined && String(r.pr) !== pr) return { ok: false, code: 'QUEUE_RECEIPT_STALE', message: `${subject} ship receipt names pr=${r.pr}, not this PR`, line };
|
||||
return { ok: true, receipt: r, head, line };
|
||||
}
|
||||
|
||||
/** The block becomes the first section of the body: an existing block is replaced in place at the top, prose follows. */
|
||||
export function placeReceiptFirst(body: string, receipt: ShipReceipt): string {
|
||||
const block = renderReceiptBlock(receipt);
|
||||
@@ -70,6 +103,9 @@ export function missingIssueLines(body: string, issues: NonNullable<ShipReceipt[
|
||||
return expected.filter(e => !have.some(l => l === e || l.startsWith(`${e} `)));
|
||||
}
|
||||
|
||||
/** Receipt fields come from the PR body, so anything printed from one stays on its own line, control characters included. */
|
||||
const oneLine = (s: string) => s.replace(/[\x00-\x1f\x7f\u0080-\u009f\u2028\u2029]+/g, ' ');
|
||||
|
||||
/** The first lines of the completion message to the coordinator: identifiers only. */
|
||||
export function receiptSummaryLine(r: ShipReceipt): string {
|
||||
const parts = [`pr=${r.pr ?? '-'}`, `head=${r.head.slice(0, 12)}`, `version=${r.version ?? '-'}`, `base=${r.base}`];
|
||||
@@ -80,7 +116,7 @@ export function receiptSummaryLine(r: ShipReceipt): string {
|
||||
if (r.session_kind) parts.push(`session_kind=${r.session_kind}`);
|
||||
if (r.artifacts_consumed) parts.push(`artifacts_consumed=${r.artifacts_consumed}`);
|
||||
if (r.approval) parts.push(`approval=${approvalSummary(r.approval)}`);
|
||||
return `SHIP_RECEIPT: ${parts.join(' ')}`;
|
||||
return `SHIP_RECEIPT: ${oneLine(parts.join(' '))}`;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -8,6 +8,9 @@
|
||||
* stamped one to the version the restamp allocates (base + the policy bump).
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { parseChangelog } from './changelog-check';
|
||||
import { matchesAny } from './ship-policy';
|
||||
|
||||
@@ -30,6 +33,62 @@ export function bumpVersion(version: string, level: string): string | null {
|
||||
return v.map((n, i) => (i < at ? n : i === at ? n + 1 : 0)).join('.');
|
||||
}
|
||||
|
||||
/**
|
||||
* An unstamped head may regenerate a release output (a generator change, for
|
||||
* example) as long as what it committed is exactly what the policy's
|
||||
* release_tool produces at its merge base's VERSION (the head's own, since
|
||||
* VERSION is unchanged on this path; the base tip may have stamped since):
|
||||
* the slot's restamp regenerates it anyway. Runs the head's tool in a
|
||||
* throwaway worktree with that VERSION and compares each changed output byte
|
||||
* for byte, reading the tool's output from a file (first 4 KB) so a lingering
|
||||
* child cannot hold the gate open, and killing a tool that outlives the limit
|
||||
* with SIGKILL so one that ignores TERM cannot either. A stamp (new
|
||||
* VERSION inside the output) differs, so it still classifies as stamped, and
|
||||
* so does an output the tool never writes (it is removed before the run).
|
||||
* The tool is head code: it runs with a bare environment (no tokens), no
|
||||
* node_modules link, and a short timeout, and its outputs are read only from
|
||||
* the staging worktree.
|
||||
*/
|
||||
const TOOL_ENV_KEYS = ['PATH', 'HOME', 'TMPDIR', 'LANG', 'LC_ALL', 'BUN_INSTALL', 'SystemRoot', 'TEMP', 'TMP'];
|
||||
export function regeneratedAtBase(o: { repoRoot: string; head: string; baseVersion: string; tool: string; outputs: string[]; timeoutMs?: number }): { kind: 'same' } | { kind: 'differs'; files: string[] } | { kind: 'error'; reason: string } {
|
||||
const timeoutMs = o.timeoutMs ?? 120_000;
|
||||
if (!o.baseVersion.trim()) return { kind: 'error', reason: 'the merge base has no VERSION to regenerate the release outputs at' };
|
||||
let staging: string;
|
||||
try { staging = fs.mkdtempSync(path.join(os.tmpdir(), 'version-gate-regen-')); } catch (e) { return { kind: 'error', reason: `cannot create a staging directory to regenerate the release outputs: ${e instanceof Error ? e.message : String(e)}` }; }
|
||||
try {
|
||||
const add = spawnSync('git', ['worktree', 'add', '--detach', '-q', staging, o.head], { cwd: o.repoRoot, encoding: 'utf8', timeout: 30_000 });
|
||||
if (add.status !== 0) return { kind: 'error', reason: `cannot stage ${o.head.slice(0, 12)} to regenerate its release outputs: ${(add.stderr || '').trim().slice(0, 200)}` };
|
||||
fs.rmSync(path.join(staging, 'VERSION'), { force: true });
|
||||
fs.writeFileSync(path.join(staging, 'VERSION'), o.baseVersion);
|
||||
for (const f of o.outputs) fs.rmSync(path.join(staging, f), { force: true });
|
||||
const env: NodeJS.ProcessEnv = { GSTACK_RESTAMP_VERSION: o.baseVersion.trim(), GSTACK_RESTAMP_STAGING: '1' };
|
||||
for (const key of TOOL_ENV_KEYS) if (process.env[key] !== undefined) env[key] = process.env[key];
|
||||
const logPath = path.join(os.tmpdir(), path.basename(staging) + '.log');
|
||||
const log = fs.openSync(logPath, 'w');
|
||||
let run: ReturnType<typeof spawnSync>;
|
||||
try { run = spawnSync('sh', ['-c', o.tool], { cwd: staging, stdio: ['ignore', log, log], timeout: timeoutMs, killSignal: 'SIGKILL', env }); } finally { fs.closeSync(log); }
|
||||
const output = (() => {
|
||||
try { const fd = fs.openSync(logPath, 'r'); try { const buf = Buffer.alloc(4096); return buf.subarray(0, fs.readSync(fd, buf)).toString('utf8'); } finally { fs.closeSync(fd); } } catch { return ''; } finally { fs.rmSync(logPath, { force: true }); }
|
||||
})();
|
||||
if (run.status !== 0) {
|
||||
const timedOut = run.signal !== null || (run.error as NodeJS.ErrnoException | undefined)?.code === 'ETIMEDOUT';
|
||||
const how = run.status !== null ? `exited ${run.status}` : timedOut ? `was killed at the ${Math.round(timeoutMs / 1000)}s limit` : (run.error?.message ?? 'did not exit');
|
||||
return { kind: 'error', reason: `release_tool \`${o.tool}\` ${how} at ${o.head.slice(0, 12)} with its merge base's VERSION, so its release outputs cannot be told from a stamp: ${output.trim().slice(0, 200)}` };
|
||||
}
|
||||
const files = o.outputs.filter(f => {
|
||||
const committed = spawnSync('git', ['show', `${o.head}:${f}`], { cwd: o.repoRoot, timeout: 30_000, maxBuffer: 64 * 1024 * 1024 });
|
||||
let generated: Buffer | null = null;
|
||||
try { generated = fs.readFileSync(path.join(staging, f)); } catch { generated = null; }
|
||||
return committed.status !== 0 || generated === null || !committed.stdout.equals(generated);
|
||||
});
|
||||
return files.length ? { kind: 'differs', files } : { kind: 'same' };
|
||||
} finally {
|
||||
spawnSync('git', ['worktree', 'remove', '--force', staging], { cwd: o.repoRoot, timeout: 30_000 });
|
||||
fs.rmSync(staging, { recursive: true, force: true });
|
||||
spawnSync('git', ['worktree', 'prune'], { cwd: o.repoRoot, timeout: 30_000 });
|
||||
}
|
||||
}
|
||||
|
||||
export function classifyTrainHead(o: { repoRoot: string; baseRef: string; head: string }): TrainHead {
|
||||
const base = `origin/${o.baseRef.replace(/^origin\//, '')}`;
|
||||
const show = (rev: string, file: string) => git(o.repoRoot, ['show', `${rev}:${file}`]);
|
||||
@@ -42,10 +101,19 @@ export function classifyTrainHead(o: { repoRoot: string; baseRef: string; head:
|
||||
const npmVersion = (rev: string) => { try { return JSON.parse(show(rev, 'package.json') ?? '{}').version ?? null; } catch { return null; } };
|
||||
const changed = (git(o.repoRoot, ['diff', '--name-only', mergeBase, o.head]) ?? '').split('\n').filter(Boolean);
|
||||
const releaseOutputs: string[] = Array.isArray(policy.release_outputs) ? policy.release_outputs : [];
|
||||
const touched = changed.filter(f => f === 'VERSION' || matchesAny(releaseOutputs, f));
|
||||
// touched: what marks the head as stamped (the labels only feed the length check below)
|
||||
const touched: string[] = changed.filter(f => f === 'VERSION');
|
||||
if (npmVersion(mergeBase) !== npmVersion(o.head)) touched.push('package.json version');
|
||||
const outputs = changed.filter(f => matchesAny(releaseOutputs, f));
|
||||
const baseVersion = show(base, 'VERSION') ?? '';
|
||||
if (!touched.length && outputs.length && typeof policy.release_tool === 'string') {
|
||||
// the head's VERSION is its merge base's (unchanged on this path); the base tip may have stamped since
|
||||
const regen = regeneratedAtBase({ repoRoot: o.repoRoot, head: o.head, baseVersion: show(mergeBase, 'VERSION') ?? '', tool: policy.release_tool, outputs });
|
||||
if (regen.kind === 'error') return { kind: 'invalid', reason: regen.reason };
|
||||
if (regen.kind === 'differs') touched.push(...regen.files);
|
||||
} else touched.push(...outputs);
|
||||
if (touched.length) {
|
||||
const expected = bumpVersion(show(base, 'VERSION') ?? '', policy.bump ?? 'patch');
|
||||
const expected = bumpVersion(baseVersion, policy.bump ?? 'patch');
|
||||
return expected ? { kind: 'stamped', expected } : { kind: 'invalid', reason: `${base}:VERSION is not a 4-part version` };
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user