Merge origin/main (v1.91.376.0) into the seed-ceiling splits (rule 19 grant, GSTA-36)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-31 17:50:01 +00:00
commit 221320e277
485 files changed
+34984 -2411

No files matched your search

+12 -3
View File
@@ -39,9 +39,14 @@ const prVersionEnv = (process.env.PR_VERSION ?? "").trim();
// head is unstamped and the lane stamps at its slot (docs/ship-policy.md#run-a-merge-train).
// The workflow passes BASE_REF and HEAD_SHA with the PR head checked out as cwd.
if (process.env.BASE_REF && process.env.HEAD_SHA) {
const train = classifyTrainHead({ repoRoot: process.cwd(), baseRef: process.env.BASE_REF, head: process.env.HEAD_SHA });
const train = classifyTrainHead({ repoRoot: process.cwd(), baseRef: process.env.BASE_REF, head: process.env.HEAD_SHA, author: process.env.PR_AUTHOR });
if (train.kind === "unstamped") {
console.log(`✓ ${tag} is unstamped under the base's train policy: VERSION and package.json match its merge base, any changed release output is what the release tool generates at the merge base's VERSION, and the CHANGELOG change is confined to [Unreleased]. The lane stamps at its slot.`);
if (train.entry === "kept") {
console.log(`✓ ${tag} keeps its merge base's VERSION and package.json version, as the base's ship policy (bump never) requires.`);
process.exit(0);
}
const entry = train.entry === "fragment" ? `its entry is ${train.fragments} CHANGELOG fragment(s)` : train.entry === "none-bot" ? "it is a bot PR with no entry (the merge step writes one from its title)" : "the CHANGELOG change is confined to [Unreleased]";
console.log(`✓ ${tag} is unstamped under the base's ship policy: VERSION and package.json match its merge base, any changed release output is what the release tool generates at the merge base's VERSION, and ${entry}. The merge step stamps it.`);
process.exit(0);
}
if (train.kind === "invalid") {
@@ -49,9 +54,13 @@ if (process.env.BASE_REF && process.env.HEAD_SHA) {
process.exit(1);
}
if (train.kind === "stamped" && prVersionEnv !== train.expected) {
console.log(`::error::${tag} claims v${prVersionEnv}, but under the base's train policy a stamped head must carry v${train.expected} (base + bump, what gstack-restamp --next allocates at its slot). Leave VERSION, package.json and the release outputs at the base's values until the slot is called.`);
console.log(`::error::${tag} claims v${prVersionEnv}, but under the base's ship policy a stamped head must carry v${train.expected} (base + the effective bump, what gstack-restamp --next allocates at the merge step). Leave VERSION, package.json and the release outputs at the base's values until the merge step stamps them.`);
process.exit(1);
}
if (train.kind === "stamped" && train.fragments) {
console.log(`✓ ${tag} carries v${train.expected}, what the merge step allocates under changelog_fragments; the land lock serializes allocation, so open claims do not apply.`);
process.exit(0);
}
}
if (parsed.offline === true) {
+3
View File
@@ -125,6 +125,8 @@ interface GenerateOptions {
root?: string;
/** When true, missing skill description should fail the build. */
strict?: boolean;
/** Skill directories a module loadout leaves out (scripts/modules.ts). */
excludedSkills?: string[];
}
export interface GenerateResult {
@@ -143,6 +145,7 @@ export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<Gener
const skills: SkillEntry[] = [];
for (const t of templates) {
if (!includesSkill(claudeHost, path.dirname(t.tmpl))) continue;
if (opts.excludedSkills?.includes(path.dirname(t.tmpl))) continue;
const filePath = path.join(root, t.tmpl);
const entry = parseSkillFrontmatter(filePath);
if (!entry) {
+49 -6
View File
@@ -23,10 +23,13 @@ import * as path from 'path';
import type { Host, TemplateContext } from './resolvers/types';
import { HOST_PATHS } from './resolvers/types';
import { RESOLVERS } from './resolvers/index';
import { rewriteCarvedSectionRefs, SKILL_BYTE_CEILING, usesLazySections } from './resolvers/sections';
import { applySectionGuards, rewriteCarvedSectionRefs, SKILL_BYTE_CEILING, usesLazySections } from './resolvers/sections';
import { ModuleError, excludedSkillDirs, selectModules } from './modules';
import type { ModuleFilter } from './resolvers/types';
import { insertRuntimePreludes } from './resolvers/runtime-root';
import { generateSharedPreambleFile, hoistsPreamble, SHARED_PREAMBLE_FILE } from './resolvers/preamble-hoist';
import { extractVoiceTriggers } from './resolvers/voice-triggers';
import { frontmatterList, withTriggerLead } from '../lib/skill-triggers';
import { ALL_HOST_NAMES, resolveHostArg, getHostConfig } from '../hosts/index';
import type { HostConfig } from './host-config';
@@ -54,6 +57,8 @@ export interface GenerationOptions {
respectDetection?: boolean;
log?: (message: string) => void;
planMode?: boolean;
/** Module loadout (scripts/modules.ts selectModules); null/undefined = `full`. */
modules?: ModuleFilter | null;
}
interface RenderOptions {
@@ -66,6 +71,7 @@ interface RenderOptions {
explainLevel: 'default' | 'terse';
gbrainDetected: boolean;
planMode: boolean;
modules: ModuleFilter | null;
}
export interface GeneratedArtifact {
@@ -144,6 +150,10 @@ function parseGenerationArgs(args: string[]): GenerationOptions {
const disabledSkills = (disabledValue ?? '').split(/[\s,]+/).filter(Boolean);
const badDisabled = disabledSkills.find(n => !/^\/?[a-z0-9-]+$/.test(n));
if (badDisabled) throw new Error(`--disabled-skills takes skill names (got ${JSON.stringify(badDisabled)})`);
const modules = selectModules({ loadout: value('--loadout'), modules: value('--modules') });
if (modules && (outDir === undefined || path.resolve(outDir) === ROOT)) {
throw new ModuleError('MODULE_SPEC_INVALID', `--${value('--modules') === undefined ? 'loadout' : 'modules'} with exclusions needs an --out-dir outside the checkout root; the committed tree is always the full render`, 'bin/gstack-modules render --host <host> --modules <spec> --out-dir <dir>');
}
const installRoot = value('--install-root');
if (installRoot !== undefined && !INSTALL_ROOT_PATTERN.test(installRoot)) {
throw new Error(`--install-root must be an absolute path of letters, digits and . _ - + @ / (got ${JSON.stringify(installRoot)})`);
@@ -159,9 +169,21 @@ function parseGenerationArgs(args: string[]): GenerationOptions {
: outDir !== undefined ? path.resolve(outDir) : null,
installRoot: installRoot ?? null,
disabledSkills,
modules,
};
}
/** A section file is left out when its skill or its manifest entry is excluded. */
function excludesSection(filter: ModuleFilter | null, skillDir: string, tmpl: string): boolean {
if (!filter) return false;
if (filter.excluded.has(`skill:${skillDir}`)) return true;
const file = path.basename(tmpl).replace(/\.tmpl$/, '');
const manifest = path.join(ROOT, skillDir, 'sections', 'manifest.json');
if (!fs.existsSync(manifest)) return false;
const entry = (JSON.parse(fs.readFileSync(manifest, 'utf-8')).sections ?? []).find((s: { file: string }) => s.file === file);
return entry ? filter.excluded.has(`section:${skillDir}/${entry.id}`) : false;
}
/** Install roots are spliced into shell lines unquoted, so only plain absolute paths. */
const INSTALL_ROOT_PATTERN = /^\/[A-Za-z0-9_.@+\/-]*$/;
@@ -489,7 +511,8 @@ function transformFrontmatter(content: string, host: Host): string {
if (fmEnd === -1) return content;
const frontmatter = content.slice(fmStart + 4, fmEnd);
const body = content.slice(fmEnd + 4);
const { name, description } = extractNameAndDescription(content);
const { name, description: fullDescription } = extractNameAndDescription(content);
const description = fm.descriptionLead ? withTriggerLead(fullDescription, frontmatterList(content, 'triggers'), fm.descriptionLead) : fullDescription;
// Description limit enforcement
if (fm.descriptionLimit) {
@@ -657,7 +680,7 @@ function resolvePlaceholders(
// section generation so both paths get the same gbrain-aware behavior.
const suppressed = effectiveSuppressedResolvers(hostConfig, options);
const onePass = (input: string): string =>
input.replace(/\{\{(\w+(?::[^}]+)?)\}\}/g, (_match, fullKey) => {
applySectionGuards(input, ctx).replace(/\{\{(\w+(?::[^}]+)?)\}\}/g, (_match, fullKey) => {
const parts = fullKey.split(':');
const resolverName = parts[0];
const args = parts.slice(1);
@@ -713,6 +736,7 @@ function buildContext(
return {
skillName, tmplPath, benefitsFrom, host, paths: HOST_PATHS[host],
preambleTier, model: options.model ?? getHostConfig(host).defaultModel, interactive, explainLevel: options.explainLevel, installRoot: options.installRoot, planMode: options.planMode,
...(options.modules ? { modules: options.modules } : {}),
};
}
@@ -842,7 +866,7 @@ function processTemplate(tmplPath: string, host: Host, options: RenderOptions):
}
// Prepend generated header (after frontmatter)
const header = GENERATED_HEADER.replace('{{SOURCE}}', path.basename(tmplPath));
const header = GENERATED_HEADER.replace('{{SOURCE}}', path.basename(tmplPath)) + (options.modules ? `<!-- loadout: ${options.modules.name} -->\n` : '');
const fmEnd = content.indexOf('---', content.indexOf('---') + 3);
if (fmEnd !== -1) {
const insertAt = content.indexOf('\n', fmEnd) + 1;
@@ -859,7 +883,7 @@ function processTemplate(tmplPath: string, host: Host, options: RenderOptions):
// --out-dir: repoint section-base paths to the out-dir (no-op otherwise).
if (host === 'claude') content = rewriteSectionBase(content, options.contentLinkRoot);
if (skillDir === '' || skillDir === '.') content = omitDisabledSkills(content, options.disabledSkills);
if (skillDir === '' || skillDir === '.') content = omitDisabledSkills(content, [...options.disabledSkills, ...excludedSkillDirs(options.modules)]);
content = rewriteInstallRoot(insertRuntimePreludes(rewriteCarvedSectionRefs(content, ctx), ctx), currentHostConfig, options.installRoot);
const preamble = hoistsPreamble(ctx) && tmplContent.includes('{{PREAMBLE}}') && !effectiveSuppressedResolvers(currentHostConfig, options).has('PREAMBLE')
@@ -877,6 +901,15 @@ export function renderAutoplanDriver(options: RenderOptions): string {
return GENERATED_HEADER.replace('{{SOURCE}}', `${AUTOPLAN_DRIVER_RELPATH}.tmpl`) + resolvePlaceholders(tmpl, ctx, getHostConfig('claude'), `${AUTOPLAN_DRIVER_RELPATH}.tmpl`, options);
}
/** ship/DRIVER.md (GSTA-167 PR 5): the compact /ship entry for a parent agent, rendered the same way. */
export const SHIP_DRIVER_RELPATH = 'ship/DRIVER.md';
export function renderShipDriver(options: RenderOptions): string {
const tmplPath = path.join(ROOT, `${SHIP_DRIVER_RELPATH}.tmpl`);
const tmpl = fs.readFileSync(tmplPath, 'utf-8');
const ctx = buildContext(tmpl, tmplPath, 'claude', options, 'ship');
return GENERATED_HEADER.replace('{{SOURCE}}', `${SHIP_DRIVER_RELPATH}.tmpl`) + resolvePlaceholders(tmpl, ctx, getHostConfig('claude'), `${SHIP_DRIVER_RELPATH}.tmpl`, options);
}
/**
* Generate one on-demand section file (`<skill>/sections/<name>.md.tmpl` →
* `<name>.md`). Sections are BODY FRAGMENTS — no frontmatter, no catalog trim,
@@ -958,6 +991,7 @@ export async function runGeneration(settings: GenerationOptions = {}): Promise<G
explainLevel: settings.explainLevel ?? 'default',
gbrainDetected: loadGbrainOverride(settings.respectDetection ?? false),
planMode: settings.planMode ?? true,
modules: settings.modules ?? null,
};
const hosts = settings.host === 'all' ? ALL_HOST_NAMES as Host[] : [settings.host ?? 'claude'];
const log = settings.log ?? (() => {});
@@ -1023,6 +1057,7 @@ export async function runGeneration(settings: GenerationOptions = {}): Promise<G
for (const template of templates) {
const skillDir = path.dirname(template.tmpl);
if (!includesSkill(hostConfig, skillDir)) continue;
if (options.modules?.excluded.has(`skill:${skillDir === '.' ? 'gstack' : skillDir}`)) continue;
const result = processTemplate(path.join(ROOT, template.tmpl), host, options);
const relativePath = rel(result.outputPath);
if (host !== 'claude') renderedNames.add(path.basename(path.dirname(result.outputPath)));
@@ -1050,6 +1085,7 @@ export async function runGeneration(settings: GenerationOptions = {}): Promise<G
for (const section of sections) {
if (!includesSkill(hostConfig, section.skillDir) || !usesLazySections(host, section.skillDir)) continue;
if (excludesSection(options.modules, section.skillDir, section.tmpl)) continue;
const result = processSectionTemplate(path.join(ROOT, section.tmpl), section.skillDir, host, options);
emit(result.outputPath, result.content, 'section', host);
tokenBudget.push({ skill: rel(result.outputPath), lines: result.content.split('\n').length, tokens: Math.round(result.content.length / 4) });
@@ -1065,6 +1101,9 @@ export async function runGeneration(settings: GenerationOptions = {}): Promise<G
if (host === 'claude' && includesSkill(hostConfig, 'autoplan') && templates.some(t => t.tmpl === 'autoplan/SKILL.md.tmpl')) {
emit(path.join(options.outputRoot, AUTOPLAN_DRIVER_RELPATH), renderAutoplanDriver(options), 'asset', host);
}
if (host === 'claude' && includesSkill(hostConfig, 'ship') && templates.some(t => t.tmpl === 'ship/SKILL.md.tmpl')) {
emit(path.join(options.outputRoot, SHIP_DRIVER_RELPATH), renderShipDriver(options), 'asset', host);
}
if (host === 'openclaw') {
for (const variant of ['lite', 'full', 'plan'] as const) {
@@ -1126,7 +1165,7 @@ export async function runGeneration(settings: GenerationOptions = {}): Promise<G
// Shared artifacts are awaited in both modes. A failure must reach the CLI
// exit code, never disappear in a fire-and-forget auxiliary writer.
try {
const index = await generateLlmsTxt();
const index = await generateLlmsTxt({ excludedSkills: excludedSkillDirs(options.modules) });
emit(path.join(options.outputRoot, 'gstack', 'llms.txt'), index.content, 'index');
for (const warning of index.warnings) diagnostics.push({ kind: 'warning', message: `[gen-llms-txt] ${warning}` });
} catch (error) {
@@ -1171,6 +1210,10 @@ export async function main(args = process.argv.slice(2)): Promise<number> {
}
return result.exitCode;
} catch (error) {
if (error instanceof ModuleError) {
console.error(`${error.code}: ${error.message}`);
return 2;
}
console.error(`ERROR: ${(error as Error).message}`);
return 1;
}
+3 -3
View File
@@ -17,7 +17,7 @@
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
"scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"},
"scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
"scripts/test-pr-profile.ts": {"blob":"188710401a08156f585aebcb699151c60e3d6273","since":6,"nonBehavioral":"#3499: RELEASE_METADATA_FILES exempts NOTICE.md, LICENSE and licenses/** and captain/templates/** joins FREE_ONLY_PR_FILES; selection only, no case runs or is judged differently"},
"scripts/test-pr-profile.ts": {"blob":"911e393d0c36c050c3a68db5a81ef21701bfd5ae","since":6,"nonBehavioral":"selection only: fragment exemption narrowed to .md files directly in the fragments dir; changed-file lists include a rename's old path; no case prompt, fixture, judge or verdict logic changes (GST-400)"},
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6},
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6},
"test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6},
@@ -29,7 +29,7 @@
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6},
"test/helpers/eval-store.ts": {"blob":"a50ea2c28f508ad9f35ea9353dd2f502ee0dcfc2","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
"test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6},
"test/helpers/llm-judge.ts": {"blob":"a3177665b662932ce485ccc976c2b36a1531a0bb","since":6,"nonBehavioral":"callJudge rewrites only the message of a JSON parse failure (names the judge output, stop reason, length and the retained llm-judge-response-parse-error event); it still throws a SyntaxError before any score is parsed, the request is unchanged, and no case's pass or fail changes"},
"test/helpers/llm-judge.ts": {"blob":"095f9c85dd0ae29da8ba10fed91510313a4442a4","since":6,"nonBehavioral":"callJudge emits one llm-judge-aborted diagnostic line (wrapper attempts, HTTP request and status counts seen through a pass-through fetch, 429 waits, connect and event timings, block kind, character counts, message and request ids, a capped reason; no content) when its caller's signal aborts it; observers swallow their own errors, and the request, retries, parsing, scores and the thrown abort reason are unchanged, so no case's pass or fail changes"},
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6},
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6},
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6},
@@ -50,7 +50,7 @@
"test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
"test/helpers/skill-fixture.ts": {"blob":"8a654dae6b0a396d839b02179119ae3e3bb13f69","since":6,"nonBehavioral":"extractSkillBody and extractSkillSections follow the shared-preamble hoist (pointer plus sections/preamble.md); every fixture they build is byte-identical to the pre-hoist output (compared for 8 body and 4 section extractions), so no case's input or verdict changes; the /ship review then reworded only comments and two error messages, so no fixture byte changes"},
"test/helpers/skill-startup.ts": {"blob":"a8d21e3817c3f35a1d06af325c92fde7d420e377","since":6,"nonBehavioral":"readSkillEffective also appends a variant's added ### subsections (office-hours' Open-ended questions); readSkillStartup, readSharedSections and readSkillEffective return bytes identical to the pinned helper for every tier 2+ render without added subsections (735 across all hosts) and for office-hours' SKILL.md before this PR, so office-hours' fixture input changes only because its prompt changed in this PR, not because the helper did"},
"test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6},
"test/helpers/test-selection.ts": {"blob":"e1b54ad5e3d3eadd8027de1b44e57554d6d99a41","since":6,"nonBehavioral":"selection only: fragment exemption narrowed to .md files directly in the fragments dir; changed-file lists include a rename's old path; no case prompt, fixture, judge or verdict logic changes (GST-400)"},
"test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6}
}
}
+6
View File
@@ -81,6 +81,12 @@ export interface HostConfig {
descriptionLimit?: number | null;
/** What to do when description exceeds limit. Default: 'error'. */
descriptionLimitBehavior?: 'error' | 'truncate' | 'warn';
/**
* Put a trigger-first lead of at most this many chars on the first line of
* the description (lib/skill-triggers.ts withTriggerLead), for hosts whose
* skills index shows only the start of a description.
*/
descriptionLead?: number;
/** Additional frontmatter fields to inject (host-wide). */
extraFields?: Record<string, unknown>;
/**
+14 -6
View File
@@ -95,21 +95,29 @@ export interface CiJob { id: number; name: string; conclusion: string | null; st
const PAGE = 100;
/** The runs API's `created` filter, by UTC day (both ends inclusive): `>=since`, or `since..until` when the window has an end. */
export function createdFilter(since: string, until?: string): string {
return `created=${encodeURIComponent(until ? `${since.slice(0, 10)}..${until.slice(0, 10)}` : `>=${since.slice(0, 10)}`)}`;
}
/**
* Every completed run of `workflow` created at or after `since`, newest first,
* paging until a page is short or older than `since`. A failure on any page
* throws: a partial history is never reported as a complete one.
* Every completed run of `workflow` created at or after `since` (and before
* `until` when given), newest first, paging until a page is short (the
* `created` filter bounds the range server-side). A run whose created_at does
* not parse is skipped. A failure on any page throws: a partial history is
* never reported as a complete one.
*/
export function listWorkflowRuns(client: GhClient, repo: string, workflow: string,
opts: { since: string; branch?: string; event?: string; maxPages?: number }): CiRun[] {
opts: { since: string; until?: string; branch?: string; event?: string; maxPages?: number }): CiRun[] {
const out: CiRun[] = [];
const query = [`status=completed`, `per_page=${PAGE}`, `created=${encodeURIComponent(`>=${opts.since.slice(0, 10)}`)}`,
const query = [`status=completed`, `per_page=${PAGE}`, createdFilter(opts.since, opts.until),
...(opts.branch ? [`branch=${encodeURIComponent(opts.branch)}`] : []), ...(opts.event ? [`event=${encodeURIComponent(opts.event)}`] : [])];
for (let page = 1; page <= (opts.maxPages ?? 20); page++) {
const body = client.getJson(`repos/${repo}/actions/workflows/${workflow}/runs?${query.join('&')}&page=${page}`) as { workflow_runs?: any[] };
const runs = Array.isArray(body?.workflow_runs) ? body.workflow_runs : [];
for (const run of runs) {
if (String(run.created_at) < opts.since) continue;
const created = Date.parse(String(run.created_at));
if (!(created >= Date.parse(opts.since)) || (opts.until && created >= Date.parse(opts.until))) continue;
if ((opts.branch && run.head_branch !== opts.branch) || (opts.event && run.event !== opts.event)) continue;
out.push({ id: Number(run.id), attempt: Number(run.run_attempt ?? 1), sha: String(run.head_sha ?? ''), branch: String(run.head_branch ?? ''),
event: String(run.event ?? ''), status: String(run.status ?? ''), conclusion: run.conclusion ?? null,
+2 -1
View File
@@ -249,7 +249,8 @@ export function computePaidCaseSelection(options: {
: selectTests(selectionFiles, table, GLOBAL_TOUCHFILES, { baseRef, cwd: rootDir }).selected;
const selection = { e2e: select(E2E_TOUCHFILES), judges: select(LLM_JUDGE_TOUCHFILES) };
if (options.profile === 'full') return { selection, reason: all ? 'run-all' : 'diff' };
const coverage = selectPrProfile({ selectedE2E: selection.e2e, selectedJudges: selection.judges, changedFiles: effectiveFiles, sourceAliases });
const coverage = selectPrProfile({ selectedE2E: selection.e2e, selectedJudges: selection.judges, changedFiles: effectiveFiles, sourceAliases,
screenPolicy: env.EVALS_SCREEN_POLICY === 'narrow' ? 'narrow' : 'broad' });
if (coverage.needsFullValidation) {
throw new Error(`PR profile requires full validation: ${coverage.missingCoverage.join(', ')}. Use --profile full and the relevant periodic cases.`);
}
+9
View File
@@ -51,6 +51,7 @@ const WINDOWS_FRAGILE_PATTERNS: Array<{ pattern: RegExp; reason: string }> = [
// when possible; this list is for environment-/runtime-specific tests where
// the failure mode is structural rather than detectable via source-file scan.
export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }> = [
{ file: 'test/ship-queue-land.test.ts', reason: 'its gh shim is an extensionless #!/usr/bin/env node script on a colon-joined PATH, which windows-latest cannot launch, so land reaches the real gh (no GH_TOKEN) and every case fails (windows-free-shard 6, run 40824965988); land is a controller command run on Linux and macOS' },
// Like test/paid-rerun.test.ts (excluded by pattern), it replays a gate capture through the paid lane's --report and --rerun-plan CLI.
{ file: 'test/shared-libs-index-flags-readonly.test.ts', reason: 'its assume-unchanged replay seeds the review through test/helpers/shared-libs-path-fixture.ts, which launches bin/gstack-review-log (a bash script) directly; windows-latest refuses it as not found in $PATH (windows-free-shard 5, run 40727825859)' },
{ file: 'test/credential-rejection.test.ts', reason: 'replays run 39696775546 through the --report and --rerun-plan CLI of the paid lane, which runs only on Linux CI; on windows-latest that CLI exits 1 with EINVAL in open() (windows-free-shard run 39982914852)' },
@@ -310,6 +311,10 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
reason: 'spawns bin/gstack-ship-queue (shebang) and a #!/bin/sh gh shim through test/helpers/restamp-fixture.ts and test/helpers/ship-queue-fixture.ts',
},
// GSTA-49 T1 (merge trains): the train lib logic runs on Windows in test/ship-queue-train-logic.test.ts.
{
file: 'test/ship-queue-train-land.test.ts',
reason: 'drives train land against test/helpers/ship-queue-train-fixture.ts plus an extensionless node gh wrapper on a colon-joined PATH; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead, as for test/ship-queue-train.test.ts and test/ship-queue-land.test.ts (windows-free-shard 6, run 40824965988)',
},
{
file: 'test/ship-queue-train.test.ts',
reason: 'drives bin/gstack-ship-queue against the stateful #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead of the shim (windows-free-shard run 39152702056)',
@@ -367,6 +372,10 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
file: 'test/restamp-gate-ahead-changelog.test.ts',
reason: 'spawns bin/gstack-restamp (shebang) and a POSIX release tool through test/helpers/restamp-fixture.ts',
},
{
file: 'test/ship-release.test.ts',
reason: 'spawns bin/gstack-ship-release (shebang) directly and writes #!/bin/sh gh stubs through test/helpers/ship-bin-fixture.ts; windows-latest exits before the bin runs (windows-free-shard 3, run 41073592483)',
},
];
// Force-include overrides: files a WINDOWS_FRAGILE_PATTERNS regex excludes for
+86
View File
@@ -0,0 +1,86 @@
/**
* bin/gstack-modules (GSTA-140, PR 1a): the module catalog and loadout renders.
*
* gstack-modules list [--json] [--pinned]
* gstack-modules validate
* gstack-modules render --host <h> (--loadout <name> | --modules <base>,-id,+id) --out-dir <d>
* [--install-root <abs>] [--model <id>] [--plan-mode on|off]
*
* Exit 0 on success, 2 on a refused input (ModuleError, printed with its code,
* fix and docs anchor), 1 on an unexpected failure.
*/
import * as crypto from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { resolveHostArg } from '../hosts/index';
import { resolveModel } from './models';
import { LOADOUTS_FILE, ModuleError, loadLoadouts, moduleCatalog, selectModules, validateLoadouts } from './modules';
import { renderLoadout } from './modules-render';
import type { Host } from './resolvers/types';
const ROOT = path.resolve(import.meta.dir, '..');
const USAGE = `Usage:
gstack-modules list [--json] [--pinned] every module id with its kind, source and pin
gstack-modules validate check ${LOADOUTS_FILE} against the catalog
gstack-modules render --host <h> (--loadout <name> | --modules <base>,-id,+id) --out-dir <d>
[--install-root <abs>] [--model <id>] [--plan-mode on|off]
render one loadout and write loadout.lock.json
Guide: docs/loadouts.md`;
export interface CliResult { code: number; stdout: string; stderr: string }
function flag(args: string[], name: string): string | undefined {
const index = args.findIndex(a => a === name || a.startsWith(`${name}=`));
if (index < 0) return undefined;
const value = args[index].startsWith(`${name}=`) ? args[index].slice(name.length + 1) : args[index + 1];
if (!value || value.startsWith('--')) throw new ModuleError('MODULE_SPEC_INVALID', `${name} requires a value`, `gstack-modules --help`);
return value;
}
export async function modulesMain(args: string[]): Promise<CliResult> {
const [command, ...rest] = args;
try {
if (!command || command === '--help' || command === '-h' || command === 'help') return { code: 0, stdout: USAGE + '\n', stderr: '' };
if (command === 'list') {
const manifest = loadLoadouts();
const catalog = moduleCatalog(manifest).filter(e => !rest.includes('--pinned') || e.pinned);
const rows = catalog.map(e => {
const bytes = fs.readFileSync(path.join(ROOT, e.source));
return { ...e, source_bytes: bytes.length, source_sha256: crypto.createHash('sha256').update(bytes).digest('hex') };
});
if (rest.includes('--json')) return { code: 0, stdout: JSON.stringify({ loadouts: Object.keys(manifest.loadouts), modules: rows }, null, 2) + '\n', stderr: '' };
const lines = rows.map(r => `${r.id.padEnd(48)} ${r.kind.padEnd(8)} ${r.pinned ? 'pinned' : ' '} ${r.source}`);
return { code: 0, stdout: [`loadouts: ${Object.keys(manifest.loadouts).join(', ')}`, ...lines].join('\n') + '\n', stderr: '' };
}
if (command === 'validate') {
const manifest = loadLoadouts();
const problems = validateLoadouts(manifest, moduleCatalog(manifest));
if (problems.length) return { code: 2, stdout: '', stderr: problems.map(p => `LOADOUTS_INVALID: ${p}`).join('\n') + '\n' };
return { code: 0, stdout: `LOADOUTS: ok (${Object.keys(manifest.loadouts).length} loadouts)\n`, stderr: '' };
}
if (command === 'render') {
const hostValue = flag(rest, '--host');
const outDir = flag(rest, '--out-dir');
if (!hostValue || !outDir) throw new ModuleError('MODULE_SPEC_INVALID', 'render needs --host and --out-dir', 'gstack-modules render --host claude --loadout full --out-dir <dir>');
let host: Host;
try { host = resolveHostArg(hostValue) as Host; } catch (error) { throw new ModuleError('MODULE_SPEC_INVALID', (error as Error).message, 'pass a host from docs/hosts.md, for example --host claude'); }
const loadout = flag(rest, '--loadout');
const modules = flag(rest, '--modules');
if (loadout === undefined && modules === undefined) throw new ModuleError('MODULE_SPEC_INVALID', 'render needs --loadout or --modules', 'add --loadout full for the control arm');
const modelValue = flag(rest, '--model');
const model = modelValue === undefined ? null : resolveModel(modelValue);
if (modelValue !== undefined && !model) throw new ModuleError('MODULE_SPEC_INVALID', `--model ${JSON.stringify(modelValue.slice(0, 80))} is not a model gstack knows`, 'pass the same --model setup uses, for example gpt-6-astra');
const planMode = flag(rest, '--plan-mode');
if (planMode !== undefined && planMode !== 'on' && planMode !== 'off') throw new ModuleError('MODULE_SPEC_INVALID', '--plan-mode takes on or off', 'drop it, or pass --plan-mode off');
const installRoot = flag(rest, '--install-root');
if (installRoot !== undefined && !/^\/[A-Za-z0-9_.@+\/-]*$/.test(installRoot)) throw new ModuleError('MODULE_SPEC_INVALID', '--install-root must be a plain absolute path', 'pass the absolute install root setup would use');
const selection = selectModules({ loadout, modules });
const lock = await renderLoadout({ host, selection, outDir, installRoot: installRoot ?? null, model, planMode: planMode !== 'off' });
return { code: 0, stdout: `RENDERED: loadout=${lock.loadout} host=${host} files=${lock.files.length} bytes=${lock.total_bytes} excluded=${lock.excluded.length} lock=${path.join(path.resolve(outDir), 'loadout.lock.json')}\n`, stderr: '' };
}
return { code: 2, stdout: '', stderr: `Unknown command ${JSON.stringify(command.slice(0, 40))}.\n${USAGE}\n` };
} catch (error) {
if (error instanceof ModuleError) return { code: 2, stdout: '', stderr: `${error.code}: ${error.message}\n` };
return { code: 1, stdout: '', stderr: `gstack-modules: unexpected failure: ${(error as Error).message}\n` };
}
}
+176
View File
@@ -0,0 +1,176 @@
/**
* gstack-modules render (GSTA-140, PR 1a): one loadout rendered into a
* directory GSTA-2 can preregister, published only when complete.
*
* Order: render into an owned staging directory beside --out-dir (Claude
* section links already name the final path), refuse dangling references to
* an excluded module, hash every file into loadout.lock.json (written last),
* then swap the staging directory into place. Any failure leaves the previous
* --out-dir untouched and removes the staging directory.
*/
import { createHash, randomBytes } from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { runGeneration } from './gen-skill-docs';
import { ModuleError, type ModuleSelection } from './modules';
import type { Host } from './resolvers/types';
import { getHostConfig } from '../hosts/index';
import type { Model } from './models';
export const LOCK_FILE = 'loadout.lock.json';
const ROOT = path.resolve(import.meta.dir, '..');
export interface RenderRequest {
host: Host;
/** null = the full loadout. */
selection: ModuleSelection | null;
outDir: string;
installRoot?: string | null;
model?: Model | null;
planMode?: boolean;
}
export interface LoadoutLock {
schema_version: 1;
loadout: string;
spec: string;
spec_sha256: string | null;
host: Host;
model: string | null;
install_root: string | null;
plan_mode: boolean;
gstack_version: string;
excluded: string[];
total_bytes: number;
files: Array<{ path: string; bytes: number; sha256: string }>;
}
function walk(dir: string, base = dir): string[] {
return fs.readdirSync(dir, { withFileTypes: true }).flatMap(entry => {
const full = path.join(dir, entry.name);
if (entry.isDirectory()) return walk(full, base);
return entry.isFile() ? [path.relative(base, full).split(path.sep).join('/')] : [];
}).sort();
}
/**
* References an excluded module leaves behind in the rendered text. A section
* counts as referenced by: its absolute path (`/<skill>/sections/<file>`, any
* file, so resolver-inlined cross-skill pointers count), an installed-relative
* pointer naming its skill, or, inside its own skill's directory, a bare
* `sections/<file>` or the inline-host "section inlined" phrase. An excluded
* skill counts as referenced by another skill naming its SKILL.md or one of
* its section files.
*/
export function danglingReferences(stage: string, selection: ModuleSelection): string[] {
const checks: Array<{ id: string; scope: string | null; test: (line: string) => boolean }> = [];
for (const id of selection.excluded) {
if (id.startsWith('section:')) {
const [skill, sectionId] = id.slice('section:'.length).split('/');
const manifest = JSON.parse(fs.readFileSync(path.join(ROOT, skill, 'sections', 'manifest.json'), 'utf-8'));
const file = (manifest.sections as Array<{ id: string; file: string }>).find(entry => entry.id === sectionId)!.file;
const stem = file.replace(/\.md$/, '');
checks.push(
{ id, scope: null, test: line => line.includes(`/${skill}/sections/${file}`) },
{ id, scope: null, test: line => line.includes(`sections/${file}\` relative to the installed`) && line.includes(`${skill}\``) },
{ id, scope: skill, test: line => line.includes(`sections/${file}`) || line.includes(`the \`${stem}\` section inlined`) },
);
} else if (id.startsWith('skill:')) {
const skill = id.slice('skill:'.length);
checks.push({ id, scope: null, test: line => line.includes(`/${skill}/SKILL.md`) || line.includes(`gstack-${skill}/SKILL.md`) });
const manifestPath = path.join(ROOT, skill, 'sections', 'manifest.json');
if (fs.existsSync(manifestPath)) {
for (const { file } of JSON.parse(fs.readFileSync(manifestPath, 'utf-8')).sections as Array<{ file: string }>) {
checks.push(
{ id, scope: null, test: line => line.includes(`/${skill}/sections/${file}`) },
{ id, scope: null, test: line => line.includes(`sections/${file}\` relative to the installed`) && line.includes(`${skill}\``) },
);
}
}
}
}
if (!checks.length) return [];
const found = new Set<string>();
for (const rel of walk(stage)) {
if (!rel.endsWith('.md')) continue;
const dirs = rel.split('/');
const lines = fs.readFileSync(path.join(stage, rel), 'utf-8').split('\n');
lines.forEach((line, index) => {
for (const check of checks) {
if (check.scope && !dirs.includes(check.scope) && !dirs.includes(`gstack-${check.scope}`)) continue;
if (check.test(line)) found.add(`${check.id}: referenced by ${rel}:${index + 1}`);
}
});
}
return [...found];
}
function owned(dir: string): boolean {
if (!fs.existsSync(dir)) return true;
return fs.readdirSync(dir).length === 0 || fs.existsSync(path.join(dir, LOCK_FILE));
}
export async function renderLoadout(request: RenderRequest): Promise<LoadoutLock> {
const final = path.resolve(request.outDir);
if (!owned(final)) {
throw new ModuleError('RENDER_DEST_NOT_OWNED', `${final} exists and holds no ${LOCK_FILE}, so it is not a gstack-modules render`, 'pass an empty or new --out-dir, or one an earlier gstack-modules render wrote');
}
const tag = `${process.pid}-${randomBytes(4).toString('hex')}`;
const stage = `${final}.stage-${tag}`;
const retired = `${final}.old-${tag}`;
fs.mkdirSync(path.dirname(final), { recursive: true });
try {
const result = await runGeneration({
host: request.host,
outputRoot: stage,
contentLinkRoot: final,
installRoot: request.installRoot ?? null,
model: request.model ?? null,
planMode: request.planMode ?? true,
modules: request.selection,
});
const errors = result.diagnostics.filter(d => d.kind === 'error').map(d => d.message);
if (result.exitCode !== 0 || errors.length) {
throw new ModuleError('RENDER_FAILED', `gen-skill-docs failed: ${errors.join('; ') || 'exit 1'}`, 'fix the named template or module, then rerun; side effects: none, safe to retry');
}
if (request.selection) {
const dangling = danglingReferences(stage, request.selection);
if (dangling.length) {
throw new ModuleError('MODULE_EXCLUSION_UNSAFE', dangling.slice(0, 5).join('; ') + (dangling.length > 5 ? ` (+${dangling.length - 5} more)` : ''), 'exclude the referencing module too, or guard the reference with {{IF_SECTION:id}} in its template; side effects: none, safe to retry');
}
}
const files = walk(stage).map(rel => {
const bytes = fs.readFileSync(path.join(stage, rel));
return { path: rel, bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') };
});
const lock: LoadoutLock = {
schema_version: 1,
loadout: request.selection?.name ?? 'full',
spec: request.selection?.spec ?? 'full',
spec_sha256: request.selection?.specSha256 ?? null,
host: request.host,
model: request.model ?? getHostConfig(request.host).defaultModel ?? null,
install_root: request.installRoot ?? null,
plan_mode: request.planMode ?? true,
gstack_version: fs.readFileSync(path.join(ROOT, 'VERSION'), 'utf-8').trim(),
excluded: [...(request.selection?.excluded ?? [])].sort(),
total_bytes: files.reduce((sum, f) => sum + f.bytes, 0),
files,
};
fs.writeFileSync(path.join(stage, LOCK_FILE), JSON.stringify(lock, null, 2) + '\n');
if (!owned(final)) {
throw new ModuleError('RENDER_DEST_NOT_OWNED', `${final} changed during the render and now holds no ${LOCK_FILE}`, 'pass an empty or new --out-dir, or one an earlier gstack-modules render wrote');
}
if (fs.existsSync(final)) fs.renameSync(final, retired);
try {
fs.renameSync(stage, final);
} catch (error) {
if (fs.existsSync(retired)) fs.renameSync(retired, final);
throw error;
}
fs.rmSync(retired, { recursive: true, force: true });
return lock;
} finally {
fs.rmSync(stage, { recursive: true, force: true });
}
}
+236
View File
@@ -0,0 +1,236 @@
/**
* Module loadouts (GSTA-140, PR 1a): the toggleable units of gstack and the
* named sets of them a render can load.
*
* A module is a skill (`skill:<dir>`), one carved section
* (`section:<skill>/<id>`, from <skill>/sections/manifest.json), one model
* overlay (`overlay:<name>`, from model-overlays/) or one preamble piece
* (`preamble:<id>`, from PREAMBLE_PIECES). The catalog is derived from those
* sources; modules/loadouts.json is the only hand-written file and holds the
* rungs, the model-id to rung table, the pinned ids and the loadouts.
*
* Every error is a ModuleError with a stable code, a fix line and a
* docs/troubleshooting.md anchor. Nothing here ever falls back to `full` on
* bad input: an eval arm that silently rendered `full` would measure the
* control.
*/
import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { discoverTemplates } from './discover-skills';
import { PREAMBLE_MODULE_IDS } from './resolvers/preamble';
import type { ModuleFilter } from './resolvers/types';
const ROOT = path.resolve(import.meta.dir, '..');
export const LOADOUTS_FILE = 'modules/loadouts.json';
export const RUNGS = ['frontier', 'strong', 'mid', 'light'] as const;
export type Rung = (typeof RUNGS)[number] | 'unknown';
export const FULL = 'full';
/** The router and the way back can never be left out (same rule as --disabled-skills). */
export const IMPLICIT_PINS = ['skill:gstack', 'skill:gstack-upgrade', 'preamble:bash'];
const NAME_RE = /^[a-z0-9-]+$/;
export type ModuleKind = 'skill' | 'section' | 'overlay' | 'preamble';
export interface ModuleEntry {
id: string;
kind: ModuleKind;
/** Repository-relative source file. */
source: string;
pinned: boolean;
}
export interface LoadoutDef {
base?: string;
include?: '*' | string[];
exclude?: string[];
preamble_tier_max?: number;
note?: string;
}
export interface LoadoutsManifest {
version: 1;
aliases: Record<string, Rung>;
rungs: Record<(typeof RUNGS)[number], string[]>;
pinned: string[];
loadouts: Record<string, LoadoutDef>;
default: Record<Rung, string>;
}
export type ModuleErrorCode =
| 'LOADOUTS_INVALID'
| 'LOADOUT_UNKNOWN'
| 'MODULE_SPEC_INVALID'
| 'MODULE_UNKNOWN'
| 'MODULE_PINNED'
| 'MODULE_EXCLUSION_UNSAFE'
| 'RENDER_FAILED'
| 'RENDER_DEST_NOT_OWNED';
export class ModuleError extends Error {
constructor(readonly code: ModuleErrorCode, problem: string, readonly fix: string) {
super(`${problem}; fix: ${fix} (${code}; docs/troubleshooting.md#${code.toLowerCase().replaceAll('_', '-')})`);
}
}
const clamp = (value: string) => JSON.stringify(value.replace(/[\x00-\x1f\x7f]/g, ' ').slice(0, 120));
export function loadLoadouts(root = ROOT): LoadoutsManifest {
const file = path.join(root, LOADOUTS_FILE);
try {
return JSON.parse(fs.readFileSync(file, 'utf-8')) as LoadoutsManifest;
} catch (error) {
throw new ModuleError('LOADOUTS_INVALID', `${LOADOUTS_FILE} is unreadable or not JSON (${(error as Error).message})`, `restore it with git checkout -- ${LOADOUTS_FILE}`);
}
}
/** Every module id with its kind and source, derived from the files that define them. */
export function moduleCatalog(manifest: LoadoutsManifest, root = ROOT): ModuleEntry[] {
const pinned = new Set([...manifest.pinned, ...IMPLICIT_PINS]);
const entries: Omit<ModuleEntry, 'pinned'>[] = [];
for (const { tmpl } of discoverTemplates(root)) {
const dir = path.dirname(tmpl);
entries.push({ id: `skill:${dir === '.' ? 'gstack' : dir}`, kind: 'skill', source: tmpl });
const manifestPath = path.join(root, dir, 'sections', 'manifest.json');
if (dir === '.' || !fs.existsSync(manifestPath)) continue;
const sections = (JSON.parse(fs.readFileSync(manifestPath, 'utf-8')).sections ?? []) as Array<{ id: string; file: string }>;
for (const s of sections) entries.push({ id: `section:${dir}/${s.id}`, kind: 'section', source: `${dir}/sections/${s.file}.tmpl` });
}
for (const file of fs.readdirSync(path.join(root, 'model-overlays')).filter(f => f.endsWith('.md')).sort()) {
entries.push({ id: `overlay:${file.slice(0, -3)}`, kind: 'overlay', source: `model-overlays/${file}` });
}
for (const id of PREAMBLE_MODULE_IDS) entries.push({ id, kind: 'preamble', source: 'scripts/resolvers/preamble.ts' });
return entries.map(e => ({ ...e, pinned: pinned.has(e.id) }));
}
export function normalizeModelId(raw: string): string {
return raw.trim().toLowerCase().replace(/\[[^\]]*\]$/, '').trim().replace(/^[a-z0-9-]+\//, '');
}
const matches = (pattern: string, id: string) => (pattern.endsWith('*') ? id.startsWith(pattern.slice(0, -1)) : id === pattern);
/** The rung a model id belongs to: aliases first, then the first rung whose pattern matches. */
export function rungFor(manifest: LoadoutsManifest, raw: string): Rung {
const id = normalizeModelId(raw);
if (Object.hasOwn(manifest.aliases, id)) return manifest.aliases[id];
for (const rung of RUNGS) if (manifest.rungs[rung].some(p => matches(p, id))) return rung;
return 'unknown';
}
/** Every problem with the manifest, or [] (free test and gstack-modules validate). */
export function validateLoadouts(manifest: LoadoutsManifest, catalog: ModuleEntry[], corpus: string[] = []): string[] {
const problems: string[] = [];
const ids = new Set(catalog.map(e => e.id));
const pinned = new Set([...(manifest.pinned ?? []), ...IMPLICIT_PINS]);
if (manifest.version !== 1) problems.push(`version must be 1 (got ${clamp(String(manifest.version))})`);
for (const rung of RUNGS) if (!Array.isArray(manifest.rungs?.[rung])) problems.push(`rungs.${rung} must be a list of model-id patterns`);
for (const [alias, rung] of Object.entries(manifest.aliases ?? {})) {
if (!(RUNGS as readonly string[]).includes(rung)) problems.push(`aliases.${alias} names no rung (${clamp(rung)})`);
}
for (const id of manifest.pinned ?? []) if (!ids.has(id)) problems.push(`pinned id ${clamp(id)} is not a module`);
const loadouts = manifest.loadouts ?? {};
const has = (name: string | undefined) => name !== undefined && Object.hasOwn(loadouts, name);
if (!has(FULL) || loadouts[FULL].include !== '*') problems.push('loadouts.full must exist with include "*"');
for (const [name, def] of Object.entries(loadouts)) {
if (!NAME_RE.test(name)) problems.push(`loadout name ${clamp(name)} must match ^[a-z0-9-]+$`);
if (def.base !== undefined && !has(def.base)) problems.push(`loadouts.${name}.base names no loadout (${clamp(def.base)})`);
for (const id of [...(def.exclude ?? []), ...(Array.isArray(def.include) ? def.include : [])]) {
if (!ids.has(id)) problems.push(`loadouts.${name} names ${clamp(id)}, which is not a module`);
}
for (const id of def.exclude ?? []) {
if (pinned.has(id)) problems.push(`loadouts.${name} excludes pinned ${id}`);
const child = id.startsWith('skill:') ? [...pinned].find(p => p.startsWith(`section:${id.slice('skill:'.length)}/`)) : undefined;
if (child) problems.push(`loadouts.${name} excludes ${id}, which holds pinned ${child}`);
}
if (def.preamble_tier_max !== undefined && ![1, 2, 3, 4].includes(def.preamble_tier_max)) problems.push(`loadouts.${name}.preamble_tier_max must be 1-4`);
const seen = new Set<string>();
for (let at: string | undefined = name; at && has(at); at = loadouts[at].base) {
if (seen.has(at)) { problems.push(`loadouts.${name} has a base cycle through ${at}`); break; }
seen.add(at);
}
}
for (const rung of [...RUNGS, 'unknown'] as const) {
if (!has(manifest.default?.[rung])) problems.push(`default.${rung} names no loadout (${clamp(String(manifest.default?.[rung]))})`);
}
for (const raw of corpus) {
const id = normalizeModelId(raw);
const hits = RUNGS.filter(rung => manifest.rungs[rung]?.some(p => matches(p, id)));
if (hits.length > 1) problems.push(`model ${id} matches more than one rung (${hits.join(', ')})`);
}
return problems;
}
function assertValid(manifest: LoadoutsManifest, catalog: ModuleEntry[]): void {
const problems = validateLoadouts(manifest, catalog);
if (problems.length) throw new ModuleError('LOADOUTS_INVALID', `${LOADOUTS_FILE}: ${problems.join('; ')}`, 'fix the named entries, then run gstack-modules validate');
}
function excludedBy(manifest: LoadoutsManifest, name: string): { excluded: Set<string>; tierMax?: number } {
const def = manifest.loadouts[name];
const inherited = def.base ? excludedBy(manifest, def.base) : { excluded: new Set<string>() };
const excluded = new Set([...inherited.excluded, ...(def.exclude ?? [])]);
if (Array.isArray(def.include)) for (const id of def.include) excluded.delete(id);
return { excluded, tierMax: def.preamble_tier_max ?? inherited.tierMax };
}
export interface ModuleSelection extends ModuleFilter {
/** The normalized spec (`<loadout>` or `<base>,-id,+id`) and its sha256, for the lock file. */
spec: string;
specSha256: string;
}
/**
* Resolve `--loadout <name>` or `--modules <base>,-id,+id` to a filter.
* Returns null for the plain `full` loadout, which renders today's bytes.
*/
export function selectModules(input: { loadout?: string; modules?: string }, root = ROOT): ModuleSelection | null {
if (input.loadout !== undefined && input.modules !== undefined) {
throw new ModuleError('MODULE_SPEC_INVALID', '--loadout and --modules were both given', 'pass one: --loadout <name> or --modules <base>,-id,+id');
}
const text = (input.modules ?? input.loadout ?? '').trim();
if (!text) return null;
const manifest = loadLoadouts(root);
const catalog = moduleCatalog(manifest, root);
assertValid(manifest, catalog);
const [base, ...edits] = text.split(',').map(t => t.trim());
const names = Object.keys(manifest.loadouts).sort().join(', ');
if (!Object.hasOwn(manifest.loadouts, base)) {
throw new ModuleError('LOADOUT_UNKNOWN', `${clamp(base)} is not a loadout (valid: ${names})`, 'gstack-modules list');
}
if (input.loadout !== undefined && edits.length) {
throw new ModuleError('MODULE_SPEC_INVALID', `--loadout takes one name (got ${clamp(text)})`, 'use --modules <base>,-id,+id for edits');
}
const { excluded, tierMax } = excludedBy(manifest, base);
const ids = new Map(catalog.map(e => [e.id, e]));
for (const edit of edits) {
const op = edit[0];
const id = edit.slice(1);
if ((op !== '-' && op !== '+') || !id) {
throw new ModuleError('MODULE_SPEC_INVALID', `${clamp(edit)} is not -<module id> or +<module id>`, 'write the spec as <loadout>,-<id>,+<id>; list ids with gstack-modules list');
}
const entry = ids.get(id);
if (!entry) throw new ModuleError('MODULE_UNKNOWN', `${clamp(id)} is not a module`, 'gstack-modules list');
if (op === '-' && entry.pinned) {
throw new ModuleError('MODULE_PINNED', `${id} is pinned and can never be excluded`, `drop -${id} from the spec (pinned ids: gstack-modules list --pinned)`);
}
const pinnedChild = op === '-' && id.startsWith('skill:') ? catalog.find(e => e.pinned && e.id.startsWith(`section:${id.slice('skill:'.length)}/`)) : undefined;
if (pinnedChild) {
throw new ModuleError('MODULE_PINNED', `${id} holds pinned ${pinnedChild.id} and can never be excluded`, `drop -${id} from the spec (pinned ids: gstack-modules list --pinned)`);
}
if (op === '-') excluded.add(id); else excluded.delete(id);
}
if (excluded.size === 0 && tierMax === undefined && base === FULL) return null;
const spec = edits.length ? [base, ...edits].join(',') : base;
return {
name: edits.length ? 'adhoc' : base,
excluded,
pinned: new Set(catalog.filter(e => e.pinned).map(e => e.id)),
preambleTierMax: tierMax,
spec,
specSha256: createHash('sha256').update(spec).digest('hex'),
};
}
/** Skill directories a selection leaves out (`skill:gstack` is the root template). */
export function excludedSkillDirs(filter: ModuleFilter | null | undefined): string[] {
return [...(filter?.excluded ?? [])].filter(id => id.startsWith('skill:')).map(id => id.slice('skill:'.length));
}
+1 -1
View File
@@ -89,7 +89,7 @@ export const SELECTION_DATA_MODULES: readonly string[] = [
// The build entry list: every compiled binary, which skills reach through their `--outfile` path instead.
'scripts/build.sh',
// The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes.
'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts',
'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts',
// The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case).
'test/helpers/exercised-commands.json',
];
+2
View File
@@ -54,6 +54,7 @@ import { generatePlanRealityRows } from './plan-reality';
import { generateSidewaysSweep, generateFlakeFixNumber } from './sideways';
import { generateCoordinatorContract } from './coordinator-contract';
import { generateAutoplanDriverCodes, generateAutoplanDriverCommands, generateAutoplanDriverFiles, generateAutoplanDriverGate, generateAutoplanDriverPhases, generateAutoplanDriverResultFormat } from './autoplan-driver';
import { generateShipDriverStops } from './ship-driver';
import { generateQAScope, generateQAExploratory, generateQAFunctional, generateQAResource, generateQAReview, generateQAReviewPreflight, generateQAMethodReads } from './qa';
import { generateVoiceTriggerRouting } from './voice-triggers';
import { generateTranscriptDiscovery, generateTranscriptSources } from './transcripts';
@@ -191,6 +192,7 @@ export const RESOLVERS: Record<string, ResolverFn> = {
AUTOPLAN_DRIVER_GATE: generateAutoplanDriverGate,
AUTOPLAN_DRIVER_FILES: generateAutoplanDriverFiles,
AUTOPLAN_DRIVER_CODES: generateAutoplanDriverCodes,
SHIP_DRIVER_STOPS: generateShipDriverStops,
SECTION,
SECTION_INDEX,
TRANSCRIPT_SOURCES: generateTranscriptSources,
+4 -3
View File
@@ -24,9 +24,10 @@ const OVERLAY_DIR = path.resolve(import.meta.dir, '../../model-overlays');
const INHERIT_RE = /^\s*\{\{INHERIT:([a-z0-9-]+(?:\.[0-9]+)*)\}\}\s*\n/;
export function readOverlay(model: string, seen: Set<string> = new Set()): string {
export function readOverlay(model: string, seen: Set<string> = new Set(), excluded: ReadonlySet<string> = new Set()): string {
if (seen.has(model)) return ''; // cycle guard
seen.add(model);
if (excluded.has(`overlay:${model}`)) return '';
const filePath = path.join(OVERLAY_DIR, `${model}.md`);
if (!fs.existsSync(filePath)) return '';
@@ -36,7 +37,7 @@ export function readOverlay(model: string, seen: Set<string> = new Set()): strin
if (!match) return raw.trim();
const baseModel = match[1];
const base = readOverlay(baseModel, seen);
const base = readOverlay(baseModel, seen, excluded);
const rest = raw.replace(INHERIT_RE, '').trim();
if (!base) return rest;
@@ -46,7 +47,7 @@ export function readOverlay(model: string, seen: Set<string> = new Set()): strin
export function generateModelOverlay(ctx: TemplateContext): string {
if (!ctx.model) return '';
const content = readOverlay(ctx.model);
const content = readOverlay(ctx.model, new Set(), ctx.modules?.excluded);
if (!content) return '';
const precedence = ctx.model === 'gpt-5.6-sol'
+5 -4
View File
@@ -17,6 +17,7 @@ import { CC_BACKGROUND_DEFAULT_SINCE, FOREGROUND_IF_AVAILABLE, BACKGROUND_RECOVE
import { outsideVoiceFailurePolicy, outsideVoiceFor, outsideVoiceInvocation, outsideVoicePreflight, outsideVoiceProvenance } from './outside-voice';
import { runtimeRootPrelude } from './runtime-root';
import { DEFENSIVE_REVIEW_FRAMING, FIXTURE_SUMMARY_MODE } from './defensive-review';
import { BLOCKING_FLOOR } from '../../lib/review-scope';
const CODEX_BOUNDARY = 'Filesystem boundary: do not read or execute any files under ~/.claude/, ~/.agents/, .claude/skills/, or agents/. They hold skill definitions, not repository code to review. Do not invoke any installed skill (Codex home skills/, .agents/), hook, or tool instruction; answer directly. Do not modify agents/openai.yaml. Review only the repository code.\\n\\n';
@@ -133,9 +134,9 @@ Subagent prompt:
Read the diff for this branch. First list changed files: \`DIFF_BASE=$(git merge-base origin/<base> HEAD) && git diff --name-status "$DIFF_BASE"\`. ${FIXTURE_SUMMARY_MODE}${isShip ? ' From Step 9\'s pass 2 on, \`DIFF_BASE=$RANGE_BASE\` applies here too; \`NO FINDINGS\` is the expected result for a clean range.' : ''}
Think like an attacker and a chaos engineer. Your job is to find ways this code will fail in production. Look for: edge cases, race conditions, security holes, resource leaks, failure modes, silent data corruption, logic errors that produce wrong results silently, error handling that swallows failures, and trust boundary violations. No compliments — just the problems. For each finding, classify as FIXABLE (you know how to fix it) or INVESTIGATE (needs human judgment). After listing findings, end your output with ONE line in the canonical format \`Recommendation: <action> because <one-line reason naming the most exploitable finding>\` — examples: \`Recommendation: Fix the unbounded retry at queue.ts:78 because it'll DoS the worker pool under sustained 429s\` or \`Recommendation: Ship as-is because the strongest finding is a theoretical race that requires conditions we can't trigger in production\`. The reason must point to a specific finding (or no-fix rationale). Generic reasons like 'because it's safer' do not qualify."
Think like an attacker and a chaos engineer. Your job is to find ways this code will fail in production. Look for: edge cases, race conditions, security holes, resource leaks, failure modes, silent data corruption, logic errors that produce wrong results silently, error handling that swallows failures, and trust boundary violations. No compliments — just the problems. For each finding, classify as FIXABLE (you know how to fix it) or INVESTIGATE (needs human judgment). A finding blocks only if it ${BLOCKING_FLOOR}; end each finding with \`Severity: Critical|High|Medium|Low\` and \`Blocks: yes|no\`. After listing findings, end your output with ONE line in the canonical format \`Recommendation: <action> because <one-line reason naming the most exploitable finding>\` — examples: \`Recommendation: Fix the unbounded retry at queue.ts:78 because it'll DoS the worker pool under sustained 429s\` or \`Recommendation: Ship as-is because the strongest finding is a theoretical race that requires conditions we can't trigger in production\`. The reason must point to a specific finding (or no-fix rationale). Generic reasons like 'because it's safer' do not qualify."
Present findings under an \`ADVERSARIAL REVIEW (${outsideVoiceFor(ctx).nativeLabel} subagent):\` header. **FIXABLE findings** ${isShip ? 'are queued for the parent; do not edit during Step 11' : "are queued for the parent's Fix-First handling at Step 5; do not edit during Step 4.8"}. **INVESTIGATE findings** are presented as informational.
Present findings under an \`ADVERSARIAL REVIEW (${outsideVoiceFor(ctx).nativeLabel} subagent):\` header. **FIXABLE or \`Blocks: yes\` findings** are CRITICAL and ${isShip ? 'are queued for the parent; do not edit during Step 11' : "are queued for the parent's Fix-First handling at Step 5; do not edit during Step 4.8"}. **Other INVESTIGATE findings** are informational; \`Blocks: no\` never lowers a FIXABLE one.
If the subagent fails or times out, record native coverage as incomplete. Continue independent passes and persistence, not release.
@@ -149,11 +150,11 @@ If \`CODEX_MODE\` is \`ready\` or \`unverified\`:
Outside prompt (supply repository context from the parent):
"${CODEX_BOUNDARY}Review the changes on this branch against the base branch. Use the supplied branch diff. If it was not supplied and you have repository tools, run DIFF_BASE=$(git merge-base origin/<base> HEAD) && git diff "$DIFF_BASE". Your job is to find ways this code will fail in production. Think like an attacker and a chaos engineer. Find edge cases, race conditions, security holes, resource leaks, failure modes, and silent data corruption paths. Be adversarial. Be thorough. No compliments — just the problems. End your output with ONE line in the canonical format \`Recommendation: <action> because <one-line reason naming the most exploitable finding>\`. Generic reasons like 'because it's safer' do not qualify; the reason must point to a specific finding or no-fix rationale."
"${CODEX_BOUNDARY}Review the changes on this branch against the base branch. Use the supplied branch diff. If it was not supplied and you have repository tools, run DIFF_BASE=$(git merge-base origin/<base> HEAD) && git diff "$DIFF_BASE". Your job is to find ways this code will fail in production. Think like an attacker and a chaos engineer. Find edge cases, race conditions, security holes, resource leaks, failure modes, and silent data corruption paths. Be adversarial. Be thorough. No compliments — just the problems. A finding blocks only if it ${BLOCKING_FLOOR}; end each finding with \`Severity: Critical|High|Medium|Low\` and \`Blocks: yes|no\`. End your output with ONE line in the canonical format \`Recommendation: <action> because <one-line reason naming the most exploitable finding>\`. Generic reasons like 'because it's safer' do not qualify; the reason must point to a specific finding or no-fix rationale."
${outsideVoiceInvocation(ctx, { timeoutMs: 540000, nativeAlreadyRequired: true, diffCommand: 'DIFF_BASE=$(git merge-base origin/<base> HEAD) && git diff "$DIFF_BASE"' })}
Present the full output verbatim. ${isShip ? 'An unavailable outside challenge does not block shipping by itself; supported findings still enter Step 11, and the structured P0/P1 and non-convergence gates still apply.' : 'This outside challenge is informational; supported findings still enter Step 5 Fix-First, whose approval and convergence gates apply.'}
Present the full output verbatim. A Critical, High or \`Blocks: yes\` finding is CRITICAL; \`Blocks: no\` never lowers one, and only a named policy ground drops it. ${isShip ? 'An unavailable outside challenge does not block shipping by itself; supported findings still enter Step 11, and the structured P0/P1 and non-convergence gates still apply.' : 'This outside challenge is informational; supported findings still enter Step 5 Fix-First, whose approval and convergence gates apply.'}
**Error handling:** Only this optional outside adversarial pass is non-blocking; native completion and structured-review decisions still apply.
${outsideVoiceFailurePolicy(ctx, { timeoutMinutes: 9, onTimeout: 'missing-coverage', stderrOnEmpty: true, fallback: 'none', escape: 1 })}
+52 -37
View File
@@ -72,6 +72,47 @@ export function generatePreamble(ctx: TemplateContext): string {
return preambleSections(ctx).join('\n\n');
}
interface PreamblePiece {
/** Module id suffix (`preamble:<id>`); null for the model overlay, whose module id is `overlay:<model>`. */
id: string | null;
minTier: number;
when?: (ctx: TemplateContext) => boolean;
render: (ctx: TemplateContext, tier: number) => string;
}
/**
* The preamble's pieces in render order. AskUserQuestion Format renders BEFORE
* the model overlay so the pacing rule is the ambient default; the overlay's
* nudges land as subordinate patches (Opus 4.7 absorbs the first pacing
* directive it hits; reversing this regresses plan-review cadence, v1.6.4.0).
* Plan-mode semantics stays right after bash (so _SESSION_ID / _BRANCH / _TEL
* are live) and before every onboarding gate. The 8 one-time onboarding
* generators live in bin/gstack-skill-start's instruction layer (plan Q2/OV6/F5).
*/
export const PREAMBLE_PIECES: readonly PreamblePiece[] = [
{ id: 'bash', minTier: 1, render: generatePreambleBash },
{ id: 'make-pdf-setup', minTier: 1, when: ctx => ctx.skillName === 'make-pdf', render: generateMakePdfSetup },
{ id: 'plan-mode-info', minTier: 1, render: generatePlanModeInfo },
{ id: 'upgrade-check', minTier: 1, render: generateUpgradeCheck },
{ id: 'brain-health', minTier: 1, render: generateBrainHealthInstruction },
{ id: 'ask-user-format', minTier: 2, render: generateAskUserFormat },
{ id: 'brain-sync', minTier: 1, render: generateBrainSyncBlock },
{ id: null, minTier: 1, render: generateModelOverlay },
{ id: 'voice', minTier: 1, render: generateVoiceDirective },
{ id: 'context-recovery', minTier: 2, render: generateContextRecovery },
{ id: 'writing-style', minTier: 2, render: generateWritingStyle },
{ id: 'completeness', minTier: 2, render: generateCompletenessSection },
{ id: 'confusion', minTier: 2, render: generateConfusionProtocol },
{ id: 'evidence', minTier: 2, render: generateEvidenceDirective },
{ id: 'context-health', minTier: 2, render: generateContextHealth },
{ id: 'question-tuning', minTier: 2, render: generateQuestionTuning },
{ id: 'repo-mode', minTier: 3, render: () => generateRepoModeSection() },
{ id: 'search-before-building', minTier: 3, render: generateSearchBeforeBuildingSection },
{ id: 'completion-status', minTier: 1, render: generateCompletionStatus },
];
export const PREAMBLE_MODULE_IDS: readonly string[] = PREAMBLE_PIECES.flatMap(p => (p.id ? [`preamble:${p.id}`] : []));
/** The preamble's generator outputs in render order, empty ones dropped. */
export function preambleSections(ctx: TemplateContext): string[] {
const tier = ctx.preambleTier;
@@ -84,41 +125,15 @@ export function preambleSections(ctx: TemplateContext): string[] {
if (tier < 1 || tier > 4) {
throw new Error(`Invalid preamble-tier: ${tier} in ${ctx.tmplPath}. Must be 1-4.`);
}
const sections = [
generatePreambleBash(ctx),
...(ctx.skillName === 'make-pdf' ? [generateMakePdfSetup(ctx)] : []),
// Plan-mode-skill semantics stays near the top: after bash (so _SESSION_ID /
// _BRANCH / _TEL env vars are live) and before all onboarding gates so
// models read the authoritative "AskUserQuestion satisfies plan mode's
// end-of-turn" rule before any other instruction. Renders for all skills
// (not interactive-gated) on hosts whose capabilities declare planMode.
generatePlanModeInfo(ctx),
generateUpgradeCheck(ctx),
// Phase 2: the 8 one-time onboarding generators (lake-intro, telemetry-
// prompt, proactive-prompt, first-run-guidance, routing-injection,
// vendoring-deprecation, spawned-session-check, writing-style-migration)
// moved into bin/gstack-skill-start's instruction-emission layer — their
// text appears at runtime only when its gate fires (plan Q2/OV6/F5).
generateBrainHealthInstruction(ctx),
// AskUserQuestion Format renders BEFORE the model overlay so the pacing rule
// is the ambient default; the overlay's behavioral nudges land as subordinate
// patches. Opus 4.7 reads top-to-bottom and absorbs the first pacing directive
// it hits; reversing this order regresses plan-review cadence (v1.6.4.0 bug).
...(tier >= 2 ? [generateAskUserFormat(ctx)] : []),
generateBrainSyncBlock(ctx),
generateModelOverlay(ctx),
generateVoiceDirective(ctx, tier),
...(tier >= 2 ? [
generateContextRecovery(ctx),
generateWritingStyle(ctx),
generateCompletenessSection(ctx),
generateConfusionProtocol(ctx),
generateEvidenceDirective(ctx),
generateContextHealth(ctx),
generateQuestionTuning(ctx),
] : []),
...(tier >= 3 ? [generateRepoModeSection(), generateSearchBeforeBuildingSection(ctx)] : []),
generateCompletionStatus(ctx),
];
return sections.filter(s => s && s.trim().length > 0);
const filter = ctx.modules;
const pieceTier = (piece: PreamblePiece): number => {
if (!filter || piece.id === null || filter.pinned.has(`preamble:${piece.id}`)) return tier;
return Math.min(tier, filter.preambleTierMax ?? tier);
};
return PREAMBLE_PIECES
.filter(piece => (piece.when?.(ctx) ?? true)
&& !(piece.id && filter?.excluded.has(`preamble:${piece.id}`))
&& pieceTier(piece) >= piece.minTier)
.map(piece => piece.render(ctx, pieceTier(piece)))
.filter(s => s && s.trim().length > 0);
}
@@ -1,17 +1,13 @@
import type { TemplateContext } from '../types';
export function generateVoiceDirective(ctx: TemplateContext, tier: number): string {
const emDashRule = ctx.skillName === 'plan-eng-review'
? 'Do not add em dashes in prose you compose during the review. Existing templates, quoted text, command output, and required copied labels may contain them.'
: 'No em dashes.';
export function generateVoiceDirective(_ctx: TemplateContext, tier: number): string {
if (tier <= 1) {
return `## Voice
Direct, concrete, builder-to-builder. Name the file, function, command, and user-visible impact. No filler.
${emDashRule} No AI vocabulary: delve, crucial, robust, comprehensive, nuanced, multifaceted, load-bearing. Never corporate or academic. Short paragraphs. End with what to do.
No em dashes. No AI vocabulary: delve, crucial, robust, comprehensive, nuanced, multifaceted, load-bearing. Never corporate or academic. Short paragraphs. End with what to do.
Reply in the language of the user's latest message unless asked otherwise. Code, commands, paths, identifiers and quoted output stay verbatim.
@@ -28,7 +24,7 @@ GStack voice: Garry-shaped product and engineering judgment.
- Be direct about quality. Bugs matter. Edge cases matter. Fix the whole thing, not the demo path.
- Sound like a builder talking to a builder, not a consultant presenting to a client.
- Never corporate, academic, PR, or hype. Avoid filler, throat-clearing, generic optimism, and founder cosplay.
- ${emDashRule} No AI vocabulary: delve, crucial, robust, comprehensive, nuanced, multifaceted, furthermore, moreover, additionally, pivotal, landscape, tapestry, underscore, foster, showcase, intricate, vibrant, fundamental, significant, load-bearing.
- No em dashes. No AI vocabulary: delve, crucial, robust, comprehensive, nuanced, multifaceted, furthermore, moreover, additionally, pivotal, landscape, tapestry, underscore, foster, showcase, intricate, vibrant, fundamental, significant, load-bearing.
- Reply in the language of the user's latest message unless asked otherwise. Code, commands, paths, identifiers, quoted output and question markers (\`D<N>\`, option letters, \`(recommended)\`) stay verbatim.
- The user has context you do not: domain knowledge, timing, relationships, taste. Cross-model agreement is a recommendation, not a decision. The user decides.
+7 -2
View File
@@ -139,7 +139,7 @@ Every other specialist, including Testing, reads the full diff with fixtures.
5. Instructions:
"You are a specialist code reviewer. Read the checklist at {checklist path}, then run
\`${diffBase(ctx)} && git diff "$DIFF_BASE"\` to get the ${ctx.skillName === 'ship' ? 'diff (`<diff-base>`: write the sha in before dispatch, the merge base on pass 1 and `RANGE_BASE` from pass 2 on; never a shell variable)' : 'full diff'}. Apply the checklist against the diff.
\`${diffBase(ctx)} && git diff "$DIFF_BASE"${ctx.skillName === 'ship' ? ' <exclude>' : ''}\` to get the ${ctx.skillName === 'ship' ? 'diff (`<diff-base>`: write the sha in before dispatch, the merge base on pass 1 and `RANGE_BASE` from pass 2 on; never a shell variable; `<exclude>`: everything after `<diff-base>` on this reviewer\'s `DIFF_CMD:` line, verbatim; empty on `DIFF_EXCLUDE: none`)' : 'full diff'}. Apply the checklist against the diff.
\`NO FINDINGS\` alone is the expected, complete result for a clean diff; never invent an item. Otherwise output one JSON object per finding per line, 400 words at most:
{\\"severity\\":\\"CRITICAL|INFORMATIONAL\\",\\"confidence\\":N,\\"path\\":\\"file\\",\\"line\\":N,\\"category\\":\\"category\\",\\"summary\\":\\"description\\",\\"fix\\":\\"recommended fix\\",\\"fingerprint\\":\\"path:line:category\\",\\"specialist\\":\\"name\\"}
@@ -186,7 +186,12 @@ their source labels: specialist scoring is not the final review's defect count.
After specialist attempts settle, collect their outputs, tagged by actual source.
\`NO FINDINGS\` is a completed, clean result, never a failed reviewer. Otherwise parse
each JSON line and skip invalid lines. Missing or unusable output is incomplete
coverage, not an empty success. Keep each specialist's returned findings for activity stats.
coverage, not an empty success. Keep each specialist's returned findings for activity stats.${isShip ? `
An output that carries a state-machine matrix record (State Machines, or a batched
reader that read its checklist) is parsed from \`${ctx.paths.binDir}/gstack-state-matrix findings <file>\`,
after \`check\` ran on the raw file: an untested cell that is the only finding on its
cell comes back INFORMATIONAL (\`"state_matrix":"untested-cell"\`), the test-gap ruling
in \`docs/review-policy.md\`; a wrong row or collapsed kinds stays CRITICAL.` : ''}
#### 2. Validate severity
+19 -1
View File
@@ -108,6 +108,9 @@ export const SECTION: ResolverFn = (ctx: TemplateContext, args?: string[]): stri
const id = args?.[0];
if (!id) throw new Error('{{SECTION:id}} requires a section id');
const entry = findSection(ctx.skillName, id);
if (ctx.modules?.excluded.has(`section:${ctx.skillName}/${id}`)) {
return `> Not loaded in the ${ctx.modules!.name} loadout: skip ${entry.trigger}. Do not offer it, and treat it as not run.`;
}
if (usesLazySections(ctx.host, ctx.skillName)) {
if (ctx.skillName === 'qa' || ctx.skillName === 'qa-only') {
@@ -129,6 +132,21 @@ export const SECTION: ResolverFn = (ctx: TemplateContext, args?: string[]): stri
return fs.readFileSync(tmplPath, 'utf-8').trimEnd();
};
const SECTION_GUARD_RE = /\{\{IF_SECTION:([a-z0-9-]+)\}\}([\s\S]*?)\{\{\/IF_SECTION\}\}/g;
/**
* {{IF_SECTION:id}}…{{/IF_SECTION}}: skeleton text that belongs to one of this
* skill's sections (an offer, a self-check naming it). The markers vanish and
* the text stays when the section loads, so `full` bytes are unchanged; a
* loadout that excludes the section drops the text with it.
*/
export function applySectionGuards(content: string, ctx: TemplateContext): string {
return content.replace(SECTION_GUARD_RE, (_match, id: string, inner: string) => {
findSection(ctx.skillName, id);
return ctx.modules?.excluded.has(`section:${ctx.skillName}/${id}`) ? '' : inner;
});
}
/**
* {{SECTION_INDEX:skill}} — situation→section table from the passive manifest.
* Lazy skills only; an index would be noise for inlined skills.
@@ -149,7 +167,7 @@ export const SECTION_INDEX: ResolverFn = (ctx: TemplateContext, args?: string[])
'|------|-------------------|',
];
for (const s of manifest.sections) {
if (s.runner_only) continue;
if (s.runner_only || ctx.modules?.excluded.has(`section:${skill}/${s.id}`)) continue;
const reference = skill === 'qa' || skill === 'qa-only' ? sectionPath(ctx, skill, s.id) : `\`sections/${s.file}\``;
if (skill === 'review' && s.id === 'review-army') {
lines.push(`| Select surfaces and read QA methods | Inline in [Step 4](#step-4-critical-pass-core-review); setup and probes run in Step 4.7 |`);
+40
View File
@@ -0,0 +1,40 @@
/**
* ship/DRIVER.md resolvers (GSTA-167 PR 5): the compact entry a parent agent
* reads to run an unattended /ship, instead of ship/SKILL.md plus its sections.
* The phases live in ship/DRIVER.md.tmpl; this file owns the one list a reader
* must trust most: which heading each STOP sends the parent to. test/ship-driver.test.ts
* resolves every anchor against the generated files.
*/
import type { ResolverFn } from './types';
/** Each STOP (or read-when line) a driver run can meet, and the one heading it reads, relative to <root>/ship/. */
export const SHIP_DRIVER_STOPS: Array<{ when: string; read: string }> = [
{ when: '`merge conflicts`', read: 'SKILL.md#step-3-merge-the-base-branch-before-tests' },
{ when: '`NEXT: carry route`', read: 'SKILL.md#step-1-pre-flight' },
{ when: '`DISTRIBUTION: candidates=`', read: 'SKILL.md#step-2-distribution-pipeline-check' },
{ when: '`APPLE:` with a store ask', read: 'sections/apple-release.md#apple-app-store-release' },
{ when: '`pregate fail` or `incomplete`', read: 'sections/tests.md#step-5-run-tests-on-merged-code' },
{ when: 'a failed or zero-run lane', read: 'sections/tests.md#test-failure-ownership-triage' },
{ when: 'a red eval or a failed free-suite shard', read: 'sections/measure.md#measure-then-fix-loop-a-red-paid-eval-or-a-failed-free-suite-shard' },
{ when: 'Greptile found', read: 'sections/greptile-early.md#step-65-early-greptile-pr-after-the-free-tests-pass' },
{ when: 'coverage below target, a failed audit', read: 'sections/test-coverage.md#step-7-test-coverage-audit' },
{ when: 'plan items NOT DONE or UNVERIFIABLE', read: 'sections/plan-completion.md#gate-logic' },
{ when: '`NEXT: stop`', read: 'sections/review-army.md#decide-whether-to-repeat-step-9' },
{ when: 'a `REVIEW_SCOPE_*` refusal, the review-log row', read: 'sections/review-army.md#step-94-fix-first-and-persistence' },
{ when: 'a required probe not passed', read: 'sections/review-army.md#step-921-exploratory-qa-before-fix-first' },
{ when: 'native adversarial incomplete, queued fixes', read: 'sections/adversarial.md#finish-the-adversarial-phase' },
{ when: 'Step 11.5 mismatch', read: 'SKILL.md#step-115-bind-the-reviews' },
{ when: 'a `version` STOP', read: 'SKILL.md#step-12-version-bump-auto-decide' },
{ when: '`Documentation: blocked` or a `DOCS_CAP:` over', read: 'sections/documentation.md#blocked-recovery' },
{ when: 'finish: changed since review', read: 'SKILL.md#2-choose-the-change-route' },
{ when: 'finish: documentation inputs changed', read: 'SKILL.md#3-resolve-documentation-freshness' },
{ when: 'finish: build, lane or pregate STOP', read: 'SKILL.md#4-verify-the-frozen-candidate' },
{ when: 'a push STOP', read: 'SKILL.md#step-17-push' },
{ when: 'unattended, no open PR at HEAD', read: 'sections/handoff.md#step-18-h-handoff-without-a-pr-automation-lanes' },
{ when: 'a receipt write or `gstack-post` refusal', read: 'sections/pr-body.md#compose-the-body-then-publish-through-gstack-post' },
{ when: '`verdict=stop` or `DELEGATE:`', read: 'sections/budget-handoff.md#budget-and-context-handoff-read-when-a-cap-is-set-or-verdictdelegate-prints' },
{ when: '`QUEUE_RECEIPT_STALE`', read: '../docs/ship-policy.md#ship-before-ready' },
];
export const generateShipDriverStops: ResolverFn = () =>
SHIP_DRIVER_STOPS.map(s => `- ${s.when}: \`${s.read}\``).join('\n');
+15
View File
@@ -125,6 +125,21 @@ export interface TemplateContext {
* still apply.
*/
planMode?: boolean;
/**
* Module loadout filter (modules/loadouts.json, gen-skill-docs --loadout/--modules).
* Undefined = the `full` loadout: every module renders and committed bytes are unchanged.
*/
modules?: ModuleFilter;
}
/** A resolved loadout: which module ids it leaves out, and which ids can never be left out. */
export interface ModuleFilter {
/** Loadout name, or `adhoc` for a --modules spec. */
name: string;
excluded: ReadonlySet<string>;
pinned: ReadonlySet<string>;
/** Unpinned preamble pieces render as if the skill's preamble-tier were at most this. */
preambleTierMax?: number;
}
/** Resolver function signature. args is populated for parameterized placeholders like {{INVOKE_SKILL:name}}. */
+17 -11
View File
@@ -17,7 +17,7 @@ import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { runScriptSync } from '../lib/script-invocation';
import { planReview, specialistsFor, type DiffClass, type Dispatch, type History, type NumbersResult } from '../lib/review-depth';
import { planReview, specialistsFor, type DiffClass, type History, type NumbersResult, type PathLines } from '../lib/review-depth';
const ROOT = path.resolve(import.meta.dir, '..');
const BYTES_PER_TOKEN = 4;
@@ -30,11 +30,12 @@ const DEFAULT_OUT: Record<string, { full: number; light: number }> = {
'outside-voice': { full: 6000, light: 6000 }, 'fix-check': { full: 4000, light: 2000 }, audit: { full: 5000, light: 2500 }, 'autoplan-voice': { full: 9000, light: 4500 },
};
export interface Fixture { name: string; commit: string; paths: string[]; deleted: string[]; stamp_only?: string[]; scope: Record<string, string>; gate_tagged: number | null; diff_lines: number; diff_bytes: number; numbers: NumbersResult }
/** per_path and registry_outputs (captured since GSTA-147) feed the runtime-line count; an older fixture counts every code-class line as runtime. */
export interface Fixture { name: string; commit: string; paths: string[]; deleted: string[]; stamp_only?: string[]; scope: Record<string, string>; gate_tagged: number | null; diff_lines: number; diff_bytes: number; numbers: NumbersResult; per_path?: PathLines[]; registry_outputs?: string[] }
/** clean: pass 1 finds nothing; nits: pass 1 fixes below-floor findings; blocker: pass 1 fixes a blocker; adversarial-fix: Step 11 finds a should that is fixed; fix-regression: a pass-1 fix introduces a blocker the next step finds. */
type Scenario = 'clean' | 'nits' | 'blocker' | 'adversarial-fix' | 'fix-regression';
const SCENARIOS: Scenario[] = ['clean', 'nits', 'blocker', 'adversarial-fix', 'fix-regression'];
interface Call { role: string; name?: string; effort: 'full' | 'light' | 'host'; diffBytes: number }
interface Call { role: string; name?: string; effort: 'full' | 'medium' | 'light' | 'host'; diffBytes: number }
const fileBytes = (rel: string) => { try { return fs.statSync(path.join(ROOT, rel)).size; } catch { return 0; } };
function promptBytes(role: string, name: string): number {
@@ -62,7 +63,7 @@ function legacyShip(f: Fixture, sc: Scenario): Call[] {
}
function base(f: Fixture) {
return { paths: f.paths, deleted: f.deleted, stampOnly: f.stamp_only ?? [], scope: f.scope, gateTagged: f.gate_tagged, diffLines: f.diff_lines, numbers: f.numbers };
return { paths: f.paths, deleted: f.deleted, stampOnly: f.stamp_only ?? [], scope: f.scope, gateTagged: f.gate_tagged, diffLines: f.diff_lines, numbers: f.numbers, perPath: f.per_path, registry: f.registry_outputs ? { sha: null, outputs: f.registry_outputs } : undefined };
}
/** The new policy, step by step through lib/review-depth's own transition table. */
@@ -85,7 +86,8 @@ function newShip(f: Fixture, sc: Scenario): { calls: Call[]; ending: string } {
for (const d of plan.dispatches) calls.push({ role: d.role, name: d.name, effort: d.effort, diffBytes: d.role === 'outside-voice' ? f.diff_bytes : diffBytes });
adversarialRan ||= plan.dispatches.some(d => d.role === 'adversarial-native');
const o = outcomes[sc][Math.min(i, outcomes[sc].length - 1)] ?? {};
h = { fullPasses: h.fullPasses + (plan.next.step === 'full' ? 1 : 0), last: plan.next.step, ...o };
const last = plan.next.step === 'scoped' ? 'full' : plan.next.step;
h = { fullPasses: h.fullPasses + (last === 'full' ? 1 : 0), last, ...o };
}
for (const a of FIXED_AUDITS) calls.push({ role: 'audit', effort: 'light', diffBytes: f.diff_bytes / 2 });
return { calls, ending };
@@ -121,10 +123,10 @@ function capture(sha: string, name: string, out: string): number {
if (r.status !== 0) throw new Error(`gstack-review-plan failed: ${r.stderr}`);
const plan = JSON.parse(r.stdout);
const diffBytes = Buffer.byteLength(git(['diff', `${sha}^`, sha], dir).stdout ?? '');
const fixture: Fixture = { name, commit: sha, paths: plan.paths, deleted: plan.deleted, stamp_only: plan.stamp_only, scope: plan.scope, gate_tagged: plan.gate_tagged, diff_lines: plan.diff_lines, diff_bytes: diffBytes, numbers: plan.numbers };
const fixture: Fixture = { name, commit: sha, paths: plan.paths, deleted: plan.deleted, stamp_only: plan.stamp_only, scope: plan.scope, gate_tagged: plan.gate_tagged, diff_lines: plan.diff_lines, diff_bytes: diffBytes, numbers: plan.numbers, per_path: plan.per_path, registry_outputs: plan.registry.outputs };
fs.mkdirSync(path.dirname(out), { recursive: true });
fs.writeFileSync(out, JSON.stringify(fixture, null, 2) + '\n');
process.stdout.write(`CAPTURED: ${name} ${sha} class=${plan.class} paths=${plan.paths.length} lines=${plan.diff_lines} -> ${out}\n`);
process.stdout.write(`CAPTURED: ${name} ${sha} class=${plan.class} depth=${plan.depth} runtime=${plan.runtime_lines} paths=${plan.paths.length} lines=${plan.diff_lines} -> ${out}\n`);
return 0;
} finally {
git(['worktree', 'remove', '--force', dir]);
@@ -135,12 +137,13 @@ function capture(sha: string, name: string, out: string): number {
export function compare(fixtures: Fixture[], out: typeof DEFAULT_OUT) {
const rows: Array<Record<string, string | number>> = [];
for (const f of fixtures) {
const cls: DiffClass = planReview({ ...base(f), history: { fullPasses: 0, last: 'none' } }).class;
const first = planReview({ ...base(f), history: { fullPasses: 0, last: 'none' } });
const cls: DiffClass = first.class;
for (const sc of SCENARIOS) {
const legacy = legacyShip(f, sc);
const next = newShip(f, sc);
const lt = tokens(legacy, out); const nt = tokens(next.calls, out);
rows.push({ fixture: f.name, class: cls, scenario: sc, legacy_dispatches: legacy.length, new_dispatches: next.calls.length, legacy_tokens: lt, new_tokens: nt, ratio: Math.round((nt / lt) * 100) / 100, ending: next.ending });
rows.push({ fixture: f.name, class: cls, depth: first.depth, runtime_lines: first.runtimeLines, scenario: sc, legacy_dispatches: legacy.length, new_dispatches: next.calls.length, legacy_tokens: lt, new_tokens: nt, ratio: Math.round((nt / lt) * 100) / 100, ending: next.ending });
}
}
return rows;
@@ -158,8 +161,8 @@ function main(argv: string[]): number {
const rows = compare(fixtures, out);
const ap = fixtures.map(f => ({ fixture: f.name, runs: autoplanRuns(planReview({ ...base(f), history: { fullPasses: 0, last: 'none' } }).class === 'docs') }));
if (argv.includes('--json')) { process.stdout.write(JSON.stringify({ tokens_source: tokFile ?? 'DEFAULT', ship: rows, autoplan: ap }) + '\n'); return 0; }
const lines = [`TOKENS: ${tokFile ? `measured (${tokFile})` : 'DEFAULT estimates (pass --tokens for measured output)'}`, '', '| fixture | class | scenario | /ship dispatches legacy → new | est. tokens legacy → new | ratio | ending |', '|---|---|---|---|---|---|---|'];
for (const r of rows) lines.push(`| ${r.fixture} | ${r.class} | ${r.scenario} | ${r.legacy_dispatches} → ${r.new_dispatches} | ${r.legacy_tokens} → ${r.new_tokens} | ${r.ratio} | ${r.ending} |`);
const lines = [`TOKENS: ${tokFile ? `measured (${tokFile})` : 'DEFAULT estimates (pass --tokens for measured output)'}`, '', '| fixture | class | depth (runtime lines) | scenario | /ship dispatches legacy → new | est. tokens legacy → new | ratio | ending |', '|---|---|---|---|---|---|---|---|'];
for (const r of rows) lines.push(`| ${r.fixture} | ${r.class} | ${r.depth} (${r.runtime_lines}) | ${r.scenario} | ${r.legacy_dispatches} → ${r.new_dispatches} | ${r.legacy_tokens} → ${r.new_tokens} | ${r.ratio} | ${r.ending} |`);
lines.push('', '| fixture | /autoplan scenario | voice dispatches legacy → new |', '|---|---|---|');
for (const a of ap) for (const r of a.runs) lines.push(`| ${a.fixture} | ${r.scenario} | ${r.legacy} → ${r.next} |`);
const median = (v: number[]) => { const m = Math.floor(v.length / 2); return v.length % 2 ? v[m]! : Math.round(((v[m - 1]! + v[m]!) / 2) * 100) / 100; };
@@ -168,6 +171,9 @@ function main(argv: string[]): number {
const byClass = new Map<string, number[]>();
for (const r of rows) byClass.set(String(r.class), [...(byClass.get(String(r.class)) ?? []), Number(r.ratio)].sort((x, y) => x - y));
lines.push(`SHIP_TOKEN_RATIO_BY_CLASS: ${[...byClass].map(([c, v]) => `${c} median=${median(v)} (n=${v.length})`).join('; ')}`);
const byDepth = new Map<string, number[]>();
for (const r of rows) byDepth.set(String(r.depth), [...(byDepth.get(String(r.depth)) ?? []), Number(r.ratio)].sort((x, y) => x - y));
lines.push(`SHIP_TOKEN_RATIO_BY_DEPTH: ${[...byDepth].map(([d, v]) => `${d} median=${median(v)} (n=${v.length})`).join('; ')}`);
process.stdout.write(lines.join('\n') + '\n');
return 0;
}
+1 -1
View File
@@ -277,7 +277,7 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
return { value: `${(total / 60_000 * 7 / sinceDays).toFixed(0)} min/week (${cancelled.length} cancelled run(s))` };
}));
metrics.push(metric('push-to-verdict', 'PR lane push-to-verdict', 'created_at to updated_at of evals.yml pull_request runs that reached success or failure (includes queue and the readiness wait)', () => {
metrics.push(metric('push-to-verdict', 'PR lane push-to-verdict', 'created_at to updated_at of evals.yml pull_request runs that reached success or failure (includes queue and the readiness wait, which pr-summary runs skip since GSTA-149, so their verdicts land about 10 min sooner)', () => {
const walls = need(prRuns).filter(run => run.conclusion === 'success' || run.conclusion === 'failure').map(run => span(run.createdAt, run.updatedAt)!);
if (walls.length === 0) throw new Error('no PR run reached a verdict in the window');
return { value: `p50 ${minutes(percentile(walls, 0.5))}, p95 ${minutes(percentile(walls, 0.95))} (${walls.length} runs)` };
+24 -3
View File
@@ -69,6 +69,13 @@ export const PR_PROFILE_FILES: Record<string, readonly string[]> = {
};
const ROOT = path.resolve(import.meta.dir, '..');
/** gstack's own changelog_fragments.dir (.gstack/ship-policy.json): release notes the merge step folds into CHANGELOG.md, read by no paid case (GST-400). */
const CHANGELOG_FRAGMENTS_DIR = ((): string | null => {
try {
const dir = JSON.parse(fs.readFileSync(path.join(ROOT, '.gstack', 'ship-policy.json'), 'utf8'))?.changelog_fragments?.dir;
return typeof dir === 'string' && dir.trim() ? dir.replace(/^(\.\/)+/, '').replace(/\/+$/, '') : null;
} catch { return null; }
})();
/** The one paid test file whose source registers a case (its touchfiles name exactly one), else null. */
export function caseOwnerFile(id: string, maps: Pick<PrProfileMaps, 'e2eTouchfiles'> = PR_PROFILE_MAPS): string | null {
@@ -153,6 +160,8 @@ export interface PrProfileSelection {
directCases: string[];
/** Unmapped changed files placed by derived references (scripts/pr-dependencies.ts), with their consumer counts. */
derivedFiles: Array<{ file: string; e2e: number; judges: number }>;
/** Narrow screens: derived files whose closure reaches every gate case, left to the train gate's full census. */
hubFiles?: string[];
/** Unmapped changed files under derivable directories that no paid case's reference closure reaches. */
noConsumerFiles: string[];
deferredPromptFiles: string[];
@@ -249,6 +258,7 @@ export const FREE_ONLY_PR_FILES = [
'.github/ISSUE_TEMPLATE/**', // Issue templates: read by GitHub and free test/gstack-doctor.test.ts.
'scripts/retired-command.ts', // One-release stubs for retired package scripts; no paid case imports it.
'careful/bin/check-careful.sh', // /careful PreToolUse hook: covered by test/hook-scripts.test.ts; no paid fixture copies or runs it.
'careful/bin/mask-commit-message.awk', // Loaded only by check-careful.sh (above): covered by test/hook-scripts.test.ts.
// Reporting and launch tools: they read CI history or start a lane, never run inside a paid case.
'scripts/test-health-report.ts',
'scripts/bump-harness-version.ts',
@@ -327,6 +337,7 @@ export function knownNonBehaviorFile(file: string): boolean {
|| ['AGENTS.md', 'CLAUDE.md', 'agents-digest/gstack-AGENTS.md'].includes(file)
|| FREE_ONLY_PR_FILES.some(pattern => matchGlob(file, pattern))
|| RELEASE_METADATA_FILES.some(pattern => matchGlob(file, pattern))
|| (CHANGELOG_FRAGMENTS_DIR !== null && path.posix.dirname(file) === CHANGELOG_FRAGMENTS_DIR && file.endsWith('.md'))
|| (PERIODIC_ONLY_PR_FILES as readonly string[]).includes(file)
|| (file.startsWith('test/fixtures/') && Object.keys(FREE_FIXTURES).some(pattern => matchGlob(file, pattern)))
// Free test files anywhere (test/, browse/test/, design/test/, make-pdf/test/): no paid case runs or reads a free test.
@@ -388,6 +399,11 @@ export function selectPrProfile(options: {
derived?: DerivedDependencies | null;
/** Exercised-path consumers for the shadow block; default: the checkout's committed map for the real maps, none for fixture maps. */
exercised?: ExercisedConsumers | null;
/**
* narrow (screens and eval:bg:pr): a derived file whose closure reaches every gate case selects nothing by itself; those
* cases run in the train gate's full census. broad (the default here, and `ready --screen --broad`): every dependent runs.
*/
screenPolicy?: 'narrow' | 'broad';
}): PrProfileSelection {
const maps = options.maps ?? PR_PROFILE_MAPS;
const profile = options.profile ?? PR_PROFILE_CASE_IDS;
@@ -436,7 +452,10 @@ export function selectPrProfile(options: {
consumers.clear(); derivedGlobal.length = 0; noConsumerFiles.length = 0;
for (const file of unmapped) deletedLabels.set(file, { label: 'import scan failed', fix: `scripts/pr-dependencies.ts could not scan the imports of ${scanFailed.slice(0, 3).join(', ')}${scanFailed.length > 3 ? ` and ${scanFailed.length - 3} more` : ''}; fix the parse error so ${file}'s consumers can be derived` });
}
const derivedFiles = [...consumers.keys()];
const gateIds = Object.keys(maps.e2eTouchfiles).filter(id => maps.tiers[id] === 'gate');
const hubFiles = options.screenPolicy === 'narrow' && gateIds.length > 0
? [...consumers.keys()].filter(file => derived?.tracked.has(file) && gateIds.every(id => consumers.get(file)!.e2e.has(id))) : [];
const derivedFiles = [...consumers.keys()].filter(file => !hubFiles.includes(file));
const unknownFiles = unmapped.filter(file => !derivedGlobal.includes(file) && !consumers.has(file) && !noConsumerFiles.includes(file));
const sharedInputs = files.filter(file => depends(file, PR_FULL_GATE_FILES));
const fallback = unknownFiles.length > 0 || sharedInputs.length > 0;
@@ -476,6 +495,7 @@ export function selectPrProfile(options: {
const reasons: string[] = [];
if (unknownFiles.length) reasons.push(`Unknown dependencies restore every gate case and judge: ${unknownFiles.map(file => `${file} (${(deletedLabels.get(file) ?? unknownFileLabel(file)).label})`).join(', ')}`);
if (sharedInputs.length) reasons.push(`Shared runtime/build inputs restore every gate case and judge: ${sharedInputs.join(', ')}`);
if (hubFiles.length && !fallback && !everything) reasons.push(`Narrow screen: derived references reach every gate case only through the static closure, so these files select no case here and the train gate's full census runs them: ${hubFiles.join(', ')} (EVALS_SCREEN_POLICY=broad restores them)`);
if (dependentsMode) reasons.push(`Derived references select every dependent gate case: ${derivedFiles.map(file => `${file} (${consumers.get(file)!.e2e.size} cases)`).join(', ')}`);
else if (!fallback) reasons.push('Changed-input selection intersected with the fast PR profile; selected judges retained');
if (files.includes(EXERCISED_MAP_PATH)) reasons.push(`${EXERCISED_MAP_PATH} is shadow-only data (it changes no selection yet), so it selects no paid case (${EXERCISED_MAP_ANCHOR})`);
@@ -491,7 +511,7 @@ export function selectPrProfile(options: {
return {
mode: fallback ? 'full-fallback' : dependentsMode ? 'dependents' : 'pr', e2e, judges, deferred,
unknownFiles, unknownFileLabels: unknownFiles.map(file => ({ file, ...(deletedLabels.get(file) ?? unknownFileLabel(file)) })), directCases,
derivedFiles: derivedFiles.map(file => ({ file, e2e: consumers.get(file)!.e2e.size, judges: consumers.get(file)!.judges.size })), noConsumerFiles,
derivedFiles: derivedFiles.map(file => ({ file, e2e: consumers.get(file)!.e2e.size, judges: consumers.get(file)!.judges.size })), hubFiles, noConsumerFiles,
deferredPromptFiles, missingCoverage, needsFullValidation: missingCoverage.length > 0, reasons, ...(shadow ? { shadow } : {}),
};
}
@@ -551,7 +571,7 @@ function shadowSelection(options: Parameters<typeof selectPrProfile>[0] & { maps
export interface CoverageSummaryInput {
profile?: string;
selection?: { e2e: string[] | null; judges: string[] | null };
prCoverage?: Pick<PrProfileSelection, 'mode' | 'deferred' | 'unknownFiles'> & Partial<Pick<PrProfileSelection, 'unknownFileLabels' | 'derivedFiles' | 'noConsumerFiles' | 'shadow'>>;
prCoverage?: Pick<PrProfileSelection, 'mode' | 'deferred' | 'unknownFiles'> & Partial<Pick<PrProfileSelection, 'unknownFileLabels' | 'derivedFiles' | 'hubFiles' | 'noConsumerFiles' | 'shadow'>>;
}
/**
@@ -572,6 +592,7 @@ export function formatPrCoverageSummary(manifest: CoverageSummaryInput, totals?:
lines.push(`- Derived dependents (scripts/pr-dependencies.ts) of ${coverage.derivedFiles.length} unmapped file(s):`);
for (const item of coverage.derivedFiles.slice(0, 40)) lines.push(` - \`${item.file}\`: ${item.e2e} case(s), ${item.judges} judge(s)`);
}
if (coverage?.hubFiles?.length) lines.push(`- Left to the train gate's full census (narrow screen): ${coverage.hubFiles.slice(0, 40).map(file => `\`${file}\``).join(', ')}`);
const shadow = coverage?.shadow;
if (shadow) lines.push(shadow.error ? `- Shadow exercised-path map: unavailable (${shadow.error}); selection unaffected`
: `- Shadow exercised-path map (not active): would select ${shadow.e2e.length} gate case(s), dropping ${shadow.wouldDrop.length} and adding ${shadow.wouldAdd.length}; predicted saving $${shadow.predictedSavingUsd.toFixed(2)}${shadow.unpricedCases ? ` (${shadow.unpricedCases} unpriced)` : ''}, a lower bound: known cost excludes the ${Math.round(100 * (shadow.unknownCostShare ?? 0))}% of observed case runs with unknown cost (PTY) (${EXERCISED_MAP_ANCHOR})`);
+10 -6
View File
@@ -53,8 +53,11 @@ function noteRecords<T extends { kind: string; repo_key?: string }>(git: Git, ki
});
}
/** The newest green-train record whose prefix trees include this digest, with its run; null when none. */
export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, digest: string, lines: string[]): { run: Run; record: TrainRecord; index: number } | null {
/**
* The newest green-train record with a prefix covering main, with its run; null when none. A plain prefix T_i
* covers main's tree modulo stamps (its digest); a stamped prefix S_i covers only main's exact tree.
*/
export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, main: { digest: string | null; tree: string }, lines: string[]): { run: Run; record: TrainRecord; index: number; exact: boolean } | null {
const runs = json<{ workflow_runs?: Run[] }>(gh(['api', '-X', 'GET', `repos/${repo}/actions/workflows/${train.gate_workflow}/runs`, '-f', 'event=workflow_dispatch', '-f', 'status=success', '-f', 'per_page=50'], GH_MS));
if (!runs) { lines.push('cannot list the train gate runs; the backstop gates main'); return null; }
const trains = (runs.workflow_runs ?? []).map(run => ({ run, nonce: /\[eval-bg ([A-Za-z0-9._-]+)\]/.exec(run.display_title ?? '')?.[1] ?? '' }))
@@ -69,7 +72,7 @@ export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, d
try { record = file ? JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')) as TrainRecord : null; } catch { record = null; }
if (!record || record.kind !== 'train-gate' || record.verdict !== 'GREEN' || record.census?.complete !== true || record.nonce !== nonce
|| record.head_sha !== run.head_sha || !Array.isArray(record.prefixes)) continue;
const hit = record.prefixes.find(p => p.stamp_free_tree === digest);
const hit = record.prefixes.find(p => p.exact ? p.tree === main.tree : main.digest !== null && p.stamp_free_tree === main.digest);
if (!hit) continue;
const checks = json<{ check_runs?: Array<{ name?: string; conclusion?: string | null }> }>(
gh(['api', `repos/${repo}/commits/${run.head_sha}/check-runs?check_name=${encodeURIComponent(train.full_gate_check)}`], GH_MS));
@@ -77,7 +80,7 @@ export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, d
lines.push(`train ${nonce} (run ${run.id}) lists this tree but its ${train.full_gate_check} check run is not a success`);
continue;
}
return { run, record, index: hit.index };
return { run, record, index: hit.index, exact: !!hit.exact };
}
return null;
} finally {
@@ -150,9 +153,10 @@ export function runBackstop(o: BackstopOptions): { action: BackstopAction; lines
}
const digest = stampFreeTree(git, o.sha, o.repoRoot, loaded.policy);
if (typeof digest !== 'string') { lines.push(`BACKSTOP: no tree modulo stamps (${digest.error}); gating main`); }
const covered = typeof digest === 'string' ? coveringRecord(gh, o.repo, train, digest, lines) : null;
const tree = git(['rev-parse', `${o.sha}^{tree}`]).out;
const covered = coveringRecord(gh, o.repo, train, { digest: typeof digest === 'string' ? digest : null, tree }, lines);
if (covered) {
lines.push(`BACKSTOP: covered: main ${o.sha.slice(0, 12)} is T${covered.index} of green train ${covered.record.nonce} (run ${covered.run.id}, tree modulo stamps ${String(digest).slice(0, 12)}); no gate`);
lines.push(`BACKSTOP: covered: main ${o.sha.slice(0, 12)} is ${covered.exact ? `S${covered.index}` : `T${covered.index}`} of green train ${covered.record.nonce} (run ${covered.run.id}, ${covered.exact ? `exact tree ${tree.slice(0, 12)}` : `tree modulo stamps ${String(digest).slice(0, 12)}`}); no gate`);
return done('covered');
}
const owned = noteRecords<FreezeRecord>(git, 'freeze', repoKey).find(f => !f.cleared_at && f.main.sha === o.sha && f.backstop);
+8 -5
View File
@@ -49,7 +49,7 @@ export const MAX_BISECT_CASES = 40;
/** A full gate's recorded cost (MEASUREMENT.md: $90.93 to $94.70; lib/gate-cost.ts uses the same scale). */
export const FULL_GATE_USD = 93.14;
/** T1's prefix commits: `train T<i>: merge #<pr> at <sha12>` (lib/ship-queue-train.ts buildChain). */
const PREFIX_SUBJECT = /^train T(\d+): merge #(\d+) at [0-9a-f]{12}$/;
const PREFIX_SUBJECT = /^train (?:T(\d+): merge #(\d+) at [0-9a-f]{12}|S(\d+): merge #(\d+) at [0-9a-f]{12} v[0-9.]+)$/;
export type Git = (args: string[]) => { ok: boolean; out: string };
export const gitIn = (cwd: string): Git => (args) => {
@@ -185,9 +185,10 @@ export function stampFreeTree(git: Git, commit: string, repoRoot: string, policy
return createHash('sha256').update(lines.sort().join('\n')).digest('hex');
}
export interface TrainPrefixTree { index: number; commit: string; tree: string; stamp_free_tree: string }
/** `exact`: a stamped prefix S_i (changelog_fragments trains), which covers only its exact tree. */
export interface TrainPrefixTree { index: number; commit: string; tree: string; stamp_free_tree: string; exact?: true }
/** T1..Tk from the train head, walking first parents over T1's prefix commits. */
/** T1..Tk (or stamped S1..Sk, never mixed) from the train head, walking first parents over the prefix commits. */
export function trainPrefixes(git: Git, head: string, repoRoot: string, policy: ShipPolicy): TrainPrefixTree[] | { error: string } {
const log = git(['log', '--first-parent', `--max-count=${MAX_PREFIXES}`, '--format=%H%x00%P%x00%s', head]);
if (!log.ok) return { error: `cannot walk ${head.slice(0, 12)}'s first parents (${log.out.split('\n')[0]})` };
@@ -196,11 +197,13 @@ export function trainPrefixes(git: Git, head: string, repoRoot: string, policy:
const [commit, parents, subject] = line.split('\0') as [string, string, string];
const m = PREFIX_SUBJECT.exec(subject ?? '');
if (!m || parents.split(' ').length !== 2) break;
const index = Number(m[1]);
const index = Number(m[1] ?? m[3]);
const exact = m[3] !== undefined;
if (found.length && exact !== !!found[0]!.exact) return { error: `${commit.slice(0, 12)} mixes stamped and plain prefixes` };
if (found.length && index !== found.at(-1)!.index - 1) return { error: `prefix T${index} follows T${found.at(-1)!.index} at ${commit.slice(0, 12)}` };
const digest = stampFreeTree(git, commit, repoRoot, policy);
if (typeof digest !== 'string') return digest;
found.push({ index, commit, tree: git(['rev-parse', `${commit}^{tree}`]).out, stamp_free_tree: digest });
found.push({ index, commit, tree: git(['rev-parse', `${commit}^{tree}`]).out, stamp_free_tree: digest, ...(exact ? { exact: true as const } : {}) });
}
if (!found.length || found.at(-1)!.index !== 1) return { error: `${head.slice(0, 12)} is not a train head (no T1..Tk prefix chain)` };
return found.reverse();
+50 -10
View File
@@ -27,10 +27,18 @@
# teardown) times the size's price. Without a key it runs the same files
# locally in parallel shards and reports spent=0.
#
# Reattach: the tree and the planned set's digest go into ubi-runner's run key
# (--key-input), so a rerun after the local runner died (a machine stop, a
# SIGKILL) reattaches to the VM still running this gate instead of paying for a
# second one. Its minutes since created_at count against the cap: the wall
# budget shrinks by them (PREGATE_REMOTE_CAP_SPENT when none is left), and
# spent= bills from the original create.
#
# Exit codes: 0 pass (or a paid-only selection, tests=0); 1 test failures (or
# a rejected file list); 2 missing input; 3 tree or plan mismatch (or the tree
# moved during the run); 4 infrastructure failure (create, setup, SSH, stop
# deadline); 5 the spend cap or pregate's deadline refuses the run. Every refusal line starts with a
# deadline, another gate on this machine owning the VM to reattach to); 5 the
# spend cap or pregate's deadline refuses the run. Every refusal line starts with a
# PREGATE_REMOTE_* reason code (docs/pregate.md#remote-gate). Nothing is
# uploaded or spent before the local tree check.
set -euo pipefail
@@ -102,6 +110,7 @@ if [ -z "${UBICLOUD_API_KEY:-${UBICLOUD_API_TOKEN:-}}" ]; then
exit "$rc"
fi
[ -n "${UBI_OWNER:-}" ] || stop 2 PREGATE_REMOTE_OWNER_REQUIRED "UBI_OWNER is not set: every VM must carry its owner" "nothing created, nothing spent" "rerun with UBI_OWNER=<your thread code>: $rerun"
size=${UBI_SIZE:-standard-16}
case $size in
standard-2) ppm=0.00104 ;; standard-4) ppm=0.00208 ;; standard-8) ppm=0.00416 ;;
@@ -116,7 +125,8 @@ usd() { awk -v m="$1" -v p="$ppm" 'BEGIN { s = m * p; c = int(s * 100); if (c <
if [ "$wall_min" -lt "$min_min" ]; then
stop 5 PREGATE_REMOTE_CAP_TOO_LOW "worst case \$$(usd "$min_min") ($min_min min x \$$ppm/min on $size) exceeds the spend cap \$$cap" "nothing created, nothing spent" "rerun with --spend-cap $(usd "$min_min") or more"
fi
# The enclosing gstack-pregate kills this gate at its --timeout (default 600 s)
# The enclosing gstack-pregate kills this gate at its --timeout (default the
# policy's timeout_s, else 600 s; pregate passes it as PREGATE_TIMEOUT_S)
# with a 5 s grace, so the VM must be told to stop early enough for teardown to
# be confirmed first: 120 s reserved for teardown plus 15 s margin. The deadline
# comes from PREGATE_TIMEOUT_S, else from the gstack-pregate process this gate
@@ -145,8 +155,6 @@ if [ -n "$outer_s" ]; then
fi
if [ "$outer_stop" -lt "$stop_after" ]; then stop_after=$outer_stop stop_why="$outer_from ${outer_s}s minus 120 s teardown reserve and 15 s margin"; fi
fi
say "$count files on one $size VM; worst case \$$(usd "$wall_min") under the cap \$$cap; the VM is told to stop after ${stop_after}s ($stop_why)"
digest=$(tr -d '\r' <"$files" | grep . | LC_ALL=C sort | sha256sum | cut -d' ' -f1)
prelude="$logs/vm-setup.sh"
{
@@ -181,9 +189,31 @@ args=(run -s "$size" --src "$root" --setup "$prelude"
--pull "/tmp/gstack-free-test-*:$logs"
--pull "work/$(basename "$root")/.context/free-test-logs:$logs")
[ -z "${GSTACK_FREE_JOBS:-}" ] || args+=(--env "GSTACK_FREE_JOBS=$GSTACK_FREE_JOBS")
args+=(--key-input "pregate-remote tree=$tree plan=$digest")
# A rerun of this gate for the same tree and planned set has the same
# ubi-runner run key, so it reattaches to the VM a killed run left behind
# (`ubi-runner: reattached to <vm> created_at=<epoch>`). The minutes that VM
# has billed since created_at come off the cap's wall budget first.
prior_vm=""
attach=$(bash "$here/ubi-runner.sh" "${args[@]}" --probe -- "bash ~/.pregate-run.sh" 2>/dev/null \
| sed -n 's/^UBI_ATTACH_INFO: vm=\([^ ]*\) created_at=\([0-9]*\) key=[0-9a-f]* busy=\([01]\)$/\1 \2 \3/p' | tail -1) || attach=""
if [ -n "$attach" ]; then
read -r prior_vm prior_created prior_busy <<<"$attach"
[ "$prior_busy" = 0 ] || stop 4 PREGATE_REMOTE_ATTACH_BUSY "another gate on this machine still runs VM $prior_vm for this tree and planned set" "wait for it; this run created nothing and spent nothing"
prior_min=$(( ($(date +%s) - prior_created + 59) / 60 ))
left_s=$(( (wall_min - 3 - prior_min) * 60 ))
if [ "$left_s" -le 0 ]; then
stop 5 PREGATE_REMOTE_CAP_SPENT "VM $prior_vm, created ${prior_min} min ago for this tree and planned set, has already used the cap's $wall_min min wall (\$$(usd "$prior_min") of \$$cap)" \
"nothing reattached; destroy it: bash scripts/ubicloud/ubi-runner.sh down $prior_vm" "or rerun with a higher --spend-cap to let it finish: $rerun"
fi
if [ "$left_s" -lt "$stop_after" ]; then stop_after=$left_s stop_why="the cap's $wall_min min wall minus 3 min for teardown and $prior_min min already billed since $prior_vm was created"; fi
say "reattaching to VM $prior_vm, created ${prior_min} min ago by an earlier run of this gate for the same tree and planned set"
fi
say "$count files on one $size VM; worst case \$$(usd "$wall_min") under the cap \$$cap; the VM is told to stop after ${stop_after}s ($stop_why)"
# The VM run gets its own session (python3, which ubi-runner already needs, for
# setsid on Linux and macOS): pregate's --timeout and stall kill signal this
# setsid on Linux and macOS): pregate's --timeout kill signals this
# script's process group and then SIGKILL it, which must never cut ubi-runner's
# teardown short. Output goes to files (a dead reader would SIGPIPE ubi-runner),
# stamped for the phase timings; this script streams it. A watchdog sends
@@ -215,11 +245,11 @@ while :; do
attempt=$(( attempt + 1 ))
: >"$log"
: >"$stream"
rm -f "$log.pid" "$log.timedout"
rm -f "$log.pid" "$log.timedout" "$log.left"
t0=$(date +%s)
vm_name="ubirun-$(bash "$here/ubi-runner.sh" owner)-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')"
say "VM $vm_name; if this run is killed before its teardown line, destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name"
python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
vm_name="ubirun-$(bash "$here/ubi-runner.sh" owner)-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')-t$(( (stop_after + 900 + 3599) / 3600 ))"
[ -n "$prior_vm" ] || say "VM $vm_name; if this run is killed before its teardown line, destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name"
UBI_LEFT_FILE="$log.left" python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
bash -c "$vm_session" pregate-remote-vm "$stop_after" "$log" "$stream" "$here/ubi-runner.sh" "${args[@]}" -n "$vm_name" &
vmrun=$!
tail -n +1 -f "$stream" &
@@ -234,6 +264,9 @@ while :; do
tailer=""
t_end=$(date +%s)
t_create=$(at 'ubi-runner: creating ')
# A reattached VM is billed from its original create.
reattached=$(sed -n 's/^[0-9]* ubi-runner: reattached to [^ ]* created_at=\([0-9][0-9]*\) .*/\1/p' "$log" | tail -1)
[ -z "$reattached" ] || t_create=$reattached
if [ -n "$t_create" ] && ! grep -q 'ubi-runner: create failed' "$log"; then minutes=$(( (t_end - t_create + 59) / 60 )); billed=$(( billed + (minutes > 0 ? minutes : 1) )); fi
if [ "$rc" -ne 0 ] && [ "$attempt" -eq 1 ] && grep -q 'vCPU quota exhausted' "$log"; then
say "vCPU quota refused the VM; one retry in ${PREGATE_REMOTE_QUOTA_WAIT_S:-60} s"
@@ -253,6 +286,13 @@ span() { if [ -n "$1" ] && [ -n "$2" ]; then echo "$(( $2 - $1 ))s"; else echo "
say "phases: create $(span "$t0" "$t_ready"), setup $(span "$t_setup" "$t_run"), run $(span "$t_run" "$t_exit"), teardown $(span "$t_exit" "$t_end"); total $(( t_end - started ))s (target 600s)"
say "log: $log"
left=""
[ ! -f "$log.left" ] || left=$(sed -n 's/^UBICLOUD: left=\([^ ]*\) .*/\1/p' "$log.left" | tail -1)
if [ "$left" != 0 ]; then
stop 4 PREGATE_REMOTE_INFRA "VM cleanup not confirmed (UBICLOUD: left=${left:-missing}); no result reported" \
"VM: $vm_name; destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name" "log: $log" "spent=$spent ($billed billed min on $size)"
fi
vm_tree=$(sed -n 's/^[0-9]* PREGATE_VM_TREE \([0-9a-f]*\)$/\1/p' "$log" | tail -1)
mismatch=$(grep -m1 -oE 'PREGATE_VM_(TREE|PLAN)_MISMATCH .*' "$log" || true)
if [ -n "$mismatch" ]; then
@@ -270,7 +310,7 @@ if [ -n "$result" ] && [ "$rc" -ne 124 ]; then
echo "PREGATE_REMOTE: tree=$vm_tree tests=$(field tests "$result") failed=$(field failed "$result") spent=$spent"
exit $(( rc == 0 ? 0 : 1 ))
fi
vm=$(sed -n 's/^[0-9]* ubi-runner: creating \([^ ]*\) .*/\1/p' "$log" | tail -1)
vm=$(sed -n -e 's/^[0-9]* ubi-runner: creating \([^ ]*\) .*/\1/p' -e 's/^[0-9]* ubi-runner: reattached to \([^ ]*\) .*/\1/p' "$log" | tail -1)
down=$(grep -E 'ubi-runner: (destroyed|gone:|WARNING|FAILED to destroy)' "$log" | tail -1 | cut -d' ' -f2- || true)
cause=$(grep -E 'ubi-runner: ' "$log" | grep -vE 'ubi-runner: (destroyed|gone:|destroy requested|waiting for an in-flight)' | tail -1 | cut -d' ' -f2- || true)
[ "$rc" -ne 124 ] || cause="the VM was told to stop after ${stop_after}s ($stop_why) and torn down"
+716 -67
View File
@@ -23,28 +23,41 @@
# Needs only bash, curl, python3, ssh, ssh-keygen, and tar locally, so it works
# from dev boxes, containers, and cloud sandboxes.
#
# Ownership: VMs are named ubirun-<owner>-<epoch>-<suffix>[-t<hours>]. <owner> is UBI_OWNER
# Ownership: VMs are named ubirun-<owner>-<epoch>-<suffix>-t<hours>. <owner> is UBI_OWNER
# (lowercased, letters and digits only, at most 12 characters, starting with a
# letter) or a random per-machine id kept in $STATE_ROOT/owner-id. Ubicloud names
# allow 63 characters of [a-z0-9-], so a full name stays under 40.
# letter). `up`, `run` and `verify` refuse without it (UBI_OWNER_REQUIRED), so
# every VM names whoever can be asked about it; read-only commands fall back to a
# per-machine id kept in $STATE_ROOT/owner-id. Ubicloud names allow 63 characters
# of [a-z0-9-], so a full name stays under 45.
#
# Teardown: `up` records each VM's state (location, create status) before it
# sends the create request, and `down` destroys and polls until the VM is
# confirmed gone or provably never existed, waiting out a create that may still
# be in flight. An interrupted `up` or `run` (EXIT, INT, TERM, HUP, QUIT) runs
# that `down`; further signals are ignored until it finishes. A create refused
# be in flight. An interrupted `up` or `run` (EXIT, INT, TERM, QUIT, and HUP
# until a remote phase is live) runs that `down`; further signals are ignored
# until it finishes. A create refused
# for vCPU quota prints the project's usage per owner.
#
# Stale-VM sweeps are off by default. With UBI_GC_HOURS set to a positive
# The owner-age sweep is off by default (the TTL reap below always runs). With UBI_GC_HOURS set to a positive
# number, every `up` first destroys this owner's VMs older than that; `gc HOURS`
# runs the same sweep on demand. Neither ever touches another owner's VMs.
#
# TTL and reaper: a name this script generates ends in -t<hours> (UBI_TTL_HOURS,
# default 4; 0 omits it), the creator's own declaration of how long the VM may
# live. `reap` destroys every VM in the project whose TTL expired more than ten
# minutes ago, whichever owner made it, and every `up` runs it first (UBI_REAP=0
# skips it), so a runner killed by a sleeping machine leaves nothing behind for
# longer than its TTL. Names without a TTL are never reaped.
# 1-999, default 3), the creator's own declaration of how long the VM may live.
# `reap` destroys every VM in the project whose TTL expired more than ten
# minutes ago, whichever owner made it, and this owner's own ubirun-<owner>-*
# names with no TTL once they are UBI_LEGACY_HOURS old (default 4; another
# owner's or an untagged TTL-less VM is never reaped); every `up` runs it first (UBI_REAP=0 skips it), so a runner
# killed by a sleeping machine leaves nothing behind for longer than its TTL.
# Names outside the ubirun- prefix are never reaped.
#
# Leftover line: `run` and `verify` end with
# UBICLOUD: left=<0|1|unknown> owner=<owner> vm=<name> others=<n>
# after teardown: left counts this invocation's own VM, checked against the
# project's VM list, and others counts this owner's other VMs (information only).
# A left other than 0 turns a passing command into exit 1; `run --keep` reports
# its live VM as left=1 kept=1 and keeps its exit status. With UBI_LEFT_FILE set,
# the line is also appended to that file, out of reach of the command's output.
#
# Quota preflight: when the project's vCPU ceiling is known (UBI_QUOTA_VCPUS, or
# the maximum a refusal printed, cached in $STATE_ROOT/quota-max), `up` adds the
@@ -57,6 +70,33 @@
# UBI_RUN_TIMEOUT seconds (0, unlimited), exiting 124 with the last 40 lines,
# so a hung verify fails fast instead of holding the VM until its TTL.
#
# Detached phases and reattach: `run` and `verify` start their setup, command
# and tier phases on the VM detached from the ssh connection (setsid; stdout to
# ~/.ubirun-<phase>.out, stderr to .err, the session leader's pid in .pid, and
# the exit status in .exit, written by rename once the streams are closed) and
# stream both files back. Before creating a VM they compute a run key locally
# (the checkout's content tree, the command, the setup script's content, every
# --env entry, each --pass name with a sha256 of its value, size, location,
# image, storage, origin/HEAD, tiers and this script's own sha256) and look for
# a run record (<state dir>/<vm>/run) with the same key whose VM still exists:
# its runner alive (lease pid and start time) refuses with UBI_ATTACH_BUSY,
# otherwise the run reattaches, printing `reattached to <vm> created_at=<epoch>
# key=<k12>`, and continues from the first phase without an exit file. After
# unpack the VM recomputes the tree and refuses a mismatch (UBI_VM_TREE_MISMATCH).
# A leader gone with no exit file exits 125 (UBI_ATTACH_LOST). Signals: INT
# and TERM tear the VM down as before; HUP once a remote phase is live detaches
# (UBI_DETACHED, exit 129) and leaves the VM under its TTL; SIGKILL or a machine
# stop leaves the record and the VM, and the next run of the same command
# reattaches. `attach NAME` resumes a recorded run explicitly. Reattach never
# extends a TTL, and the reaper still applies.
#
# Tier checkpoints: a .gstack/ubicloud.json with `tiers` runs each tier as its
# own detached phase after setup, and records start/pass/fail per tier under
# UBI_CHECKPOINT_DIR/<run key>/ (write-once JSON plus the tier's log, published
# under a lock with a monotonic generation). A rerun reuses a pass younger than
# UBI_CHECKPOINT_MAX_AGE whose log re-hashes, so a resume on a fresh VM reruns
# only setup and the tiers without a pass.
#
# `verify` is the one entry point for a repository's full remote verify: it
# reads .gstack/ubicloud.json (setup, command, size, location, env, pass, pull)
# and runs it through `run`, so lanes share one committed definition instead of
@@ -75,7 +115,8 @@ CREATE_GRACE="${UBI_CREATE_GRACE:-180}"
DOWN_TIMEOUT="${UBI_DOWN_TIMEOUT:-900}"
POLL="${UBI_POLL_SECONDS:-5}"
ALLOWED_STORAGE="${UBI_ALLOWED_STORAGE:-320 640}"
TTL_HOURS="${UBI_TTL_HOURS:-4}"
TTL_HOURS="${UBI_TTL_HOURS:-3}"
LEGACY_HOURS="${UBI_LEGACY_HOURS:-4}"
REAP="${UBI_REAP:-1}"
REAP_GRACE=600
IDLE_TIMEOUT="${UBI_IDLE_TIMEOUT:-1800}"
@@ -87,6 +128,11 @@ PREFIX="ubirun"
if [ -n "${UBI_VM_RECORD:-}" ]; then VM_RECORD="$UBI_VM_RECORD"
elif [ -d "$HOME/.capy/work" ]; then VM_RECORD="$HOME/.capy/work/ubi-runner/vms.tsv"
else VM_RECORD="$STATE_ROOT/vms.tsv"; fi
if [ -n "${UBI_CHECKPOINT_DIR:-}" ]; then CHECKPOINT_DIR="$UBI_CHECKPOINT_DIR"
elif [ -d "$HOME/.capy/work" ]; then CHECKPOINT_DIR="$HOME/.capy/work/ubi-runner/checkpoints"
else CHECKPOINT_DIR="$STATE_ROOT/checkpoints"; fi
CHECKPOINT_MAX_AGE="${UBI_CHECKPOINT_MAX_AGE:-86400}"
REATTACH_MARGIN="${UBI_REATTACH_MARGIN:-900}"
die() { echo "ubi-runner: $*" >&2; exit 1; }
log() { echo "ubi-runner: $*" >&2; }
@@ -96,8 +142,7 @@ for bin in curl python3 ssh ssh-keygen tar; do
done
UBICLOUD_API_KEY="${UBICLOUD_API_KEY:-${UBICLOUD_API_TOKEN:-}}"
[[ "$GC_HOURS" =~ ^[0-9]+$ ]] || die "UBI_GC_HOURS must be a whole number of hours (unset or 0 disables the sweep)"
[[ "$TTL_HOURS" =~ ^(0|[1-9][0-9]{0,2})$ ]] || die "UBI_TTL_HOURS must be a whole number of hours from 0 to 999 with no leading zero (0 names VMs without a TTL)"
for v in IDLE_TIMEOUT RUN_TIMEOUT HEARTBEAT QUOTA_WAIT; do
for v in IDLE_TIMEOUT RUN_TIMEOUT HEARTBEAT QUOTA_WAIT CHECKPOINT_MAX_AGE REATTACH_MARGIN; do
[[ "${!v}" =~ ^[0-9]+$ ]] || die "UBI_$v must be a whole number of seconds (0 disables it)"
done
[ -z "$QUOTA_VCPUS" ] || [[ "$QUOTA_VCPUS" =~ ^[0-9]+$ ]] || die "UBI_QUOTA_VCPUS must be a whole number of vCPUs"
@@ -121,6 +166,21 @@ owner_id() {
}
OWNER=$(owner_id)
# require_owner: up, run and verify create VMs, and a VM must name its owner
# and carry a TTL. Only creates check UBI_TTL_HOURS, so an old value never
# blocks down, reap or list.
require_owner() {
[ -n "${UBI_OWNER:-}" ] || die "UBI_OWNER is not set: every VM must carry its owner; export UBI_OWNER=<your thread or session code> (for example gsta164) and rerun (UBI_OWNER_REQUIRED)"
[[ "$TTL_HOURS" =~ ^[1-9][0-9]{0,2}$ ]] || die "UBI_TTL_HOURS must be a whole number of hours from 1 to 999 with no leading zero (every VM carries a TTL; set it above your longest run)"
}
# leftover FIELDS...: print the UBICLOUD: leftover line, and append it to
# UBI_LEFT_FILE when set, so a caller can read it apart from the command's output.
leftover() {
echo "UBICLOUD: $*" >&2
[ -z "${UBI_LEFT_FILE:-}" ] || echo "UBICLOUD: $*" >>"$UBI_LEFT_FILE"
}
# api OUT ARGS...: send one Ubicloud CLI command, write the body to OUT, and
# print the HTTP status (000 when no response arrived).
api() {
@@ -269,9 +329,414 @@ watched() {
return "$rc"
}
# vm_rows: "location name owner expires" for every VM in the project; owner is
# "-" for untagged ubirun-<epoch>-* names and "." for VMs this runner did not
# name; expires is the epoch the name's -t<hours> TTL ends at, or "-".
proc_start() { ps -o lstart= -p "$1" 2>/dev/null | tr -s ' ' || true; }
fsize() { if [ -f "$1" ]; then wc -c <"$1" | tr -d ' '; else echo 0; fi; }
sha256() { if command -v sha256sum >/dev/null 2>&1; then sha256sum | cut -d' ' -f1; else shasum -a 256 | cut -d' ' -f1; fi; }
# TREE_SH: print the content tree of the current directory, the same way here
# and on the VM: at a checkout's root, `git add -A` into a copy of its index
# (tracked plus untracked files that are not ignored, the files pack uploads);
# elsewhere, the same into a scratch repository over the directory.
TREE_SH='idx=$(mktemp -d "${TMPDIR:-/tmp}/ubirun-tree.XXXXXX") || exit 1
if [ "$(git rev-parse --show-toplevel 2>/dev/null)" = "$(pwd -P)" ]; then
gi=$(git rev-parse --git-path index)
if [ "$(git config --bool core.splitIndex)" != true ] && [ -f "$gi" ]; then cp "$gi" "$idx/index"; fi
else
git init -q --bare "$idx/git" && export GIT_DIR="$idx/git" GIT_WORK_TREE=.
fi
GIT_INDEX_FILE="$idx/index" git add -A >/dev/null 2>&1 && GIT_INDEX_FILE="$idx/index" git write-tree
rc=$?; rm -rf "$idx"; exit $rc'
# vm_tree NAME QDEST: the content tree of QDEST on the VM.
vm_tree() {
remote "$1" "command -v git >/dev/null || { sudo apt-get update -qq && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git; } >/dev/null 2>&1; cd $2 && bash -c $(sq "$TREE_SH")" </dev/null
}
base_sha() { git -C "$1" rev-parse -q --verify 'refs/remotes/origin/HEAD^{commit}' 2>/dev/null || echo none; }
# phase_status NAME PHASE: "exit <rc>" once the phase wrote its exit file,
# "alive" while its session leader runs, "lost" when the leader is gone with no
# exit file, "none" when it never started.
phase_status() {
local p="~/.ubirun-$2"
remote "$1" "if [ -f $p.exit ]; then echo exit \$(cat $p.exit); elif [ -s $p.pid ] && kill -0 \$(cat $p.pid) 2>/dev/null; then echo alive; elif [ -s $p.pid ]; then echo lost; else echo none; fi" </dev/null
}
# phase_start NAME PHASE INNER: start INNER on the VM in its own session,
# detached from this ssh connection. The leader records its pid, runs INNER with
# stdout and stderr in files, and writes the exit status by rename only after
# INNER's streams are closed.
phase_start() {
local p="~/.ubirun-$2" inner
inner=$(sq "echo \$\$ >$p.pid.tmp && mv $p.pid.tmp $p.pid; ( $3 ) >$p.out 2>$p.err </dev/null; echo \$? >$p.exit.tmp && mv $p.exit.tmp $p.exit")
remote "$1" "rm -f $p.exit $p.pid; : >$p.out; : >$p.err; if command -v setsid >/dev/null 2>&1; then setsid -f bash -c $inner </dev/null >/dev/null 2>&1; else nohup bash -c $inner </dev/null >/dev/null 2>&1 & fi" </dev/null >/dev/null
}
# stream_cmd PHASE OUT_OFFSET ERR_OFFSET: follow both output files from the
# given byte offsets until the session leader exits.
stream_cmd() {
local p="~/.ubirun-$1"
printf '%s' "i=0; while [ ! -s $p.pid ] && [ ! -f $p.exit ] && [ \$i -lt 600 ]; do sleep 0.1; i=\$((i + 1)); done; pid=\$(cat $p.pid 2>/dev/null); f=; [ -z \"\$pid\" ] || f=\"-f --pid=\$pid\"; tail -c +$(( $3 + 1 )) \$f $p.err >&2 & tail -c +$(( $2 + 1 )) \$f $p.out; wait"
}
PHASE_LIVE="" STREAM_PID="" RUN_KEY="" TREE=""
# on_hup: once a remote phase is live, HUP detaches: the VM keeps running under
# its TTL and the next run of the same command reattaches. Before that, HUP
# tears down like INT and TERM.
on_hup() {
[ -n "$PHASE_LIVE" ] || exit 130
[ -z "$STREAM_PID" ] || kill -TERM -- "-$STREAM_PID" 2>/dev/null || true
trap - EXIT
rm -f "$(state_dir "$RUN_VM")/lease"
leftover "left=1 owner=$OWNER vm=$RUN_VM detached=1"
echo "UBI_DETACHED: vm=$RUN_VM key=${RUN_KEY:0:12} reattach: rerun the same command" >&2
exit 129
}
# detached NAME PHASE INNER: run INNER as a detached phase (or reattach to it),
# streaming its output to this process and to the VM's state dir
# (<phase>.out, <phase>.err and the combined <phase>.log) under the same
# heartbeat and watchdog as watched. RUN_TIMEOUT counts from the phase's
# recorded start, not from a reattach. Returns the phase's exit status, 124
# after a watchdog kill, 125 when the session is gone without an exit status.
detached() {
local name=$1 phase=$2 inner=$3 dir st rc=0 tries=0 start now idle elapsed last_beat why old_traps spid sst oo oe
load "$name"
dir=$(state_dir "$name")
local lout="$dir/$phase.out" lerr="$dir/$phase.err" out="$dir/$phase.log" p="~/.ubirun-$phase"
st=none
if [ -f "$dir/phase.$phase.start" ]; then
st=$(phase_status "$name" "$phase") || st=""
fi
case $st in
none)
: >"$lout"; : >"$lerr"; : >"$out"
date +%s >"$dir/phase.$phase.start"
phase_start "$name" "$phase" "$inner" || { log "could not start $phase on $name"; return 255; } ;;
alive) log "$phase is still running on $name; following it" ;;
"exit "*) ;;
lost) log "UBI_ATTACH_LOST: vm=$name phase=$phase: its session is gone and left no exit status ($p.exit)"; return 125 ;;
*) log "could not read the state of $phase on $name"; return 255 ;;
esac
PHASE_LIVE=1
start=$(cat "$dir/phase.$phase.start")
# A runner killed on this machine leaves its stream client behind; it would
# append to the same local copies.
if [ -f "$dir/stream" ] && read -r spid sst <"$dir/stream" && [ -n "$spid" ] && [ "$(proc_start "$spid")" = "$sst" ]; then
kill -TERM -- "-$spid" 2>/dev/null || true
fi
oo=$(fsize "$lout") oe=$(fsize "$lerr")
if [ "$st" != none ]; then head -c "$oo" "$lout"; head -c "$oe" "$lerr" >&2; fi
old_traps=$(trap -p INT TERM)
trap '[ -z "$STREAM_PID" ] || kill -TERM -- "-$STREAM_PID" 2>/dev/null; exit 130' INT TERM
last_beat=$(date +%s)
while :; do
touch "$out"
# shellcheck disable=SC2046
( exec python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
ssh $(ssh_opts) "ubi@$IP" "umask 022; $(stream_cmd "$phase" "$oo" "$oe")" ) \
</dev/null > >(tee -a "$lout" | tee -a "$out") 2> >(tee -a "$lerr" | tee -a "$out" >&2) &
STREAM_PID=$!
echo "$STREAM_PID $(proc_start "$STREAM_PID")" >"$dir/stream"
why=""
while :; do
sleep 1
kill -0 "$STREAM_PID" 2>/dev/null || break
now=$(date +%s)
elapsed=$(( now - start ))
idle=$(( now - $(mtime "$out") ))
if [ "$IDLE_TIMEOUT" -gt 0 ] && [ "$idle" -ge "$IDLE_TIMEOUT" ]; then
why="printed nothing for ${idle}s (UBI_IDLE_TIMEOUT=$IDLE_TIMEOUT)"
elif [ "$RUN_TIMEOUT" -gt 0 ] && [ "$elapsed" -ge "$RUN_TIMEOUT" ]; then
why="ran ${elapsed}s (UBI_RUN_TIMEOUT=$RUN_TIMEOUT)"
fi
[ -z "$why" ] || break
if [ "$HEARTBEAT" -gt 0 ] && [ $(( now - last_beat )) -ge "$HEARTBEAT" ]; then
log "heartbeat: $phase ${elapsed}s elapsed, last output ${idle}s ago"
last_beat=$now
fi
done
if [ -n "$why" ]; then
log "watchdog: $phase $why; killing it. Last output:"
tail -n 40 "$out" >&2
kill -TERM -- "-$STREAM_PID" 2>/dev/null || kill "$STREAM_PID" 2>/dev/null || true
wait "$STREAM_PID" 2>/dev/null || true
STREAM_PID=""
# A phase that finished in the instant before the kill keeps its exit 0.
if [ "$(phase_status "$name" "$phase" 2>/dev/null)" = "exit 0" ]; then
log "watchdog: $phase finished before it could be killed; its exit 0 stands"
rc=0
else
remote_cancel "$name" "$p.pid"
rc=124
fi
break
fi
wait "$STREAM_PID" 2>/dev/null || true
STREAM_PID=""
st=$(phase_status "$name" "$phase") || st=""
case $st in
"exit "*) rc=${st#exit }; break ;;
lost|none) log "UBI_ATTACH_LOST: vm=$name phase=$phase: its session is gone and left no exit status ($p.exit)"; rc=125; break ;;
esac
tries=$(( tries + 1 ))
if [ "$tries" -ge 5 ]; then log "lost the connection to $phase on $name 5 times; giving up (the VM keeps it until teardown)"; rc=255; break; fi
log "the stream of $phase on $name dropped; following it again"
sleep "$POLL"
oo=$(fsize "$lout") oe=$(fsize "$lerr")
done
rm -f "$dir/stream"
trap - INT TERM; eval "$old_traps"
return "$rc"
}
# write_record DIR: the run record a later run of the same command reattaches
# by (key, tree, and this invocation for `attach NAME`). Remote files of each
# phase: ~/.ubirun-<phase>.{out,err,exit,pid} (setup, command, tier-<name>).
qwords() { local x; for x in "$@"; do printf '%q ' "$x"; done; }
write_record() {
{
printf 'RUN_KEY=%q\nTREE=%q\nPULL_ROOT=%q\nRESUME=%q\nRERUN_FAILED=%q\n' "$RUN_KEY" "$TREE" "$PULL_ROOT" "$RESUME" "$RERUN_FAILED"
printf 'RUN_ARGS=(%s)\n' "$(qwords "${RUN_ARGS[@]}")"
printf 'TIER_NAMES=(%s)\n' "$(qwords ${TIER_NAMES[@]+"${TIER_NAMES[@]}"})"
printf 'TIER_CMDS=(%s)\n' "$(qwords ${TIER_CMDS[@]+"${TIER_CMDS[@]}"})"
echo "# remote files: ~/.ubirun-<phase>.{out,err,exit,pid}; phase starts: phase.<phase>.start; lease: lease (pid start time)"
} >"$1/run.$$" && mv "$1/run.$$" "$1/run"
}
lease_take() { echo "$$ $(proc_start $$)" >"$1/lease.$$" && mv "$1/lease.$$" "$1/lease"; }
lease_busy() {
local pid st
[ -f "$1/lease" ] || return 1
read -r pid st <"$1/lease" || true
[ -n "$pid" ] && [ "$pid" != "$$" ] && kill -0 "$pid" 2>/dev/null && [ "$(proc_start "$pid")" = "$st" ]
}
# vm_alive NAME: the recorded VM still answers show. One confirmed gone is
# cleaned up (state dir and scratch record), so its record is never tried again.
vm_alive() {
local n=$1 out code
( load "$n"; [ "$CREATED" = yes ] && [ -n "$IP" ] ) 2>/dev/null || return 1
out=$(mktemp)
code=$(load "$n"; api "$out" vm "$LOCATION/$n" show)
rm -f "$out"
case $code in
200) return 0 ;;
404) log "run record for $n: the VM is gone; starting a new one"; cmd_down "$n" >/dev/null 2>&1 || true; return 1 ;;
*) return 1 ;;
esac
}
# ttl_short NAME SETUP: true when NAME's TTL (its created_at plus the name's
# -t<hours>, minus now) does not cover the run still to do plus
# REATTACH_MARGIN, with the numbers in TTL_WHY. The run still to do is the
# UBI_RUN_TIMEOUT bound of each phase that has not finished (setup when SETUP is
# set, then the command or every tier the plan does not reuse), counted from a
# started phase's recorded start; it is 0 when UBI_RUN_TIMEOUT is 0.
TTL_WHY=""
ttl_short() {
local name=$1 dir created left now need=0 p i phases=()
[[ "$name" =~ -t([0-9]{1,3})$ ]] || return 1
dir=$(state_dir "$name")
created=$(load "$name"; echo "$CREATE_AT")
now=$(date +%s)
left=$(( created + 10#${BASH_REMATCH[1]} * 3600 - now ))
[ -z "$2" ] || phases+=(setup)
if [ ${#TIER_NAMES[@]} -gt 0 ]; then
for i in "${!TIER_NAMES[@]}"; do [ "${TIER_ACT[i]}" = reuse ] || phases+=("tier-${TIER_NAMES[i]}"); done
else
phases+=(command)
fi
if [ "$RUN_TIMEOUT" -gt 0 ]; then
for p in "${phases[@]}"; do
if [ -f "$dir/phase.$p.start" ]; then i=$(( RUN_TIMEOUT - (now - $(cat "$dir/phase.$p.start")) )); [ "$i" -gt 0 ] || i=0
else i=$RUN_TIMEOUT; fi
need=$(( need + i ))
done
fi
TTL_WHY="ttl_left=${left}s needs=$(( need + REATTACH_MARGIN ))s (run still to do ${need}s + UBI_REATTACH_MARGIN=${REATTACH_MARGIN}s)"
[ "$left" -lt $(( need + REATTACH_MARGIN )) ]
}
# find_record: the VM of a run record with this RUN_KEY whose VM still exists.
find_record() {
local f n
for f in "$STATE_ROOT"/*/run; do
[ -f "$f" ] || continue
grep -qx "RUN_KEY=$RUN_KEY" "$f" || continue
n=$(basename "$(dirname "$f")")
if vm_alive "$n"; then echo "$n"; return 0; fi
done
}
# Tier checkpoints: write-once records <tier>.<epoch>.<pid>.json beside a copy
# of the tier's log, under CHECKPOINT_DIR/<run key>/, published under a lock
# with a monotonic generation; the newest generation per tier decides.
CKPT_PY='
import fcntl, hashlib, json, os, shutil, sys, time
op, d = sys.argv[1], sys.argv[2]
def records():
out = []
for f in (os.listdir(d) if os.path.isdir(d) else []):
if not f.endswith(".json") or f.startswith("."):
continue
try:
r = json.load(open(os.path.join(d, f)))
except (OSError, ValueError):
continue
if isinstance(r, dict) and isinstance(r.get("generation"), int):
r["_file"] = f
out.append(r)
return out
def newest(rs, tier):
rs = [r for r in rs if r.get("tier") == tier]
return max(rs, key=lambda r: r["generation"]) if rs else None
def sha(p):
h = hashlib.sha256()
with open(p, "rb") as fh:
for b in iter(lambda: fh.read(65536), b""):
h.update(b)
return h.hexdigest()
if op == "decide":
now, maxage, resume, rerun = int(sys.argv[3]), int(sys.argv[4]), sys.argv[5] == "1", sys.argv[6] == "1"
rs = records()
for tier in sys.argv[7:]:
r = newest(rs, tier)
f = r["_file"] if r else "-"
age = now - int(r.get("finished_at") or r.get("started_at") or now) if r else 0
dur = int(r.get("duration_s") or 0) if r else 0
code = str(r.get("exit", "-")) if r else "-"
log = os.path.join(d, str(r.get("log"))) if r and r.get("log") else ""
if not resume:
act, why = "run", "--no-resume"
elif r is None:
act, why = "run", "no record"
elif r.get("status") == "start":
act, why = "run", "interrupted (only a start record, %s)" % f
elif r.get("status") == "pass":
if age >= maxage:
act, why = "run", "pass %s is %ds old (UBI_CHECKPOINT_MAX_AGE=%d)" % (f, age, maxage)
elif not log or not os.path.isfile(log) or sha(log) != r.get("log_sha256"):
act, why = "run", "pass %s: its log does not re-hash" % f
else:
act, why = "reuse", "-"
elif r.get("status") == "fail":
act, why = ("rerun", "prior fail %s (exit %s; --rerun-failed)" % (f, code)) if rerun else ("blocked", "-")
else:
act, why = "run", "unreadable record %s" % f
print("\t".join([tier, act, f, str(age), str(dur), code, why]))
elif op == "publish":
tier, status, vm, started, finished, code, log, startgen, key, tree, pid = sys.argv[3:14]
os.makedirs(d, exist_ok=True)
with open(os.path.join(d, ".lock"), "a") as lk:
fcntl.flock(lk, fcntl.LOCK_EX)
rs = records()
gen = max([r["generation"] for r in rs] + [0]) + 1
if status != "start":
r = newest(rs, tier)
if r is None or str(r["generation"]) != startgen:
print("stale writer: its start record is generation %s, the newest for %s is %s" % (startgen, tier, r["generation"] if r else "none"), file=sys.stderr)
sys.exit(2)
rec = {"tier": tier, "status": status, "generation": gen, "vm": vm, "key": key, "tree": tree, "started_at": int(started)}
if status != "start":
rec.update(finished_at=int(finished), duration_s=int(finished) - int(started), exit=int(code))
epoch = int(time.time())
while os.path.exists(os.path.join(d, "%s.%d.%s.json" % (tier, epoch, pid))) or os.path.exists(os.path.join(d, "%s.%d.%s.log" % (tier, epoch, pid))):
epoch += 1
stem = "%s.%d.%s" % (tier, epoch, pid)
if status != "start":
shutil.copyfile(log, os.path.join(d, stem + ".log"))
rec["log"], rec["log_sha256"] = stem + ".log", sha(os.path.join(d, stem + ".log"))
tmp = os.path.join(d, ".%s.tmp" % stem)
with open(tmp, "w") as fh:
json.dump(rec, fh, sort_keys=True)
fh.write("\n")
os.link(tmp, os.path.join(d, stem + ".json"))
os.unlink(tmp)
print(gen)
'
# ckpt_publish TIER STATUS VM STARTED FINISHED EXIT LOG STARTGEN: print the new
# record's generation; fails (nothing published) for a stale writer.
ckpt_publish() { python3 -c "$CKPT_PY" publish "$CHECKPOINT_DIR/$RUN_KEY" "$@" "$RUN_KEY" "$TREE" "$$"; }
TIER_ACT=() TIER_TO_RUN=0 TIER_PASSED=0 TIER_REUSED="" TIER_FAILED="" TIER_PRIOR="" TIER_SAVED=0 TIER_BLOCKED="" TIER_BLOCKED_RC=1
# tier_plan: decide each tier from its newest checkpoint and print one
# UBI_VERIFY_RESUME line per tier before anything runs.
tier_plan() {
local plan t act f age dur code why
plan=$(python3 -c "$CKPT_PY" decide "$CHECKPOINT_DIR/$RUN_KEY" "$(date +%s)" "$CHECKPOINT_MAX_AGE" "$RESUME" "$RERUN_FAILED" "${TIER_NAMES[@]}") \
|| die "verify: could not read the tier checkpoints in $CHECKPOINT_DIR/$RUN_KEY"
while IFS=$'\t' read -r t act f age dur code why; do
case $act in
reuse)
echo "UBI_VERIFY_RESUME: tier=$t reuse $f (${age}s)" >&2
TIER_REUSED=${TIER_REUSED:+$TIER_REUSED,}$t TIER_SAVED=$(( TIER_SAVED + dur )) TIER_PASSED=$(( TIER_PASSED + 1 )) ;;
blocked)
echo "UBI_VERIFY_RESUME: tier=$t fail $f (${age}s, exit $code): not rerun without --rerun-failed" >&2
if [ -z "$TIER_BLOCKED" ]; then TIER_BLOCKED=$t; if [[ "$code" =~ ^[0-9]+$ ]]; then TIER_BLOCKED_RC=$code; fi; fi ;;
rerun)
echo "UBI_VERIFY_RESUME: tier=$t run: $why" >&2
TIER_PRIOR=${TIER_PRIOR:+$TIER_PRIOR,}$t:$code TIER_TO_RUN=$(( TIER_TO_RUN + 1 )) ;;
*)
echo "UBI_VERIFY_RESUME: tier=$t run: $why" >&2
TIER_TO_RUN=$(( TIER_TO_RUN + 1 )) ;;
esac
TIER_ACT+=("$act")
done <<<"$plan"
}
verify_line() {
echo "UBI_VERIFY: tree=${TREE:0:12} key=${RUN_KEY:0:12} tiers=${#TIER_NAMES[@]} passed=$TIER_PASSED reused=${TIER_REUSED:-none} failed=${TIER_FAILED:-none} prior_failures=${TIER_PRIOR:-none} saved_s=$TIER_SAVED"
}
# run_tiers NAME QDEST: run every tier the plan did not reuse, in order, each
# as its own detached phase; stop at the first failure and return its status.
# A pass is recorded only when the VM's tree after the tier equals the tree
# before it.
run_tiers() {
local name=$1 qdest=$2 dir i t phase before after trc started finished gen
dir=$(state_dir "$name")
for i in "${!TIER_NAMES[@]}"; do
[ "${TIER_ACT[i]}" != reuse ] || continue
t=${TIER_NAMES[i]} phase="tier-${TIER_NAMES[i]}"
if [ ! -f "$dir/phase.$phase.start" ]; then
before=$(vm_tree "$name" "$qdest") || before=unavailable
echo "$before" >"$dir/phase.$phase.tree"
if gen=$(ckpt_publish "$t" start "$name" "$(date +%s)" 0 0 "" 0); then echo "$gen" >"$dir/phase.$phase.gen"
else echo "UBI_CHECKPOINT_WRITE_FAILED: tier=$t start record (the tier still runs)" >&2; fi
fi
log "running tier $t on $name: ${TIER_CMDS[i]}"
trc=0
detached "$name" "$phase" "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "${TIER_CMDS[i]}")" || trc=$?
started=$(cat "$dir/phase.$phase.start" 2>/dev/null || date +%s) finished=$(date +%s)
log "tier $t exited $trc after $(( finished - started ))s"
case $trc in 125|255) TIER_FAILED=$t; return "$trc" ;; esac
gen=$(cat "$dir/phase.$phase.gen" 2>/dev/null || echo 0)
if [ "$trc" -ne 0 ]; then
ckpt_publish "$t" fail "$name" "$started" "$finished" "$trc" "$dir/$phase.log" "$gen" >/dev/null \
|| echo "UBI_CHECKPOINT_WRITE_FAILED: tier=$t fail record (the result stands)" >&2
TIER_FAILED=$t
return "$trc"
fi
TIER_PASSED=$(( TIER_PASSED + 1 ))
after=$(vm_tree "$name" "$qdest") || after=unavailable
before=$(cat "$dir/phase.$phase.tree" 2>/dev/null || echo unavailable)
if [ "$after" != "$before" ] || [ "$after" = unavailable ]; then
echo "UBI_VERIFY_TREE_MOVED: tier=$t before=${before:0:12} after=${after:0:12}: a tier must leave the source tree as it found it; no pass record" >&2
continue
fi
ckpt_publish "$t" pass "$name" "$started" "$finished" 0 "$dir/$phase.log" "$gen" >/dev/null \
|| echo "UBI_CHECKPOINT_WRITE_FAILED: tier=$t pass record (the result stands)" >&2
done
}
# vm_rows: "location name owner expires created" for every VM in the project;
# owner is "-" for untagged ubirun-<epoch>-* names and "." for VMs this runner
# did not name; expires is the epoch the name's -t<hours> TTL ends at, or "-";
# created is the name's epoch, or "-".
vm_rows() {
local loc name owner created expires
cli vm list -N -f location,name | while read -r loc name; do
@@ -284,23 +749,31 @@ vm_rows() {
if [ -n "$created" ] && [[ "$name" =~ -t([0-9]{1,3})$ ]]; then
expires=$(( 10#$created + 10#${BASH_REMATCH[1]} * 3600 ))
fi
echo "$loc $name $owner $expires"
echo "$loc $name $owner $expires ${created:--}"
done
}
# reap: destroy every VM whose name carries a TTL that ended more than
# REAP_GRACE seconds ago, whichever owner created it (the TTL is that owner's
# own declaration). Names without a TTL are never touched.
# own declaration), and the caller's own TTL-less ubirun-<UBI_OWNER>-* names at
# least LEGACY_HOURS old. Another owner's TTL-less VM, an untagged one, and names
# outside the prefix are never touched: only a TTL lets reap judge another lane's VM.
cmd_reap() {
local now rows loc name owner expires n=0 failed=0
local now rows loc name owner expires created n=0 failed=0
[[ "$LEGACY_HOURS" =~ ^[1-9][0-9]{0,3}$ ]] || die "UBI_LEGACY_HOURS must be a whole number of hours from 1 to 9999 with no leading zero (how old a TTL-less ubirun-* VM must be before reap destroys it)"
now=$(date +%s)
rows=$(vm_rows) || { log "reap: could not list the project's VMs; nothing destroyed"; return 1; }
while read -r loc name owner expires; do
while read -r loc name owner expires created; do
[ -n "$name" ] || continue
[ "$expires" != - ] || continue
[ "$now" -ge $(( expires + REAP_GRACE )) ] || continue
if [ "$expires" != - ]; then
[ "$now" -ge $(( expires + REAP_GRACE )) ] || continue
log "reaping expired $loc/$name (owner $owner; its TTL ended $(( (now - expires) / 60 )) min ago)"
else
[ -n "${UBI_OWNER:-}" ] && [ "$owner" = "$OWNER" ] && [ "$created" != - ] || continue
[ $(( now - 10#$created )) -ge $(( LEGACY_HOURS * 3600 )) ] || continue
log "reaping TTL-less $loc/$name (owner $owner; $(( (now - 10#$created) / 3600 ))h old, legacy sweep at ${LEGACY_HOURS}h)"
fi
n=$(( n + 1 ))
log "reaping expired $loc/$name (owner $owner; its TTL ended $(( (now - expires) / 60 )) min ago)"
cmd_down "$name" -l "$loc" || { failed=$(( failed + 1 )); log "failed to reap $loc/$name"; }
done <<<"$rows"
[ "$n" -gt 0 ] || log "reap: no expired VMs"
@@ -379,9 +852,7 @@ cmd_records() {
}
new_name() {
local ttl=""
[ "$TTL_HOURS" -eq 0 ] || ttl="-t$TTL_HOURS"
echo "$PREFIX-$OWNER-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')$ttl"
echo "$PREFIX-$OWNER-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')-t$TTL_HOURS"
}
# vcpus SIZE: the vCPU count a Ubicloud size name carries (standard-16 -> 16).
@@ -438,6 +909,7 @@ check_storage() {
UP_NAME=""
UP_READY=""
cmd_up() {
require_owner
local size=$DEFAULT_SIZE location=$DEFAULT_LOCATION name="" storage="" image=ubuntu-noble
while [ $# -gt 0 ]; do
case $1 in
@@ -715,8 +1187,31 @@ inside_dir() {
# PULL_ROOT: set by verify; every pull destination must resolve inside it at
# pull time (an earlier pull may have placed a symlink where a later one lands).
PULL_ROOT=""
# run_teardown RC: run's EXIT handler. Destroys RUN_VM, ignoring further
# signals, then checks the project's VM list for it and prints the leftover
# line; a VM still listed, or a list that cannot be read, fails the run.
run_teardown() {
local rc=$1 rows left others
trap "" INT TERM HUP QUIT
cmd_down "$RUN_VM" || log "WARNING: failed to destroy $RUN_VM; run: $0 down $RUN_VM"
if rows=$(vm_rows 2>/dev/null); then
left=$(awk -v n="$RUN_VM" '$2==n' <<<"$rows" | grep -c . || true)
others=$(awk -v n="$RUN_VM" -v o="$OWNER" '$3==o && $2!=n' <<<"$rows" | grep -c . || true)
else
left=unknown others=unknown
fi
leftover "left=$left owner=$OWNER vm=$RUN_VM others=$others"
[ "$left" = 0 ] && return 0
log "VM cleanup not confirmed for $RUN_VM (left=$left); destroy it with: $0 down $RUN_VM"
[ "$rc" -ne 0 ] || exit 1
}
TIER_NAMES=() TIER_CMDS=() RESUME=1 RERUN_FAILED=0 FORCE_ATTACH="" RUN_ARGS=()
cmd_run() {
local up_args=() src=. dest="" setup="" keep=0 pulls=() envs=()
require_owner
local up_args=() src=. dest="" setup="" keep=0 pulls=() envs=() passes=() key_inputs=() probe=0
RUN_ARGS=("$@")
while [ $# -gt 0 ]; do
case $1 in
-S|--storage) check_storage "$2"; up_args+=("$1" "$2"); shift 2 ;;
@@ -725,57 +1220,161 @@ cmd_run() {
--dest) dest=$2; shift 2 ;;
--setup) setup=$2; shift 2 ;;
--env) envs+=("$2"); shift 2 ;;
--pass) pass_name "$2"; [ -n "${!2+x}" ] || die "--pass $2: not set locally"; envs+=("$2=${!2}"); shift 2 ;;
--pass) pass_name "$2"; [ -n "${!2+x}" ] || die "--pass $2: not set locally"; passes+=("$2"); shift 2 ;;
--pull) pulls+=("$2"); shift 2 ;;
--keep) keep=1; shift ;;
--key-input) key_inputs+=("$2"); shift 2 ;;
--probe) probe=1; shift ;;
--resume) RESUME=1; shift ;;
--no-resume) RESUME=0; shift ;;
--rerun-failed) RERUN_FAILED=1; shift ;;
--) shift; break ;;
*) die "run: unknown option $1" ;;
esac
done
[ $# -gt 0 ] || die "run: missing command after --"
[ $# -gt 0 ] || [ ${#TIER_NAMES[@]} -gt 0 ] || die "run: missing command after --"
[ -z "$setup" ] || [ -f "$setup" ] || die "setup script not found: $setup"
command -v git >/dev/null || die "git is required: the run key fingerprints the checkout with it"
src=$(cd "$src" && pwd)
[ -n "$dest" ] || dest="work/$(basename "$src")"
local i name=""
local i name="" size=$DEFAULT_SIZE location=$DEFAULT_LOCATION image=ubuntu-noble storage="" e n
for (( i = 0; i < ${#up_args[@]}; i += 2 )); do
case ${up_args[i]} in -n|--name) name=${up_args[i+1]} ;; esac
case ${up_args[i]} in
-n|--name) name=${up_args[i+1]} ;;
-s|--size) size=${up_args[i+1]} ;;
-l|--location) location=${up_args[i+1]} ;;
-b|--image) image=${up_args[i+1]} ;;
-S|--storage) storage=${up_args[i+1]} ;;
esac
done
# The run key: everything that decides what the VM runs, computed before any
# VM exists. Pass values enter only as their sha256.
if [ -z "$FORCE_ATTACH" ]; then
TREE=$(cd "$src" && bash -c "$TREE_SH") || die "could not fingerprint $src (git add -A into a temporary index failed)"
RUN_KEY=$({
echo "ubi-runner run key v1"
echo "tree $TREE"
printf 'command %q\n' "$*"
if [ -n "$setup" ]; then echo "setup $(sha256 <"$setup")"; else echo "setup none"; fi
for e in ${envs[@]+"${envs[@]}"}; do printf 'env %q\n' "$e"; done
for n in ${passes[@]+"${passes[@]}"}; do printf 'pass %s %s\n' "$n" "$(printf %s "${!n}" | sha256)"; done
printf 'vm %q %q %q %q\n' "$size" "$location" "$image" "$storage"
echo "base $(base_sha "$src")"
echo "runner $(sha256 <"${BASH_SOURCE[0]}")"
for e in ${key_inputs[@]+"${key_inputs[@]}"}; do printf 'input %q\n' "$e"; done
for i in "${!TIER_NAMES[@]}"; do printf 'tier %q %q\n' "${TIER_NAMES[i]}" "${TIER_CMDS[i]}"; done
} | sha256)
fi
[ ${#TIER_NAMES[@]} -eq 0 ] || tier_plan
local attach=""
if [ -n "$FORCE_ATTACH" ]; then
vm_alive "$FORCE_ATTACH" || die "attach: $FORCE_ATTACH no longer exists; rerun the original command to start fresh"
attach=$FORCE_ATTACH
else
attach=$(find_record)
fi
if [ "$probe" = 1 ]; then
if [ -z "$attach" ]; then echo "UBI_ATTACH_INFO: none key=${RUN_KEY:0:12}"
else
local busy=0
! lease_busy "$(state_dir "$attach")" || busy=1
if [ "$busy" = 0 ] && [ -z "$FORCE_ATTACH" ] && ttl_short "$attach" "$setup"; then
echo "UBI_ATTACH_INFO: none key=${RUN_KEY:0:12} ttl_short vm=$attach $TTL_WHY"
else
echo "UBI_ATTACH_INFO: vm=$attach created_at=$(load "$attach"; echo "$CREATE_AT") key=${RUN_KEY:0:12} busy=$busy"
fi
fi
return 0
fi
if [ ${#TIER_NAMES[@]} -gt 0 ] && { [ -n "$TIER_BLOCKED" ] || [ "$TIER_TO_RUN" -eq 0 ]; } && [ -n "$attach" ] && ! lease_busy "$(state_dir "$attach")"; then
log "verify: no tier needs $attach, which a dead runner left behind; destroying it"
cmd_down "$attach" >/dev/null || log "WARNING: failed to destroy $attach; run: $0 down $attach"
fi
if [ ${#TIER_NAMES[@]} -gt 0 ] && [ -n "$TIER_BLOCKED" ]; then
TIER_FAILED=$TIER_BLOCKED
verify_line
return "$TIER_BLOCKED_RC"
fi
if [ ${#TIER_NAMES[@]} -gt 0 ] && [ "$TIER_TO_RUN" -eq 0 ]; then
log "verify: every tier passed under this key; no VM needed"
verify_line
return 0
fi
if [ -n "$attach" ] && lease_busy "$(state_dir "$attach")"; then
die "UBI_ATTACH_BUSY: vm=$attach key=${RUN_KEY:0:12}: runner pid $(cut -d' ' -f1 <"$(state_dir "$attach")/lease") on this machine still owns it; wait for it or stop it, then rerun"
fi
if [ -n "$attach" ] && [ -z "$FORCE_ATTACH" ] && ttl_short "$attach" "$setup"; then
log "UBI_REATTACH_TTL_SHORT: vm=$attach key=${RUN_KEY:0:12} $TTL_WHY: starting a fresh VM (tier checkpoints carry over) and destroying $attach"
cmd_down "$attach" >/dev/null || log "WARNING: failed to destroy $attach; run: $0 down $attach"
attach=""
fi
[ -z "$attach" ] || name=$attach
[ -n "$name" ] || { name=$(new_name); up_args+=(-n "$name"); }
RUN_VM=$name
# Armed before the create request: an interrupted `up` still destroys its VM.
# A second signal must not cut teardown short, so the EXIT handler ignores them.
trap 'trap "" INT TERM HUP QUIT; cmd_down "$RUN_VM" || log "WARNING: failed to destroy $RUN_VM; run: $0 down $RUN_VM"' EXIT
trap 'exit 130' INT TERM HUP QUIT
cmd_up "${up_args[@]}" >/dev/null
if [[ "$name" =~ -t([0-9]+)$ ]]; then
log "$name carries a ${BASH_REMATCH[1]}h TTL (any later run of this script reaps it); if this process dies first: $0 down $name"
trap 'run_teardown "$?"' EXIT
trap 'exit 130' INT TERM QUIT
trap on_hup HUP
if [ -n "$attach" ]; then
load "$name"
log "reattached to $name created_at=$CREATE_AT key=${RUN_KEY:0:12}"
else
log "$name has no TTL; if this process dies before its teardown line, destroy it with: $0 down $name"
cmd_up "${up_args[@]}" >/dev/null
trap 'exit 130' INT TERM QUIT
trap on_hup HUP
if [[ "$name" =~ -t([0-9]+)$ ]]; then
log "$name carries a ${BASH_REMATCH[1]}h TTL (any later run of this script reaps it); if this process dies first: $0 down $name"
elif [[ "$name" =~ ^$PREFIX-$OWNER-[0-9]{10}- ]]; then
log "$name has no TTL (reap destroys it once it is ${LEGACY_HOURS}h old); if this process dies before its teardown line, destroy it with: $0 down $name"
else
log "$name has no TTL and no reapable name, so nothing reaps it; if this process dies before its teardown line, destroy it with: $0 down $name"
fi
fi
local dir
dir=$(state_dir "$name")
write_record "$dir"
lease_take "$dir"
if [ "$keep" = 1 ]; then
trap - EXIT
# A kept VM's finished run is not reattached by the next run; a detach (HUP) keeps the record.
trap 'rm -f "$(state_dir "$RUN_VM")/run"' EXIT
leftover "left=1 owner=$OWNER vm=$name kept=1"
log "--keep: leaving $name running; destroy with: $0 down $name"
fi
local e
for e in ${envs[@]+"${envs[@]}"}; do printf 'export %s=%q\n' "${e%%=*}" "${e#*=}"; done \
| remote "$name" 'umask 077 && cat > ~/.ubirun-env'
cmd_sync "$name" "$src" "$dest"
local qdest rc=0
local qdest rc=0 vmt
qdest=$(printf %q "$dest")
if [ ! -f "$dir/synced" ]; then
{ for e in ${envs[@]+"${envs[@]}"}; do printf 'export %s=%q\n' "${e%%=*}" "${e#*=}"; done
for n in ${passes[@]+"${passes[@]}"}; do printf 'export %s=%q\n' "$n" "${!n}"; done; } \
| remote "$name" 'umask 077 && cat > ~/.ubirun-env'
cmd_sync "$name" "$src" "$dest"
# A subdirectory of a checkout uploads the whole .git with only its own
# files, so its tree is compared VM to VM (tiers) but not to this side.
if [ -z "$(git -C "$src" rev-parse --show-prefix 2>/dev/null)" ]; then
vmt=$(vm_tree "$name" "$qdest") || vmt=unavailable
if [ "$vmt" != "$TREE" ]; then
log "UBI_VM_TREE_MISMATCH: vm=$name tree=${vmt:0:12} expected=${TREE:0:12}: the unpacked checkout is not the one this run was keyed on"
exit 3
fi
fi
[ -z "$setup" ] || remote "$name" 'cat > ~/.ubirun-setup.sh' <"$setup"
: >"$dir/synced"
fi
if [ -n "$setup" ]; then
remote "$name" 'cat > ~/.ubirun-setup.sh' <"$setup"
log "running setup $(basename "$setup")"
watched "$name" setup "cd $qdest && . ~/.ubirun-env && bash -l ~/.ubirun-setup.sh" || { rc=$?; log "setup failed (exit $rc)"; exit "$rc"; }
detached "$name" setup "cd $qdest && . ~/.ubirun-env && bash -l ~/.ubirun-setup.sh" || { rc=$?; log "setup failed (exit $rc)"; exit "$rc"; }
fi
local start
start=$(date +%s)
log "running on $name: $*"
watched "$name" command "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "$*")" || rc=$?
log "command exited $rc after $(( $(date +%s) - start ))s"
if [ ${#TIER_NAMES[@]} -gt 0 ]; then
run_tiers "$name" "$qdest" || rc=$?
else
log "running on $name: $*"
detached "$name" command "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "$*")" || rc=$?
log "command exited $rc after $(( $(date +%s) - $(cat "$dir/phase.command.start" 2>/dev/null || date +%s) ))s"
fi
local p
for p in ${pulls[@]+"${pulls[@]}"}; do
@@ -786,9 +1385,23 @@ cmd_run() {
fi
cmd_pull "$name" "${p%%:*}" "${p#*:}" || log "pull failed: $p"
done
[ ${#TIER_NAMES[@]} -eq 0 ] || verify_line
return "$rc"
}
# attach NAME: resume the run recorded for NAME on this machine (the same
# invocation, key and tiers), as a rerun of the same command would.
cmd_attach() {
local name=${1:-}
[ -n "$name" ] || die "attach: missing NAME"
valid_name "$name" || die "invalid VM name '$name'"
[ -f "$(state_dir "$name")/run" ] || die "attach: no run record for $name on this machine ($(state_dir "$name")/run); a machine replacement loses it"
# shellcheck disable=SC1091
. "$(state_dir "$name")/run"
FORCE_ATTACH=$name
cmd_run "${RUN_ARGS[@]}"
}
# verify [run opts] [--trust-config] [-- COMMAND]: run the repository's
# committed remote verify. .gstack/ubicloud.json under --src (default .)
# supplies setup, command, size, location, storage, image, env, pass and pull;
@@ -801,7 +1414,7 @@ cmd_run() {
# local variable it lists under pass is forwarded only when this command line
# also names it (--pass NAME) or grants every listed name (--trust-config).
cmd_verify() {
local src=. i j g n cfg cfg_setup cfg_cmd parsed err rc=0 trust=0 cfg_args=() cfg_pass=() user_pre=() user_cmd=() granted=() missing=()
local src=. i j g n cfg cfg_setup cfg_cmd parsed err rc=0 trust=0 cfg_args=() cfg_pass=() user_pre=() user_cmd=() granted=() missing=() cfg_tier_names=() cfg_tier_cmds=()
for (( i = 1; i <= $#; i++ )); do
[ "${!i}" = -- ] && break
if [ "${!i}" = --src ]; then j=$(( i + 1 )); src=${!j}; fi
@@ -810,7 +1423,7 @@ cmd_verify() {
[ -f "$cfg" ] || die "verify: no $cfg; commit one ({\"setup\": \"scripts/ubicloud/bootstrap.sh\", \"command\": \"bash scripts/ubicloud/verify.sh\"}) so every lane runs the same verify instead of its own wrapper"
err=$(mktemp)
parsed=$(python3 -c '
import json, os, shlex, sys
import json, os, re, shlex, sys
cfg, src = sys.argv[1], sys.argv[2]
try:
c = json.load(open(cfg))
@@ -836,7 +1449,25 @@ for item in c.get("pull", []):
if not sep or not remote:
print("pull entry %r must be REMOTE_GLOB:LOCAL_DIR" % (item,), file=sys.stderr); sys.exit(2)
args += ["--pull", q(remote + ":" + inside(local, "pull destination"))]
tiers = c.get("tiers")
names = []
if tiers is not None:
if "command" in c:
print("UBI_CONFIG_TIERS_AND_COMMAND: give either command or tiers, not both", file=sys.stderr); sys.exit(2)
if not isinstance(tiers, list) or not tiers:
print("UBI_CONFIG_TIER_NAME: tiers must be a non-empty list of {\"name\", \"command\"}", file=sys.stderr); sys.exit(2)
for t in tiers:
name = t.get("name") if isinstance(t, dict) else None
if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,31}", name):
print("UBI_CONFIG_TIER_NAME: tier name %r must match ^[a-z0-9][a-z0-9_-]{0,31}$" % (name,), file=sys.stderr); sys.exit(2)
if name in names:
print("UBI_CONFIG_TIER_NAME: tier name %r appears twice" % name, file=sys.stderr); sys.exit(2)
if not isinstance(t.get("command"), str) or not t["command"].strip():
print("UBI_CONFIG_TIER_NAME: tier %r needs a command" % name, file=sys.stderr); sys.exit(2)
names.append(name)
setup = c.get("setup", "")
print("cfg_tier_names=(" + " ".join(q(n) for n in names) + ")")
print("cfg_tier_cmds=(" + " ".join(q(t["command"]) for t in (tiers or [])) + ")")
print("cfg_args=(" + " ".join(args) + ")")
print("cfg_pass=(" + " ".join(q(x) for x in c.get("pass", [])) + ")")
print("cfg_setup=" + q(inside(setup, "setup") if setup else ""))
@@ -853,7 +1484,7 @@ print("cfg_cmd=" + q(c.get("command", "")))
case $1 in
--) shift; user_cmd=("$@"); break ;;
--trust-config) trust=1; shift ;;
--keep) user_pre+=("$1"); shift ;;
--keep|--resume|--no-resume|--rerun-failed|--probe) user_pre+=("$1"); shift ;;
--pass|-S|--storage|-s|--size|-l|--location|-n|--name|-b|--image|--src|--dest|--setup|--env|--pull)
[ $# -ge 2 ] || die "verify: $1 needs a value"
[ "$1" != --pass ] || granted+=("$2")
@@ -868,11 +1499,16 @@ print("cfg_cmd=" + q(c.get("command", "")))
missing+=("$n")
done
[ ${#missing[@]} -eq 0 ] || die "verify: $cfg asks to forward local ${missing[*]}; grant each on this command line with --pass NAME, or every name it lists with --trust-config"
[ ${#user_cmd[@]} -gt 0 ] || [ -n "$cfg_cmd" ] || die "verify: $cfg names no command and none was given after --"
[ ${#user_cmd[@]} -gt 0 ] || user_cmd=("$cfg_cmd")
# A trailing -- COMMAND replaces the configured tiers as it replaces command.
if [ ${#user_cmd[@]} -eq 0 ] && [ ${#cfg_tier_names[@]} -gt 0 ]; then
TIER_NAMES=("${cfg_tier_names[@]}") TIER_CMDS=("${cfg_tier_cmds[@]}")
else
[ ${#user_cmd[@]} -gt 0 ] || [ -n "$cfg_cmd" ] || die "verify: $cfg names no command and none was given after --"
[ ${#user_cmd[@]} -gt 0 ] || user_cmd=("$cfg_cmd")
fi
log "verify: $cfg"
PULL_ROOT=$(cd "$src" && pwd)
cmd_run ${cfg_args[@]+"${cfg_args[@]}"} ${cfg_setup:+--setup "$cfg_setup"} --src "$src" ${user_pre[@]+"${user_pre[@]}"} -- "${user_cmd[@]}"
cmd_run ${cfg_args[@]+"${cfg_args[@]}"} ${cfg_setup:+--setup "$cfg_setup"} --src "$src" ${user_pre[@]+"${user_pre[@]}"} -- ${user_cmd[@]+"${user_cmd[@]}"}
}
usage() {
@@ -880,13 +1516,17 @@ usage() {
usage: ubi-runner.sh <command> [args]
run [up opts] [--src DIR] [--dest PATH] [--setup FILE] [--env K=V]...
[--pass NAME]... [--pull REMOTE_GLOB:LOCAL_DIR]... [--keep] -- COMMAND
up + sync + setup + command + pull + destroy; exits with COMMAND's status
verify [run opts] [--trust-config] [-- COMMAND]
[--pass NAME]... [--pull REMOTE_GLOB:LOCAL_DIR]... [--keep] [--key-input S]... [--probe] -- COMMAND
up + sync + setup + command + pull + destroy; exits with COMMAND's status.
A rerun of the same command (same run key) reattaches to its live VM;
--probe prints UBI_ATTACH_INFO (and the tier plan) and exits
verify [run opts] [--trust-config] [--resume|--no-resume] [--rerun-failed] [-- COMMAND]
run the repository's committed verify (.gstack/ubicloud.json under --src);
its setup script and every pull destination stay inside --src (checked
again at pull time), and its pass names are forwarded only when granted
here (--pass NAME, or all: --trust-config)
here (--pass NAME, or all: --trust-config). With tiers, a pass checkpoint
under the same run key is reused (resume is the default)
attach NAME resume the run recorded for NAME on this machine, as a rerun would
up [-s SIZE] [-l LOCATION] [-n NAME] [-S STORAGE_GIB] [-b IMAGE]
create a VM and wait for SSH; prints its name
(STORAGE_GIB: one of $ALLOWED_STORAGE)
@@ -902,20 +1542,28 @@ usage: ubi-runner.sh <command> [args]
owner print this caller's owner tag ($OWNER)
records VMs this machine created and has not confirmed gone ($VM_RECORD)
gc HOURS destroy this owner's VMs older than HOURS (never other owners')
reap destroy every VM whose -t<hours> TTL has ended (any owner; TTL-less names never)
reap destroy every VM whose -t<hours> TTL has ended (any owner) and this
owner's TTL-less ubirun-* VMs older than UBI_LEGACY_HOURS
cli ARGS... raw Ubicloud CLI passthrough (e.g. cli vm list)
VMs are named $PREFIX-<owner>-<epoch>-<suffix>-t<ttl hours>; set UBI_OWNER (e.g. your
thread code) to tag them, otherwise a per-machine id is used.
signals: INT and TERM tear the VM down. HUP, once a remote phase is live, detaches
(UBI_DETACHED, exit 129; the VM runs on under its TTL). SIGKILL or a machine stop leaves
the run record and the VM; rerun the same command to reattach (UBI_ATTACH_BUSY while
another runner owns it; exit 125 UBI_ATTACH_LOST when the remote session left no status).
VMs are named $PREFIX-<owner>-<epoch>-<suffix>-t<ttl hours>; up, run and verify need
UBI_OWNER (e.g. your thread code); run and verify end with a UBICLOUD: left=<n> line.
defaults: size=$DEFAULT_SIZE location=$DEFAULT_LOCATION (env UBI_SIZE, UBI_LOCATION)
env: UBI_TTL_HOURS (${TTL_HOURS}h in the name; 0 omits it), UBI_REAP (0: up skips the reap),
env: UBI_TTL_HOURS (${TTL_HOURS}h in the name, 1-999), UBI_REAP (0: up skips the reap),
UBI_LEGACY_HOURS (${LEGACY_HOURS}h: reap's bound for this owner's TTL-less ubirun-* names),
UBI_IDLE_TIMEOUT (${IDLE_TIMEOUT}s of silence kills the remote command, exit 124; 0 disables),
UBI_RUN_TIMEOUT (${RUN_TIMEOUT}s wall clock; 0 unlimited), UBI_HEARTBEAT (${HEARTBEAT}s between heartbeat lines),
UBI_QUOTA_VCPUS (project vCPU ceiling for the preflight; else the last refusal's maximum),
UBI_QUOTA_WAIT (${QUOTA_WAIT}s to wait for quota room instead of refusing),
UBI_GC_HOURS (unset/0: no sweep on up), UBI_CREATE_GRACE (${CREATE_GRACE}s),
UBI_DOWN_TIMEOUT (${DOWN_TIMEOUT}s), UBI_API_TIMEOUT (${API_TIMEOUT}s), UBI_VM_RECORD (scratch VM-id file),
UBI_ALLOWED_STORAGE (the -S sizes Ubicloud accepts; default "320 640")
UBI_ALLOWED_STORAGE (the -S sizes Ubicloud accepts; default "320 640"),
UBI_CHECKPOINT_DIR (tier checkpoints; $CHECKPOINT_DIR), UBI_CHECKPOINT_MAX_AGE (${CHECKPOINT_MAX_AGE}s a pass is reused)
EOF
}
@@ -928,6 +1576,7 @@ esac
case $cmd in
run) cmd_run "$@" ;;
verify) cmd_verify "$@" ;;
attach) cmd_attach "$@" ;;
up) trap 'trap "" INT TERM HUP QUIT; [ -z "$UP_NAME" ] || [ -n "$UP_READY" ] || cmd_down "$UP_NAME" || log "WARNING: failed to destroy $UP_NAME; run: $0 down $UP_NAME"' EXIT
trap 'exit 130' INT TERM HUP QUIT
cmd_up "$@" ;;