Merge commit 'dd9834d44d2650818cb6d8e17c57f795cb56bf98' into gsta161/opus-lite-loadout

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

# Conflicts:
#	scripts/resolvers/preamble/generate-preamble-bash.ts
This commit is contained in:
garrytan committed 2026-11-02 05:58:00 +00:00
commit caaa1419b0
571 files changed
+38331 -8101

No files matched your search

+157
View File
@@ -0,0 +1,157 @@
#!/usr/bin/env -S bun --no-env-file --config=/dev/null
/**
* ci-tree-reuse — run each free suite once per tree (GSTA-156, docs/pregate.md#ci-reuse).
*
* bun run scripts/ci-tree-reuse.ts record --plan <plan.json> --results <dir> --os <OS>[,<OS>…] --out <tree.json>
* The producer. A Free Tests or Windows Free Tests aggregate runs it only after its strict verification
* passed, and uploads the file as the `tested-tree` artifact: run_id, run_attempt, event, sha (GITHUB_SHA),
* head_sha (the pull request head for pull_request, else GITHUB_SHA), tree (git's tree of GITHUB_SHA),
* workflow, os (the OSes the run's test jobs ran on), files (every test file in the plan) and tests.
*
* bun run scripts/ci-tree-reuse.ts find --workflow <file.yml> --profile <run_profiles name>
* The main-push consumer (the `reuse` job). A push to main whose tree a pull request already tested green
* needs no second run: it finds the pull request whose merge commit is the pushed commit (same-repository
* head, merged into main), requires that head's tree to equal the pushed tree, lists that head's runs of
* this workflow (pull_request or workflow_dispatch, newest first, at most MAX_CANDIDATES), and accepts the
* first one lib/run-evidence.ts verifyRun accepts under the profile in the pushed tree's
* .gstack/pregate.json. A pull_request run counts only when its merge commit's tree is its head's tree
* (verifyRun). It writes reused=true, run_id, run_url and tree to GITHUB_OUTPUT, or reused=false and a
* reason, with one REUSE: line in the step summary. Every lookup failure is a refusal, never a pass: the
* workflow then runs every job as before. Pull request and dispatch runs never consume.
*/
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { ghApiJson, ghRunDeps, runProfileErrors, verifyRun, type ApiRun, type RunDeps, type RunProfile } from '../lib/run-evidence';
export const MAX_PULLS = 10;
export const MAX_CANDIDATES = 10;
const SHA = /^[0-9a-f]{40}$/;
const OSES = ['Linux', 'macOS', 'Windows'];
export interface ApiPull { number: number; merged_at: string | null; merge_commit_sha: string | null; base: { ref: string; repo?: { full_name: string } | null }; head: { sha: string; repo: { full_name: string } | null } }
export interface ReuseDeps extends RunDeps { pullsForCommit(sha: string): ApiPull[] }
export type ReuseVerdict =
| { reused: true; run_id: number; run_attempt: number; event: string; pr: number; head_sha: string; tree: string }
| { reused: false; reason: string; detail: string };
const refuse = (reason: string, detail: string): ReuseVerdict => ({ reused: false, reason, detail });
const startedAt = (r: ApiRun) => r.run_started_at ?? r.created_at ?? '';
/** The run a push of `pushSha` to `base` may reuse, or why none. Pure over `deps`, so every refusal has a fixture. */
export function findReuse(deps: ReuseDeps, o: { repo: string; base: string; pushSha: string; workflow: string; profileName: string; profile: RunProfile; now: Date }): ReuseVerdict {
if (!SHA.test(o.pushSha)) return refuse('usage', `${JSON.stringify(o.pushSha)} is not a commit sha`);
if (o.profile.workflow !== o.workflow) return refuse('profile_mismatch', `profile ${o.profileName} names ${o.profile.workflow}, not ${o.workflow}`);
try {
const pulls = deps.pullsForCommit(o.pushSha).slice(0, MAX_PULLS);
const merged = pulls.filter(p => p.merged_at && p.base.ref === o.base && p.merge_commit_sha === o.pushSha && (!p.base.repo || p.base.repo.full_name.toLowerCase() === o.repo.toLowerCase()));
if (!merged.length) return refuse('no_pr', `no pull request merged into ${o.base} has ${o.pushSha.slice(0, 12)} as its merge commit (${pulls.length} associated)`);
if (merged.length > 1) return refuse('ambiguous_pr', `${merged.length} merged pull requests name ${o.pushSha.slice(0, 12)}: ${merged.map(p => `#${p.number}`).join(', ')}`);
const pr = merged[0]!;
if ((pr.head.repo?.full_name ?? '').toLowerCase() !== o.repo.toLowerCase()) return refuse('fork_head', `#${pr.number}'s head is on ${pr.head.repo?.full_name ?? 'a deleted repository'}, not ${o.repo}`);
if (!SHA.test(pr.head.sha)) return refuse('api_error', `#${pr.number} has no head commit`);
const pushTree = deps.commitTree(o.pushSha);
const headTree = deps.commitTree(pr.head.sha);
if (!pushTree || !headTree) return refuse('api_error', `cannot read the tree of ${pushTree ? pr.head.sha : o.pushSha}`);
if (headTree !== pushTree) return refuse('tree_mismatch', `#${pr.number}'s head ${pr.head.sha.slice(0, 12)} has tree ${headTree.slice(0, 12)}, the push has ${pushTree.slice(0, 12)}`);
const events: string[] = o.profile.events.filter(e => e === 'pull_request' || e === 'workflow_dispatch');
const candidates = deps.runsForHead(o.workflow, pr.head.sha).filter(r => events.includes(r.event))
.sort((a, b) => startedAt(b).localeCompare(startedAt(a)) || b.id - a.id).slice(0, MAX_CANDIDATES);
if (!candidates.length) return refuse('no_candidate', `#${pr.number}'s head ${pr.head.sha.slice(0, 12)} has no ${events.join(' or ') || 'pull_request or workflow_dispatch'} run of ${o.workflow}`);
const refusals: string[] = [];
for (const run of candidates) {
const v = verifyRun(deps, { repo: o.repo, runRef: String(run.id), profileName: o.profileName, profile: o.profile, now: o.now });
if (!v.ok) { refusals.push(`run ${run.id} (${run.event}) reason=${v.reason} ${v.detail}`); continue; }
if (v.evidence.tree !== pushTree) { refusals.push(`run ${run.id} tested tree ${v.evidence.tree.slice(0, 12)}`); continue; }
return { reused: true, run_id: v.evidence.run_id, run_attempt: v.evidence.run_attempt, event: v.evidence.event, pr: pr.number, head_sha: pr.head.sha, tree: pushTree };
}
return refuse('no_verified_run', refusals.join(' | '));
} catch (e: any) {
return refuse('api_error', String(e?.message ?? e).slice(0, 300));
}
}
/** tree.json for the `tested-tree` artifact; throws on any missing or malformed input, so a broken record fails the aggregate instead of uploading. */
export function treeRecord(env: NodeJS.ProcessEnv, o: { plan: { shards: Array<{ files: string[] }> }; results: Array<{ outcome?: { summary?: { testsRan?: number } } }>; os: string[]; tree: string }) {
const int = (name: string) => { const v = Number(env[name]); if (!Number.isSafeInteger(v) || v < 1) throw new Error(`${name} is not a positive integer`); return v; };
const sha = env.GITHUB_SHA ?? '';
const event = env.GITHUB_EVENT_NAME ?? '';
const head = event === 'pull_request' ? env.PR_HEAD_SHA ?? '' : sha;
if (!SHA.test(sha) || !SHA.test(head)) throw new Error(`GITHUB_SHA ${JSON.stringify(sha)} or the head ${JSON.stringify(head)} is not a commit sha (PR_HEAD_SHA is required for pull_request)`);
if (!/^[a-z_]+$/.test(event)) throw new Error(`GITHUB_EVENT_NAME ${JSON.stringify(event)} is not an event name`);
const workflow = /^[^/]+\/[^/]+\/(\.github\/workflows\/[A-Za-z0-9_.-]+\.ya?ml)@/.exec(env.GITHUB_WORKFLOW_REF ?? '')?.[1];
if (!workflow) throw new Error(`GITHUB_WORKFLOW_REF ${JSON.stringify(env.GITHUB_WORKFLOW_REF)} names no workflow file`);
if (!/^[0-9a-f]{40,64}$/.test(o.tree)) throw new Error(`${JSON.stringify(o.tree)} is not a tree id`);
if (!o.os.length || o.os.some(x => !OSES.includes(x))) throw new Error(`--os must list some of ${OSES.join(', ')}`);
const files = [...new Set(o.plan.shards.flatMap(s => s.files))].sort();
if (!files.length) throw new Error('the plan lists no test files');
const counts = o.results.map(r => r.outcome?.summary?.testsRan);
const tests = o.results.length && counts.every(n => Number.isSafeInteger(n) && n! >= 0) ? counts.reduce((a, b) => a! + b!, 0)! : files.length;
return { run_id: int('GITHUB_RUN_ID'), run_attempt: int('GITHUB_RUN_ATTEMPT'), event, sha, head_sha: head, tree: o.tree, workflow, os: o.os, files, tests };
}
function flag(args: string[], name: string): string | undefined { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; }
function need(args: string[], name: string): string { const v = flag(args, name); if (!v || v.startsWith('--')) throw new Error(`${name} <value> is required`); return v; }
function record(args: string[]): number {
const plan = JSON.parse(fs.readFileSync(need(args, '--plan'), 'utf8'));
const dir = need(args, '--results');
const results = fs.readdirSync(dir).filter(f => f.endsWith('.json')).map(f => JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')));
const git = spawnSync('git', ['rev-parse', '--verify', `${process.env.GITHUB_SHA ?? ''}^{tree}`], { encoding: 'utf8', timeout: 30_000 });
const rec = treeRecord(process.env, { plan, results, os: need(args, '--os').split(','), tree: git.status === 0 ? git.stdout.trim() : '' });
const out = need(args, '--out');
fs.mkdirSync(path.dirname(path.resolve(out)), { recursive: true });
fs.writeFileSync(out, JSON.stringify(rec, null, 2) + '\n');
console.log(`TESTED_TREE: run ${rec.run_id}/${rec.run_attempt} ${rec.event} head ${rec.head_sha.slice(0, 12)} tree ${rec.tree.slice(0, 12)} files=${rec.files.length} tests=${rec.tests} os=${rec.os.join(',')}`);
return 0;
}
/** The consumer's GITHUB_OUTPUT and step-summary text for a verdict. reused=true is written last, after the run it names. */
export function reuseOutputs(v: ReuseVerdict, repo: string, server = 'https://github.com'): { line: string; output: string } {
if (!v.reused) return { line: `REUSE: refused reason=${v.reason} — ${v.detail}; every job runs`, output: `reused=false\nreason=${v.reason}\n` };
const url = `${server}/${repo}/actions/runs/${v.run_id}`;
return { line: `REUSE: accepted run ${v.run_id} (${v.event}, attempt ${v.run_attempt}) of #${v.pr} at ${v.head_sha.slice(0, 12)}, tree ${v.tree.slice(0, 12)}: ${url}`, output: `run_id=${v.run_id}\nrun_url=${url}\ntree=${v.tree}\nreused=true\n` };
}
function find(args: string[]): number {
const env = process.env;
const workflow = need(args, '--workflow');
const profileName = need(args, '--profile');
const repo = env.GITHUB_REPOSITORY ?? '';
const settle = (v: ReuseVerdict) => {
const { line, output } = reuseOutputs(v, repo, env.GITHUB_SERVER_URL);
console.log(line);
if (env.GITHUB_STEP_SUMMARY) fs.appendFileSync(env.GITHUB_STEP_SUMMARY, `${line}\n`);
if (env.GITHUB_OUTPUT) fs.appendFileSync(env.GITHUB_OUTPUT, output);
return 0;
};
if (env.GITHUB_EVENT_NAME !== 'push' || env.GITHUB_REF !== 'refs/heads/main') return settle(refuse('not_main_push', `${env.GITHUB_EVENT_NAME ?? 'no'} event on ${env.GITHUB_REF ?? 'no ref'}; only a push to main consumes`));
if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) return settle(refuse('usage', `GITHUB_REPOSITORY ${JSON.stringify(repo)} is not owner/name`));
let profiles: Record<string, RunProfile>;
try {
const config = JSON.parse(fs.readFileSync('.gstack/pregate.json', 'utf8'));
const errors = runProfileErrors(config.run_profiles, undefined);
if (errors.length) return settle(refuse('profile_invalid', errors.map(e => `${e.path}: ${e.message}`).join('; ')));
profiles = config.run_profiles ?? {};
} catch (e: any) { return settle(refuse('profile_invalid', `.gstack/pregate.json: ${String(e?.message ?? e).slice(0, 200)}`)); }
const profile = profiles[profileName];
if (!profile) return settle(refuse('profile_invalid', `.gstack/pregate.json declares no run profile ${profileName}`));
const deps: ReuseDeps = {
...ghRunDeps(repo, process.cwd()),
commitTree: sha => (SHA.test(sha) ? ghApiJson<{ tree?: { sha?: string } }>(`repos/${repo}/git/commits/${sha}`).tree?.sha ?? null : null),
pullsForCommit: sha => ghApiJson<ApiPull[]>(`repos/${repo}/commits/${sha}/pulls?per_page=${MAX_PULLS}`),
};
return settle(findReuse(deps, { repo, base: 'main', pushSha: env.GITHUB_SHA ?? '', workflow, profileName, profile, now: new Date() }));
}
if (import.meta.main) {
const [verb, ...args] = process.argv.slice(2);
try {
if (verb === 'record') process.exitCode = record(args);
else if (verb === 'find') process.exitCode = find(args);
else { console.error('usage: ci-tree-reuse.ts record --plan <f> --results <dir> --os <OS,…> --out <f> | find --workflow <f.yml> --profile <name>'); process.exitCode = 2; }
} catch (e: any) {
console.error(`ci-tree-reuse ${verb}: ${e?.message ?? e}`);
process.exitCode = 1;
}
}
+5 -2
View File
@@ -10,6 +10,8 @@ export type CsoRuntimeBuildStack = typeof CSO_RUNTIME_STACKS[number];
const IMAGE = /^[a-z0-9.-]+(?::[0-9]+)?\/[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}$/;
const TAGGED_IMAGE = /^[a-z0-9.-]+(?::[0-9]+)?\/[a-z0-9][a-z0-9._/-]*:[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
const RELEASE_TAG = /:v?\d+\.\d+\.\d+(?:[._-][A-Za-z0-9._-]*)?$/;
const TWO_PART_RELEASE_TAG = /:v?\d+\.\d+(?:\.\d+)?(?:[._-][A-Za-z0-9._-]*)?$/;
const REVISION = /^[a-z0-9][a-z0-9._-]{0,100}$/;
const REQUIRED_VERSIONS: Record<CsoRuntimeBuildStack, string[]> = {
node: ['node', 'npm', 'cso-preparation'],
@@ -56,8 +58,9 @@ function repositoryOfTag(image: string): string {
return image.slice(0, colon);
}
function assertPinnedFamily(source: unknown, indexImage: unknown, images: unknown, label: string): Record<CsoRuntimeBuildPlatform, string> {
function assertPinnedFamily(source: unknown, indexImage: unknown, images: unknown, label: string, releaseTag = RELEASE_TAG): Record<CsoRuntimeBuildPlatform, string> {
if (typeof source !== 'string' || !TAGGED_IMAGE.test(source)) throw new Error(`INVALID_${label}_SOURCE`);
if (!releaseTag.test(source)) throw new Error(`FLOATING_${label}_SOURCE: ${source} must name an exact release version, not a moving major, minor or alias tag`);
if (typeof indexImage !== 'string' || !IMAGE.test(indexImage)) throw new Error(`UNPINNED_${label}_INDEX`);
if (!record(images)) throw new Error(`MISSING_${label}_IMAGES`);
const repository = repositoryOfTag(source);
@@ -101,7 +104,7 @@ export function imageBuildMatrix(input: unknown): { include: ImageBuildRow[] } {
if (typeof stack !== 'string' || !CSO_RUNTIME_STACKS.includes(stack as CsoRuntimeBuildStack) || seenStacks.has(stack)) throw new Error('INVALID_STACK');
seenStacks.add(stack);
const typedStack = stack as CsoRuntimeBuildStack;
const bases = assertPinnedFamily(source, indexImage, baseImages, 'BASE');
const bases = assertPinnedFamily(source, indexImage, baseImages, 'BASE', typedStack === 'postgresql' ? TWO_PART_RELEASE_TAG : RELEASE_TAG);
if (!record(versions)) throw new Error('MISSING_TOOL_VERSIONS');
const exactVersion = (value: unknown) => typeof value === 'string' &&
(typedStack === 'postgresql'
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+4 -4
View File
@@ -17,7 +17,7 @@
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
"scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"},
"scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
"scripts/test-pr-profile.ts": {"blob":"911e393d0c36c050c3a68db5a81ef21701bfd5ae","since":6,"nonBehavioral":"selection only: fragment exemption narrowed to .md files directly in the fragments dir; changed-file lists include a rename's old path; no case prompt, fixture, judge or verdict logic changes (GST-400)"},
"scripts/test-pr-profile.ts": {"blob":"44ae88bebf8f716f70a3bd836c8bedb03e6653e8","since":6,"nonBehavioral":"FREE_ONLY_PR_FILES gains scripts/lib/shard-jobs.ts (free-runner module) and .github/workflows/windows-smoke.yml (a workflow no paid case reads); both change only which PR paths select paid cases, never how a case runs or is judged"},
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6},
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6},
"test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6},
@@ -29,7 +29,7 @@
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6},
"test/helpers/eval-store.ts": {"blob":"a50ea2c28f508ad9f35ea9353dd2f502ee0dcfc2","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
"test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6},
"test/helpers/llm-judge.ts": {"blob":"095f9c85dd0ae29da8ba10fed91510313a4442a4","since":6,"nonBehavioral":"callJudge emits one llm-judge-aborted diagnostic line (wrapper attempts, HTTP request and status counts seen through a pass-through fetch, 429 waits, connect and event timings, block kind, character counts, message and request ids, a capped reason; no content) when its caller's signal aborts it; observers swallow their own errors, and the request, retries, parsing, scores and the thrown abort reason are unchanged, so no case's pass or fail changes"},
"test/helpers/llm-judge.ts": {"blob":"f14ca0db6504480698acd662351b6f0d6ef31a7b","since":6,"nonBehavioral":"llm-judge.ts adds a pass-through byte counter (bytesReceived, lastByteMs) to the aborted-judge diagnostic; the request, response bytes, deadline and scoring are unchanged"},
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6},
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6},
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6},
@@ -41,9 +41,9 @@
"test/helpers/pty/launch.ts": {"blob":"e4413de380b799659170e7da248d51afda3b0e3f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6},
"test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6},
"test/helpers/pty/runners/floor.ts": {"blob":"54e2145832bb98913644d10ff303792d67f9362e","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."},
"test/helpers/pty/runners/floor.ts": {"blob":"a9938cb98f285474ebc4860cd61012504a36466e","since":6,"nonBehavioral":"The PTY floor and observation runners end a plan-mode turn at an idle prompt (the plan-mode footer without its running 'esc to interrupt' segment, after a 10 s settle) instead of waiting out the deadline. The floor runner exits as api_error only after the idle screen with its error panel held still for 10 s and every tick's question and assessment steps found nothing to credit (the transcript is re-read each tick), so a native or prose question already on screen is credited as before, even when its transcript line lands after the screen draws done (test/helpers/claude-pty-runner.runners.unit.test.ts, the prose-question-before-panel case). With no qualifying question an idle turn stays idle, so a trial that failed at the deadline as observer_timeout now fails earlier as api_error. An observation returns what the deadline would have returned. A running footer, extra footer text and a cut footer still wait for the deadline. No pass becomes a fail or a fail a pass. The session ledger may now record failure_cause api_error or refusal instead of observer_timeout or session_timeout for a plan-mode session whose last idle turn shows a panel; failure_class, the gate red class and pass rates are unchanged, so no pass-rate series resets"},
"test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6},
"test/helpers/pty/screen.ts": {"blob":"f3743152fc9bb7a8e8c70002f75bf45914f260d5","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
"test/helpers/pty/screen.ts": {"blob":"1384f765e48e511d5f843df2660b2ec70bbcb6b2","since":6,"nonBehavioral":"The PTY floor and observation runners end a plan-mode turn at an idle prompt (the plan-mode footer without its running 'esc to interrupt' segment, after a 10 s settle) instead of waiting out the deadline. The floor runner exits as api_error only after the idle screen with its error panel held still for 10 s and every tick's question and assessment steps found nothing to credit (the transcript is re-read each tick), so a native or prose question already on screen is credited as before, even when its transcript line lands after the screen draws done (test/helpers/claude-pty-runner.runners.unit.test.ts, the prose-question-before-panel case). With no qualifying question an idle turn stays idle, so a trial that failed at the deadline as observer_timeout now fails earlier as api_error. An observation returns what the deadline would have returned. A running footer, extra footer text and a cut footer still wait for the deadline. No pass becomes a fail or a fail a pass. The session ledger may now record failure_cause api_error or refusal instead of observer_timeout or session_timeout for a plan-mode session whose last idle turn shows a panel; failure_class, the gate red class and pass rates are unchanged, so no pass-rate series resets"},
"test/helpers/pty/session.ts": {"blob":"54fe04b61d953e4e55e2fc251532c04a49fab66f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6},
"test/helpers/session-ledger.ts": {"blob":"a10bf28bf100f135b93c73818670dd147253cdd3","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
+10 -3
View File
@@ -322,9 +322,16 @@ function flag(args: string[], name: string): string | undefined {
return index >= 0 ? args[index + 1] : undefined;
}
async function main(args: string[]): Promise<void> {
/** The W2 per-judge table plus the arm benchmark's paired judge (run by --config only; the manifest prices the W2 table). */
export async function loadCalibrationConfigs(): Promise<{ w2: CalibrationConfig[]; all: CalibrationConfig[] }> {
// The per-judge table (builders, schemas, verdict rules) is committed separately from this harness.
const { CALIBRATION_CONFIGS } = await import(path.join(import.meta.dir, 'judge-calibration-configs.ts')) as { CALIBRATION_CONFIGS: CalibrationConfig[] };
const { ARM_PAIR_CALIBRATION } = await import(path.join(import.meta.dir, '..', 'test', 'helpers', 'arm-pair-judge.ts')) as { ARM_PAIR_CALIBRATION: CalibrationConfig };
return { w2: CALIBRATION_CONFIGS, all: [...CALIBRATION_CONFIGS, ARM_PAIR_CALIBRATION] };
}
async function main(args: string[]): Promise<void> {
const { w2: CALIBRATION_CONFIGS, all: configs } = await loadCalibrationConfigs();
const command = args[0];
if (command === 'manifest') {
const client = new Anthropic();
@@ -338,8 +345,8 @@ async function main(args: string[]): Promise<void> {
console.table(manifest.entries);
return;
}
const config = CALIBRATION_CONFIGS.find(candidate => candidate.id === flag(args, 'config'));
if (!config) throw new Error(`unknown --config; expected one of ${CALIBRATION_CONFIGS.map(candidate => candidate.id).join(', ')}`);
const config = configs.find(candidate => candidate.id === flag(args, 'config'));
if (!config) throw new Error(`unknown --config; expected one of ${configs.map(candidate => candidate.id).join(', ')}`);
const items = loadCorpus(config.id);
const runsDir = path.join(CORPUS_ROOT, config.id, 'runs');
const samplesPath = path.join(runsDir, 'comparison-1.samples.jsonl');
+48 -7
View File
@@ -259,7 +259,38 @@ export function readFlakeLedgerDir(dir: string): FlakeLedgerEntry[] {
export const FLAKE_POLICY = { maxRate: 0.05, minRuns: 20 } as const;
export interface FlakyFile { file: string; flakyRuns: number; entries: number; rate: number; enforced: boolean; failing: boolean }
export interface FlakeWindow { runs: number; flakyRuns: number; since: string; files: FlakyFile[]; enforceable: boolean }
/** `skipped`: reused main pushes left out of the window (reuseChecker); `note`: runs that could not be checked. */
export interface FlakeWindow { runs: number; flakyRuns: number; since: string; files: FlakyFile[]; enforceable: boolean; skipped?: number; note?: string }
/** The most job listings one reuseChecker spends. */
export const REUSE_LOOKUPS_MAX = 40;
/**
* Finds runs that reused their merged pull request's verified run (#3709): such
* a run uploads no artifact and its `reuse` job succeeds, because every suite
* job was skipped. It holds no flake evidence, so flake counts leave it out of
* their runs and their denominator (#3725). A run with any artifact costs no
* call. A run with none costs one job listing, at most REUSE_LOOKUPS_MAX per
* checker; past the cap, or when the listing fails, the run counts as it did
* before #3725 and note() says so.
*/
export function reuseChecker(client: GhClient, repo: string) {
let lookups = 0;
let unchecked = 0;
let reason = '';
return {
reused(run: Pick<CiRun, 'id' | 'attempt'>, artifactNames: string[]): boolean {
if (artifactNames.length) return false;
if (lookups >= REUSE_LOOKUPS_MAX) { unchecked++; reason ||= `cap of ${REUSE_LOOKUPS_MAX} job listings reached`; return false; }
lookups++;
try {
return listRunJobs(client, repo, run.id, run.attempt).some(job => job.name === 'reuse' && job.conclusion === 'success');
} catch (error) { unchecked++; reason ||= unavailableReason(error); return false; }
},
note: (): string | undefined => (unchecked
? `${unchecked} run(s) with no artifacts not checked for main-push reuse (${reason}); counted as before #3725` : undefined),
};
}
/**
* Rank files by the share of runs in which they needed the retry pass. A file
@@ -286,9 +317,10 @@ export function rankFlakyFiles(runEntries: Map<number, FlakeLedgerEntry[]>, runC
/**
* Read the flake ledgers of main's completed free-tests runs: the runs created
* in the last `sinceDays`, widened to the newest `minRuns` runs (within
* `maxLookbackDays`) so the 5% rule has a window of at least 20 runs. A run
* with no `flake-ledger-*` artifact had no flaky passes (the upload is skipped
* when the ledger file does not exist).
* `maxLookbackDays`) so the 5% rule has a window of at least 20 runs. Reused
* main pushes (reuseChecker) are left out and older runs take their place. A
* run with no `flake-ledger-*` artifact had no flaky passes (the upload is
* skipped when the ledger file does not exist).
*/
export function readCiFlakeLedgers(opts: { client: GhClient; repo: string; sinceDays: number; now: number; cacheDir: string;
minRuns?: number; maxLookbackDays?: number; workflow?: string }): FlakeWindow {
@@ -297,16 +329,25 @@ export function readCiFlakeLedgers(opts: { client: GhClient; repo: string; since
const all = listWorkflowRuns(opts.client, opts.repo, opts.workflow ?? 'free-tests.yml',
{ since: iso(Math.max(opts.sinceDays, opts.maxLookbackDays ?? 60)), branch: 'main' })
.filter(run => run.conclusion === 'success' || run.conclusion === 'failure');
const recent = all.filter(run => run.createdAt >= iso(opts.sinceDays));
const window = recent.length >= minRuns ? recent : all.slice(0, minRuns);
const since = iso(opts.sinceDays);
const recent = all.filter(run => run.createdAt >= since).length;
const fetcher = historyFetcher(opts.client);
const reuse = reuseChecker(opts.client, opts.repo);
const window: CiRun[] = [];
let skipped = 0;
for (const [index, run] of all.entries()) {
if (index >= recent && window.length >= minRuns) break;
if (reuse.reused(run, fetcher.listArtifacts(opts.repo, run.id).map(artifact => artifact.name))) skipped++;
else window.push(run);
}
const runEntries = new Map<number, FlakeLedgerEntry[]>();
for (const run of window) {
const dirs = downloadRunArtifacts({ repo: opts.repo, run, cacheDir: opts.cacheDir, fetcher, match: name => name.startsWith('flake-ledger-') });
const entries = dirs.flatMap(readFlakeLedgerDir);
if (entries.length) runEntries.set(run.id, entries);
}
return rankFlakyFiles(runEntries, window.length, window[window.length - 1]?.createdAt ?? iso(opts.sinceDays));
const note = reuse.note();
return { ...rankFlakyFiles(runEntries, window.length, window[window.length - 1]?.createdAt ?? since), skipped, ...(note ? { note } : {}) };
}
/** Markdown table of one run's merged flake ledgers, for a job summary. */
+422
View File
@@ -0,0 +1,422 @@
/**
* Sub-shards inside one CI job: `--ci-run <plan> --shard i --result r --jobs k`
* (GSTA-203 PR 2, principle "buy cores, not shards").
*
* A CI shard's files split into k disjoint, identity-bound sub-shards that run
* side by side through the free runner's own process-shard path (one bun child
* each, serial inside, own sandbox, group kill, log), never Bun's worker
* `--parallel` (decision V3/D6 in scripts/test-free-shards.ts). The rules:
*
* - k = 1 (the default) is today's single child, unchanged.
* - Files in TREE_MUTATING or SLOW_UNDER_LOAD run after the parallel
* sub-shards settle, serially, in one trailing sub-shard.
* - The parallel phase uses the concurrent-phase home guard; every child
* uses the shared guard inside it, exactly as full-suite mode does.
* - The whole CI shard keeps its k = 1 wall: the parallel sub-shards get it,
* and the serial sub-shard gets what is left of it, so k never extends a
* deadline; every launched child settles before the parent goes on.
* - The parent merges every sub-shard result before the single retry pass,
* so the five-file retry cap stays per CI shard; a missing, duplicate or
* incomplete sub-shard result fails the shard.
* - Sub-shard runs never write the duration seed (only `--record-durations`
* does, from one-file children).
*
* Per-file test and expect() counts come from bun's JUnit reporter, which
* attributes both per file inside a multi-file child on the pinned Bun 1.4.2
* (the console stream is byte-identical with or without it). Counts are
* evidence for k > 1 comparisons; they never decide a verdict.
*/
import * as fs from 'node:fs';
import * as os from 'node:os';
import type { FreeCiPlan, FreeCiResult, FreeShardOutcome } from '../test-free-shards';
import { guardFreeHome, sharedFreeHome, type FreeHomeGuard, type FreeHomeGuardFactory } from './free-home-guard';
import { normalizeRelativePath } from './shard-engine';
/**
* Files that pass alone but time out as a sub-shard under load. Each entry
* names the CI run id where it timed out (FREE_SUBSHARD_TIMEOUT prints it).
* They run in the trailing serial sub-shard under `--jobs`; at k = 1 nothing
* changes. Keys are pinned against the live file census by
* test/shard-jobs.test.ts. Never raise a timeout instead of adding an entry.
*/
export const SLOW_UNDER_LOAD: Record<string, { reason: string; runId: string }> = {};
export const FREE_JOBS_INVALID = 'FREE_JOBS_INVALID';
export const FREE_SUBSHARD_TIMEOUT = 'FREE_SUBSHARD_TIMEOUT';
export const FREE_SUBSHARD_REJECTED = 'FREE_SUBSHARD_REJECTED';
const DOCS = 'docs/TESTING_INTERNALS.md#sub-shards-inside-one-ci-job';
const JOBS_FIX = `--jobs applies only to --ci-run and defaults to 1; GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run. See ${DOCS}`;
/** `--jobs <k>` at parse time: digits only, at least 1. */
export function parseJobsFlag(raw: string | undefined): number {
if (raw === undefined || !/^\d+$/.test(raw) || Number(raw) < 1) {
throw new Error(`${FREE_JOBS_INVALID}: --jobs needs a positive whole number; got ${JSON.stringify(raw ?? '')}. ${JOBS_FIX}`);
}
return Number(raw);
}
export function rejectJobsWithoutCiRun(): never {
throw new Error(`${FREE_JOBS_INVALID}: --jobs needs --ci-run. ${JOBS_FIX}`);
}
/** `--jobs <k>` against the planned shard: k may not exceed its file count (an empty shard allows 1). */
export function validateJobsForShard(jobs: number, fileCount: number, shard: number): void {
const max = Math.max(1, fileCount);
if (jobs > max) {
throw new Error(`${FREE_JOBS_INVALID}: --jobs ${jobs} is more than CI shard ${shard}'s ${fileCount} file(s); use 1 to ${max}. ${JOBS_FIX}`);
}
}
export interface FileTestCounts { tests: number; pass: number; fail: number; skip: number; todo: number; expects: number }
export interface ShardTestCounts {
source: 'bun-junit';
/** Every reported file was planned and the per-file tests add up to bun's own `Ran N tests` count. */
complete: boolean;
problem?: string;
files: Record<string, FileTestCounts>;
totals: FileTestCounts;
}
const zeroCounts = (): FileTestCounts => ({ tests: 0, pass: 0, fail: 0, skip: 0, todo: 0, expects: 0 });
const addCounts = (into: FileTestCounts, from: FileTestCounts) => {
for (const key of Object.keys(into) as Array<keyof FileTestCounts>) into[key] += from[key];
};
const TAG = /<(\/?)(testsuites|testsuite|testcase|failure|error|skipped)\b((?:[^>"']|"[^"]*"|'[^']*')*?)(\/?)>/g;
const ATTR = /([\w:-]+)="([^"]*)"/g;
const ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
const decode = (text: string) => text.replace(/&(#x[0-9a-f]+|#\d+|\w+);/gi, (whole, entity: string) => entity[0] === '#'
? String.fromCodePoint(entity[1].toLowerCase() === 'x' ? parseInt(entity.slice(2), 16) : Number(entity.slice(1)))
: ENTITIES[entity] ?? whole);
const attributes = (text: string): Record<string, string> =>
Object.fromEntries([...text.matchAll(ATTR)].map(([, key, value]) => [key, decode(value)]));
/**
* Per-file counts from bun's JUnit XML. A testcase is a fail when it holds a
* failure or error, a todo when skipped with message TODO, a skip when
* skipped otherwise, else a pass; expects sum each testcase's assertions.
* Printed paths are matched back to planned files. Bun prints them relative
* to the child's real cwd, so a plain relative path must match exactly; an
* absolute or ../-prefixed path matches only when what precedes the planned
* file is the child's root. Files that never ran a test are absent.
*/
export function parseJunitCounts(xml: string | null, planned: readonly string[], testsRan: number | null, root = process.cwd()): ShardTestCounts {
const files: Record<string, FileTestCounts> = {};
const totals = zeroCounts();
const result = (complete: boolean, problem?: string): ShardTestCounts =>
({ source: 'bun-junit', complete, ...(problem ? { problem } : {}), files, totals });
if (xml === null) return result(false, 'bun wrote no JUnit report');
const plannedSet = new Set(planned);
const unknown = new Set<string>();
const roots = new Set([root, (() => { try { return fs.realpathSync(root); } catch { return root; } })()].map(normalizeRelativePath));
const isRoot = (prefix: string) => [...roots].some(dir => dir === prefix || dir.endsWith(`/${prefix}`));
const canonical = (printed: string) => {
const normalized = normalizeRelativePath(printed);
const stripped = normalized.replace(/^(?:\.{1,2}\/)+/, '');
if (plannedSet.has(stripped)) return stripped;
if (stripped === normalized && !/^(?:\/|[A-Za-z]:\/)/.test(stripped)) return stripped;
const matches = planned.filter(file => stripped.endsWith(`/${file}`) && isRoot(stripped.slice(0, -file.length - 1)));
return matches.length === 1 ? matches[0] : stripped;
};
let current: { file: string; state: 'pass' | 'fail' | 'skip' | 'todo'; expects: number } | null = null;
const finish = () => {
if (!current) return;
const entry = files[current.file] ??= zeroCounts();
entry.tests += 1;
entry[current.state] += 1;
entry.expects += current.expects;
if (!plannedSet.has(current.file)) unknown.add(current.file);
current = null;
};
for (const [, closing, name, body, selfClosing] of xml.matchAll(TAG)) {
if (name === 'testcase') {
if (closing) { finish(); continue; }
const attrs = attributes(body);
current = { file: canonical(attrs.file ?? ''), state: 'pass', expects: Number(attrs.assertions ?? 0) || 0 };
if (selfClosing) finish();
} else if (current && !closing && (name === 'failure' || name === 'error')) {
current.state = 'fail';
} else if (current && !closing && name === 'skipped' && current.state === 'pass') {
current.state = attributes(body).message === 'TODO' ? 'todo' : 'skip';
}
}
finish();
for (const entry of Object.values(files)) addCounts(totals, entry);
if (unknown.size) return result(false, `JUnit named unplanned file(s): ${[...unknown].sort().join(', ')}`);
if (testsRan === null) return result(false, "bun's summary has no test count");
if (totals.tests !== testsRan) return result(false, `JUnit counted ${totals.tests} tests, bun's summary ${testsRan}`);
return result(true);
}
const JUNIT_MAX_BYTES = 64 * 1024 * 1024;
/** The child-written report, only when it is a regular file of bounded size (never a link, FIFO or device). */
export function readJunitReport(file: string): string | null {
let fd: number;
try { fd = fs.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) | (fs.constants.O_NONBLOCK ?? 0)); } catch { return null; }
try {
const stat = fs.fstatSync(fd);
return stat.isFile() && stat.size <= JUNIT_MAX_BYTES ? fs.readFileSync(fd, 'utf8') : null;
} catch { return null; } finally { fs.closeSync(fd); }
}
/** One shard's counts from its sub-shards' counts; complete only when every part is and no file repeats. */
export function mergeTestCounts(parts: Array<ShardTestCounts | undefined>): ShardTestCounts | undefined {
if (parts.every(part => !part)) return undefined;
const files: Record<string, FileTestCounts> = {};
const totals = zeroCounts();
const problems: string[] = [];
parts.forEach((part, index) => {
if (!part) { problems.push(`sub-shard ${index + 1} has no counts`); return; }
if (!part.complete) problems.push(`sub-shard ${index + 1}: ${part.problem ?? 'incomplete'}`);
for (const [file, counts] of Object.entries(part.files)) {
if (files[file]) problems.push(`${file} counted twice`);
files[file] = { ...counts };
}
addCounts(totals, part.totals);
});
return { source: 'bun-junit', complete: problems.length === 0, ...(problems.length ? { problem: problems.join('; ') } : {}), files, totals };
}
export interface SubShardAssignment {
/** `<plan id prefix>:<shard>.<index>`: binds a result to one plan, shard and file set. */
id: string;
index: number;
phase: 'parallel' | 'serial';
files: string[];
}
/**
* Split one CI shard into disjoint sub-shards: up to k parallel ones packed by
* `pack`, then one serial sub-shard for the files `isSerial` names. Throws
* unless the assignments cover the shard's files exactly once.
*/
export function assignSubShards(
planId: string, shard: { shard: number; files: string[] }, jobs: number,
isSerial: (file: string) => boolean, pack: (files: string[], count: number) => string[][],
): SubShardAssignment[] {
const serial = shard.files.filter(isSerial);
const parallel = shard.files.filter(file => !isSerial(file));
const groups = parallel.length ? pack(parallel, Math.min(jobs, parallel.length)).filter(files => files.length) : [];
const assignments: SubShardAssignment[] = [
...groups.map(files => ({ phase: 'parallel' as const, files })),
...(serial.length ? [{ phase: 'serial' as const, files: serial }] : []),
].map((entry, index) => ({ id: `${planId.slice(0, 12)}:${shard.shard}.${index + 1}`, index: index + 1, ...entry }));
const assigned = assignments.flatMap(entry => entry.files);
if (new Set(assigned).size !== assigned.length || assigned.length !== shard.files.length
|| !shard.files.every(file => assigned.includes(file))) {
throw new Error(`${FREE_SUBSHARD_REJECTED}: sub-shard assignment for CI shard ${shard.shard} does not cover its files exactly once`);
}
return assignments;
}
export type SubShardSettlement = { assignment: SubShardAssignment } & ({ outcome: FreeShardOutcome } | { error: string });
/** Why one sub-shard's result cannot stand for its assignment; empty when it can. */
export function subShardProblems(settled: SubShardSettlement): string[] {
const { assignment } = settled;
if (!('outcome' in settled)) return [`${assignment.id} has no result (${settled.error})`];
const { outcome } = settled;
const problems: string[] = [];
if (JSON.stringify(outcome.files) !== JSON.stringify(assignment.files)) problems.push(`${assignment.id} reported another file set`);
const summary = outcome.summary;
if (!summary || (summary.sawTerminalSummary && summary.filesRan !== assignment.files.length)) {
problems.push(`${assignment.id} reported ${summary?.filesRan ?? 'no'} executed file(s) for ${assignment.files.length} assigned`);
} else if (outcome.status === 'passed' && (summary.filesRan !== assignment.files.length || summary.sawTerminalSummary !== true
|| outcome.exitCode !== 0 || outcome.failingFiles.length || outcome.unattributedFailures)) {
problems.push(`${assignment.id} passed without a complete execution summary`);
}
return problems;
}
export interface SubShardRecord {
id: string;
phase: SubShardAssignment['phase'];
files: string[];
status: FreeShardOutcome['status'] | 'missing';
exitCode: number | null;
elapsedMs: number;
summary?: FreeShardOutcome['summary'];
failingFiles: string[];
unattributedFailures: number;
}
/**
* One CI shard outcome from every sub-shard: the planned file order, summed
* summaries, the union of failing files, and the worst status. Every rejected
* sub-shard adds unattributed evidence, which also vetoes the retry pass.
*/
export function mergeSubShardOutcomes(
shard: { shard: number; files: string[] }, settled: SubShardSettlement[], elapsedMs: number, extraProblems: string[] = [],
): { outcome: FreeShardOutcome; records: SubShardRecord[]; problems: string[] } {
const problems = [...extraProblems];
const assigned = settled.flatMap(entry => entry.assignment.files);
if (new Set(assigned).size !== assigned.length) problems.push('a file is assigned to more than one sub-shard');
if (assigned.length !== shard.files.length || !shard.files.every(file => assigned.includes(file))) problems.push('sub-shards do not cover the CI shard');
for (const entry of settled) problems.push(...subShardProblems(entry));
const outcomes = settled.flatMap(entry => ('outcome' in entry ? [entry.outcome] : []));
const missing = settled.length - outcomes.length;
const status: FreeShardOutcome['status'] = outcomes.some(outcome => outcome.status === 'timed-out') ? 'timed-out'
: problems.length || outcomes.some(outcome => outcome.status !== 'passed') ? 'failed' : 'passed';
const testsRan = outcomes.map(outcome => outcome.summary?.testsRan ?? null);
const failures = outcomes.flatMap(outcome => outcome.failures ?? []);
const inFlight = outcomes.flatMap(outcome => outcome.inFlight ?? []);
const counts = mergeTestCounts(settled.map(entry => ('outcome' in entry ? entry.outcome.counts : undefined)));
const outcome: FreeShardOutcome = {
shard: shard.shard,
files: shard.files,
status,
exitCode: status === 'passed' ? 0 : outcomes.find(entry => entry.exitCode)?.exitCode ?? 1,
elapsedMs,
groupPid: null,
failingFiles: status === 'passed' ? [] : [...new Set(outcomes.flatMap(entry => entry.failingFiles))],
unattributedFailures: status === 'passed' ? 0 : outcomes.reduce((sum, entry) => sum + entry.unattributedFailures, 0) + problems.length,
...(failures.length ? { failures } : {}),
summary: {
testsRan: missing || testsRan.some(value => value === null) ? null : testsRan.reduce<number>((sum, value) => sum + value!, 0),
filesRan: outcomes.reduce((sum, entry) => sum + (entry.summary?.filesRan ?? 0), 0),
sawTerminalSummary: !missing && outcomes.every(entry => entry.files.length === 0 || entry.summary?.sawTerminalSummary === true),
},
...(counts ? { counts } : {}),
...(inFlight.length ? { inFlight } : {}),
};
const records = settled.map((entry): SubShardRecord => {
const { id, phase, files } = entry.assignment;
if (!('outcome' in entry)) return { id, phase, files, status: 'missing', exitCode: null, elapsedMs: 0, failingFiles: [], unattributedFailures: 1 };
const { status: subStatus, exitCode, elapsedMs: subElapsed, summary, failingFiles, unattributedFailures } = entry.outcome;
return { id, phase, files, status: subStatus, exitCode, elapsedMs: subElapsed, summary, failingFiles, unattributedFailures };
});
return { outcome, records, problems };
}
/** `--ci-verify` re-checks a k > 1 result's sub-shard records; empty means they stand. */
export function subShardRecordProblems(result: FreeCiResult): string[] {
if (result.jobs === undefined && result.subShards === undefined) return [];
const records = result.subShards ?? [];
if (!Number.isInteger(result.jobs) || result.jobs! < 2 || !records.length) return ['sub-shard result without a valid --jobs and records'];
const files = records.flatMap(record => record.files);
const problems: string[] = [];
if (new Set(files).size !== files.length || files.length !== result.outcome.files.length
|| !result.outcome.files.every(file => files.includes(file))) problems.push('sub-shard records do not cover the shard exactly once');
const prefix = `${result.planId.slice(0, 12)}:${result.outcome.shard}.`;
if (records.some((record, index) => record.id !== `${prefix}${index + 1}`)
|| records.filter(record => record.phase === 'parallel').length > result.jobs!
|| records.some((record, index) => record.phase === 'serial' ? index !== records.length - 1 : record.phase !== 'parallel')) {
problems.push('sub-shard ids or phases do not match the plan and --jobs');
}
if (records.some(record => record.status === 'missing' || record.summary?.filesRan !== record.files.length
|| (record.files.length > 0 && record.summary?.sawTerminalSummary !== true))) problems.push('a sub-shard did not run every assigned file');
if (result.outcome.status === 'passed' && records.some(record => record.status !== 'passed')) problems.push('a passing shard holds a failed sub-shard');
return problems;
}
export interface CiRunContext { planPath: string; runId: string; runner: string }
/** Where this run is: the CI run id (with attempt) and runner, else `local` and the host. */
export function ciRunContext(planPath: string, env: NodeJS.ProcessEnv = process.env): CiRunContext {
const runId = env.GITHUB_RUN_ID ? `${env.GITHUB_RUN_ID}${env.GITHUB_RUN_ATTEMPT ? `/attempt ${env.GITHUB_RUN_ATTEMPT}` : ''}` : 'local';
const cpus = os.availableParallelism?.() ?? os.cpus().length;
return { planPath, runId, runner: `${env.RUNNER_NAME || os.hostname()} (${process.platform}, ${cpus} CPUs)` };
}
const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
/** The FREE_SUBSHARD_TIMEOUT text: the file, k, runner, a serial reproduce command and the fix. */
export function subShardTimeoutText(
shard: number, jobs: number, record: SubShardRecord, inFlight: string[], wallTimeoutMs: number, context: CiRunContext, shardWallMs = wallTimeoutMs,
): string[] {
const suspects = inFlight.filter(file => record.files.includes(file));
const files = suspects.length ? suspects : record.files;
const seconds = (ms: number) => Math.round(ms / 1000);
const fix = record.phase !== 'serial'
? `if it passes serially, add it to SLOW_UNDER_LOAD in scripts/lib/shard-jobs.ts with run id ${context.runId}; never raise the timeout`
: wallTimeoutMs < shardWallMs
? `the parallel sub-shards left this serial sub-shard ${seconds(wallTimeoutMs)}s of the ${seconds(shardWallMs)}s shard wall; if the k=1 command passes, the parallel phase used up the budget and this shard needs a lower --jobs; never raise the timeout`
: 'this file already runs alone after the parallel sub-shards; fix or split it, never raise the timeout';
return [
`[test:free] ${FREE_SUBSHARD_TIMEOUT}: CI shard ${shard} sub-shard ${record.id} (${record.files.length} files) hit its ${seconds(wallTimeoutMs)}s wall at --jobs ${jobs} on ${context.runner}, run ${context.runId}.`,
`[test:free] in flight: ${suspects.length ? suspects.join(', ') : 'unknown (no open file chunk; see the sub-shard log)'}`,
`[test:free] reproduce serially: printf '%s\\n' ${files.map(quote).join(' ')} > files.txt && GSTACK_FREE_JOBS=1 bun run scripts/test-free-shards.ts --files-from files.txt`,
`[test:free] or the whole CI shard at k=1: bun run scripts/test-free-shards.ts --ci-run ${quote(context.planPath)} --shard ${shard} --result shard-${shard}.json --jobs 1`,
`[test:free] ${fix}. See ${DOCS}`,
];
}
export interface RunShardOptions { wallTimeoutMs: number; verbose: boolean; testCounts: true; homeGuard?: FreeHomeGuardFactory }
export interface CiShardInput {
plan: FreeCiPlan;
shard: FreeCiPlan['shards'][number];
revision: string;
/** Sub-shard count; 1 is today's single child. */
jobs: number;
wallTimeoutMs: number;
verbose: boolean;
context: CiRunContext;
}
export interface CiShardDeps {
runShard(files: string[], shardNumber: number, totalShards: number, options: RunShardOptions): Promise<FreeShardOutcome>;
/** The free runner's single flaky-retry pass over every collected outcome. */
retry(outcomes: FreeShardOutcome[]): Promise<{ exitCode: number; retry: FreeShardOutcome | null }>;
pack(files: string[], count: number): string[][];
isSerial(file: string): boolean;
terminationRequested(): boolean;
phaseGuard?(files: string[], shards: number): FreeHomeGuard;
log?(line: string): void;
}
/** Run one planned CI shard at k sub-shards and build its result JSON. */
export async function runCiShard(input: CiShardInput, deps: CiShardDeps): Promise<{ result: FreeCiResult; exitCode: number }> {
const { plan, shard, revision, jobs, wallTimeoutMs, verbose } = input;
const log = deps.log ?? ((line: string) => console.log(line));
if (jobs === 1) {
const outcome = await deps.runShard(shard.files, shard.shard, plan.shards.length, { wallTimeoutMs, verbose, testCounts: true });
const retried = await deps.retry([outcome]);
return { result: { planId: plan.id, revision, outcome, retry: retried.retry }, exitCode: retried.exitCode };
}
const startedAt = Date.now();
const assignments = assignSubShards(plan.id, shard, jobs, deps.isSerial, deps.pack);
const total = assignments.length;
log(`[test:free] CI shard ${shard.shard}/${plan.shards.length} at --jobs ${jobs}: ${assignments
.map(entry => `sub-shard ${entry.id} = runner shard ${entry.index}/${total} (${entry.phase}, ${entry.files.length} files)`).join('; ')}`);
const walls = new Map<string, number>();
const run = (entry: SubShardAssignment, homeGuard?: FreeHomeGuardFactory, wall = wallTimeoutMs): Promise<SubShardSettlement> => {
walls.set(entry.id, wall);
return deps.runShard(entry.files, entry.index, total, { wallTimeoutMs: wall, verbose, testCounts: true, ...(homeGuard ? { homeGuard } : {}) })
.then(outcome => ({ assignment: entry, outcome }), (error: unknown) => ({ assignment: entry, error: error instanceof Error ? error.message : String(error) }));
};
const parallel = assignments.filter(entry => entry.phase === 'parallel');
const serial = assignments.filter(entry => entry.phase === 'serial');
const phaseProblems: string[] = [];
const settled: SubShardSettlement[] = [];
if (parallel.length) {
const guard = deps.phaseGuard ?? ((files: string[], shards: number) => guardFreeHome(files, process.env, { kind: 'concurrent', shards }));
const phaseHome = guard(parallel.flatMap(entry => entry.files), parallel.length);
settled.push(...await Promise.all(parallel.map(entry => run(entry, sharedFreeHome))));
const homeChange = phaseHome.verify();
if (homeChange) phaseProblems.push(homeChange);
}
for (const entry of serial) {
const left = startedAt + wallTimeoutMs - Date.now();
settled.push(deps.terminationRequested() ? { assignment: entry, error: 'cancelled before the serial phase' }
: left <= 0 ? { assignment: entry, error: 'no wall left after the parallel phase' } : await run(entry, undefined, left));
}
const merged = mergeSubShardOutcomes(shard, settled, Date.now() - startedAt, phaseProblems);
for (const problem of merged.problems) console.error(`[test:free] ${FREE_SUBSHARD_REJECTED}: CI shard ${shard.shard}: ${problem}`);
for (const record of merged.records.filter(entry => entry.status === 'timed-out')) {
for (const line of subShardTimeoutText(shard.shard, jobs, record, merged.outcome.inFlight ?? [], walls.get(record.id) ?? wallTimeoutMs, input.context, wallTimeoutMs)) console.error(line);
}
log(`[test:free] CI shard ${shard.shard}: ${merged.outcome.status} across ${total} sub-shards in ${Math.round(merged.outcome.elapsedMs / 1000)}s`);
const retried = await deps.retry([merged.outcome]);
return {
result: { planId: plan.id, revision, outcome: merged.outcome, retry: retried.retry, jobs, subShards: merged.records },
exitCode: retried.exitCode,
};
}
/** The duration seed takes only one-file children's walls, never a sub-shard's or a shard's. */
export function assertSoloSeedSample(file: string, outcome: Pick<FreeShardOutcome, 'files' | 'summary'>): void {
if (outcome.files.length !== 1 || outcome.files[0] !== file || (outcome.summary?.filesRan ?? 0) > 1) {
throw new Error(`Duration seed sample for ${file} came from a ${outcome.files.length}-file child; seeds are recorded only from one-file serial children.`);
}
}
+75
View File
@@ -258,6 +258,26 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
file: 'test/claude-overlay-setup-installs.test.ts',
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/claude-overlay-fixture.ts, so the source scan misses them',
},
{
file: 'test/claude-overlay-setup-lifecycle.test.ts',
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/claude-overlay-fixture.ts, so the source scan misses them',
},
{
file: 'test/claude-overlay-setup-project.test.ts',
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/claude-overlay-fixture.ts, so the source scan misses them',
},
{
file: 'test/host-renders-prune-shared-root.test.ts',
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/install-fixture.ts, so the source scan misses them',
},
{
file: 'test/host-renders-prune-migration.test.ts',
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/install-fixture.ts, so the source scan misses them',
},
{
file: 'test/setup-codex-scope-runtime-links.test.ts',
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/install-fixture.ts and test/helpers/setup-codex-scope-cyclic.ts, so the source scan misses them',
},
{
file: 'test/cso-scanner-cli.test.ts',
reason: 'drives the prebuilt POSIX CSO launcher with /usr/bin/git and a POSIX-only PATH; native Windows launcher behavior is covered by the dedicated cso-windows-launcher gate',
@@ -295,18 +315,63 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
file: 'test/ship-queue-train-land.test.ts',
reason: 'drives train land against test/helpers/ship-queue-train-fixture.ts plus an extensionless node gh wrapper on a colon-joined PATH; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead, as for test/ship-queue-train.test.ts and test/ship-queue-land.test.ts (windows-free-shard 6, run 40824965988)',
},
{
file: 'test/ship-queue-land-lock.test.ts',
reason: 'drives land and train land against test/helpers/ship-queue-train-fixture.ts plus an extensionless node gh wrapper on a colon-joined PATH; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead, as for test/ship-queue-train-land.test.ts and test/ship-queue-land.test.ts (windows-free-shard 6, run 40824965988)',
},
{
file: 'test/ship-queue-train.test.ts',
reason: 'drives bin/gstack-ship-queue against the stateful #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead of the shim (windows-free-shard run 39152702056)',
},
{
file: 'test/ship-queue-train-bisect.test.ts',
reason: 'split from test/ship-queue-train.test.ts (GSTA-151): drives bin/gstack-ship-queue against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts',
},
{
file: 'test/ship-queue-train-verify.test.ts',
reason: 'split from test/ship-queue-train.test.ts (GSTA-151): drives bin/gstack-ship-queue against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts',
},
{
file: 'test/ubicloud-runner-tiers.test.ts',
reason: 'split from test/ubicloud-runner.test.ts, excluded as a whole: runs scripts/ubicloud/ubi-runner.sh through bash with the #!/usr/bin/env bash curl, ssh and make shims test/helpers/ubi-runner-fake-vm.ts puts on a colon-joined PATH, like test/ubicloud-runner-run.test.ts',
},
{
file: 'test/ship-queue-train-freeze.test.ts',
reason: 'split from test/ship-queue-train.test.ts (GSTA-151): drives bin/gstack-ship-queue against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts',
},
{
file: 'test/ship-queue-train-cost.test.ts',
reason: 'drives bin/gstack-ship-queue train verdict against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead of the shim (windows-free-shard 2 on #3478 at d5e6a7d)',
},
// GSTA-151 splits: parts of files the curation excluded as a whole keep that platform scope.
{
file: 'test/docsync-fault-interface-legacy.test.ts',
reason: 'split from test/docsync-fault-interface.test.ts, excluded as a whole by its POSIX mode-bit checks; replays the same docsync callbacks through the generated bun-test child',
},
{
file: 'test/docsync-fault-interface-faults.test.ts',
reason: 'split from test/docsync-fault-interface.test.ts, excluded as a whole by its POSIX mode-bit checks; replays the same docsync callbacks through the generated bun-test child',
},
{
file: 'test/shared-libs-checker-interface-evidence.test.ts',
reason: 'runs bin/gstack-review-log (shebang) through test/helpers/shared-libs-checker-evidence.ts captures (Windows CreateProcess does not parse shebangs)',
},
{
file: 'test/shared-libs-checker-interface-evidence-declared.test.ts',
reason: 'split from test/shared-libs-checker-interface-evidence.test.ts: runs bin/gstack-review-log (shebang) through test/helpers/shared-libs-checker-evidence.ts captures (Windows CreateProcess does not parse shebangs)',
},
{
file: 'test/shared-libs-checker-interface-evidence-replays.test.ts',
reason: 'split from test/shared-libs-checker-interface-evidence.test.ts, which spawned bin/gstack-review-log (shebang); its prerequisite checks run the same review-log commands through bash',
},
{
file: 'test/ship-queue-train-ops.test.ts',
reason: 'drives bin/gstack-ship-queue (screen verdicts, scratch-merge screens, prune --stale, watch, forget, verify --post-merge, ready --dispatch-ci, --stack-tests) against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts',
},
{
file: 'test/ship-queue-train-generic.test.ts',
reason: 'drives bin/gstack-ship-queue train build, verdict, land, watch, init and doctor against the same #!/bin/sh gh shim plus an extensionless node gh wrapper on a colon-joined PATH; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead (windows-free-shard 1, run 41218228855)',
},
{
file: 'test/ship-policy-compat.test.ts',
reason: 'spawns bin/gstack-ship-policy, bin/gstack-restamp and bin/gstack-tree-receipt (shebangs) through test/helpers/restamp-fixture.ts',
@@ -323,6 +388,10 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
file: 'test/ship-release.test.ts',
reason: 'spawns bin/gstack-ship-release (shebang) directly and writes #!/bin/sh gh stubs through test/helpers/ship-bin-fixture.ts; windows-latest exits before the bin runs (windows-free-shard 3, run 41073592483)',
},
{
file: 'test/ship-rereview-triggers.test.ts',
reason: 'drives bin/gstack-ship-release, gstack-docs-candidate and gstack-wtree (shebangs) through test/helpers/ship-bin-fixture.ts like test/ship-release.test.ts; windows-latest fails every bin case at the first spawn (windows-free-shard 6, run 41193336209)',
},
];
// Force-include overrides: files a WINDOWS_FRAGILE_PATTERNS regex excludes for
@@ -337,16 +406,22 @@ export const KNOWN_WINDOWS_SAFE: Array<{ file: string; reason: string }> = [
{ file: 'test/cso-windows-docker.test.ts', reason: '/cso docker.exe discovery under known-folder roots (real path, no reparse points) and the native-transport-unsupported outcome; pure win32 path logic' },
{ file: 'test/copilot-windows-bash.test.ts', reason: 'the Copilot glossary line that runs bash blocks in Git for Windows Bash; reads host config only' },
{ file: 'test/ship-hook-windows-paths.test.ts', reason: 'runs bin/ helpers through explicit bash and Bun argv with forward-slash paths; never executes a shebang; the path-spelling simulation is skipIf win32' },
{ file: 'test/review-scope-windows-launch.test.ts', reason: '#3541: runs gstack-review-scope through lib/script-invocation.ts, which must reach the bash gstack-review-read through Git Bash (red at main: windows-free-shard 3, run 41228091417); os.tmpdir state only' },
{ file: 'test/jsonl-merge-encoding.test.ts', reason: '#3659: runs gstack-jsonl-merge through lib/script-invocation.ts (Git Bash on win32); the CRLF half of the bug only reproduces with a Windows Python' },
{ file: 'test/gstack-wtree-windows-paths.test.ts', reason: '#3101: runs bin/gstack-wtree through explicit bash in an os.tmpdir linked worktree with forward-slash paths; never executes a shebang; the drive-letter simulation is skipIf win32' },
{ file: 'test/setup-hook-syntax-gate.test.ts', reason: '#3088/#3100: runs bin/gstack-hook-check and the setup hook sections through explicit bash with forward-slash paths; never executes a shebang. Its Bun drive-path spelling is the Windows bug it guards' },
{ file: 'test/bun-config-shell-bins.test.ts', reason: '#3415: runs the record bins through lib/script-invocation.ts (Git Bash on Windows) with MSYS path conversion off, the case it guards; its bun and cygpath stand-ins and simulated-OSTYPE cases are skipIf win32' },
{ file: 'test/ship-receipt-session-kind.test.ts', reason: '#3541: runs gstack-ship-receipt write through lib/script-invocation.ts, which must reach the bash gstack-session-kind through Git Bash; os.tmpdir git fixture only' },
{ file: 'test/path-spelling.test.ts', reason: 'sources bin/gstack-path-spelling.sh through explicit bash and checks lib/path-spelling.ts; on Windows it exercises the real Git Bash spellings' },
{ file: 'test/timeline-stop-hook-stdin.test.ts', reason: '#3628: the hook runs through Bun argv with a stdin pipe held open; the return-within-budget timing must meet real Windows process startup' },
{ file: 'test/state-root-parity.test.ts', reason: 'runs the bash twin and lib/state-root.ts over an env table with PATH empty; no shebang execution, raw-string comparison is platform-neutral' },
{ file: 'test/generator-eexist.test.ts',
reason: 'E4: runs the generators through Bun argv with the Windows EEXIST emulation preload; no shebang execution' },
{ file: 'test/gstack-config-gbrain-refresh.test.ts', reason: 'E6: runs bin/gstack-config through explicit bash; its gbrain/python3 shims are found by bash PATH lookup, never launched by CreateProcess' },
{ file: 'test/hook-scripts.test.ts', reason: '#3067 PowerShell and NotebookEdit hook coverage: every hook runs through spawnSync("bash", [script]) with the payload on stdin, scratch state lives under os.tmpdir(); its /tmp/ literals are payload strings, never opened; symlink, newline-path and POSIX-PATH cases are skipIf win32' },
{ file: 'test/gstack-doctor.test.ts', reason: 'runs bin/gstack-doctor and setup --status through explicit bash argv in an os.tmpdir fixture; its bun/codex/claude/probe stubs are found by bash PATH lookup, and expected paths use bash spellings' },
{ file: 'test/gstack-doctor-check.test.ts', reason: 'split from test/gstack-doctor.test.ts: runs bin/gstack-doctor --check through explicit bash argv in the test/helpers/doctor-fixture.ts os.tmpdir fixture; its stubs are found by bash PATH lookup, and expected paths use bash spellings' },
{ file: 'test/gstack-doctor-launch.test.ts', reason: 'split from test/gstack-doctor.test.ts: runs bin/gstack-doctor through explicit bash argv in the test/helpers/doctor-fixture.ts os.tmpdir fixture; Windows bash and platform rows are injected through GSTACK_DOCTOR_PLATFORM, and POSIX-path cases skip on win32' },
// Fix wave PR 5 (item 10b, #3197): the fixture runBin launches bins through lib/script-invocation.ts
// (bun bins as Bun argv, bash bins through Git Bash); passed on windows-latest in the fix wave PR 5 windows-free-tests run.
{ file: 'test/gstack-regen.test.ts', reason: 'runs bin/gstack-regen through the fixture runBin (lib/script-invocation.ts) and its regen commands through the resolved Git Bash' },
+5 -1
View File
@@ -216,7 +216,11 @@
"test/skill-e2e-triage.test.ts": 1000,
"test/skill-e2e-workflow.test.ts": 121594,
"test/skill-llm-eval.test.ts": 339704,
"test/skill-routing-e2e.test.ts": 81360
"test/skill-routing-e2e.test.ts": 81360,
"test/skill-e2e-planning-wave.test.ts#planning-wave-planted-facts": 74000,
"test/skill-e2e-planning-wave.test.ts#planning-wave-schema": 52000,
"test/skill-e2e-planning-wave.test.ts#planning-wave-tasks-rerun": 37000,
"test/skill-e2e-planning-wave.test.ts#planning-wave-closing-signals": 27000
}
}
}
+1 -1
View File
@@ -89,7 +89,7 @@ export const SELECTION_DATA_MODULES: readonly string[] = [
// The build entry list: every compiled binary, which skills reach through their `--outfile` path instead.
'scripts/build.sh',
// The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes.
'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts',
'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', 'lib/result-codes-plan.ts', 'lib/result-codes-autoplan.ts',
// The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case).
'test/helpers/exercised-commands.json',
];
+6 -6
View File
@@ -17,17 +17,17 @@ import type { ResolverFn } from './types';
/** One entry per command line the loop runs, as the field-wave plan specifies them. */
export const AUTOPLAN_DRIVER_COMMANDS: Array<{ cmd: string; does: string }> = [
{ cmd: 'gstack-autoplan next --out <dir> --plan <plan.md> [scope flags] [--light] [--spend-cap <usd>] [--max-rounds <n>] [--scaffold]', does: `first call: init the run, print PHASE:, SCOPE:, DRIVER: and one ATTEMPT: per voice; --max-rounds caps review rounds per phase (default ${DEFAULT_MAX_ROUNDS} unattended), and an unattended Capy run reserves ${DEFAULT_HOST_ESTIMATE_USD.toFixed(2)} per host attempt unless --estimate-usd <n> says otherwise, so --spend-cap binds` },
{ cmd: 'gstack-autoplan next --out <dir> --plan <plan.md> [scope flags] [--light] [--spend-cap <usd>] [--max-rounds <n>] [--estimate-usd <n>] [--scaffold] [--ceo-only]', does: `first call: init the run, print PHASE:, SCOPE:, DRIVER: and one ATTEMPT: per voice; --max-rounds caps review rounds per phase (default ${DEFAULT_MAX_ROUNDS} unattended); a run with --spend-cap, in any session kind (or any unattended Capy run), reserves ${DEFAULT_HOST_ESTIMATE_USD.toFixed(2)} per voice for each attempt, so the cap binds; --estimate-usd <n> sets that reservation and the run keeps it for later phases; --ceo-only (command line only, never a scope marker) reviews CEO alone and ends \`status=reviewed\` with \`<dir>/ceo-review.md\` (\`REVIEWED:\`): no gate, and the gate verbs refuse it (\`CEO_ONLY_NO_GATE\`)` },
{ cmd: 'gstack-autoplan run --out <dir> --plan <plan.md> --native-model <id> --outside-model <id> --native-runner api --runner api [--max-tokens <n>] [--spend-cap <usd>]', does: 'hosts without subagents: one command calls both models directly (ANTHROPIC_API_KEY, OPENAI_API_KEY) and loops next/submit to the gate; a response cut off at the output limit is reported truncated, so raise --max-tokens (default 16384 for both providers; OpenAI counts reasoning tokens toward it); each api call reserves its priced worst case (a model with no price reserves the per-voice default under a cap), a native call that went out but bound no result settles at its usage, or its full reservation when the usage is unknown, and a failed outside call stays counted at its reservation' },
{ cmd: 'gstack-autoplan next --out <dir> [--redispatch <attempt>]', does: 'later calls: open the next phase or reprint open attempts; never redispatches on its own' },
{ cmd: 'gstack-autoplan submit --out <dir> --phase <p> --voice native|outside --record <file.md> --model <id> [--attempt <id>]', does: 'bind a free-form review record (canonical fence, one findings table or list, or `No findings.`)' },
{ cmd: 'gstack-autoplan submit ... --record <file.md> --attest "<reason>" --attempt <id>', does: 'bind a record with no INPUT: line as receipt: attested (a wrong INPUT: always refuses)' },
{ cmd: 'gstack-autoplan submit --out <dir> --phase <p> --voice native|outside --result <file> --model <id> --attempt <id>', does: 'bind a canonical result (the RESULT FORMAT below)' },
{ cmd: 'gstack-autoplan submit --out <dir> --phase <p> --voice native|outside --result <file> --model <id> --attempt <id> [--usage-usd <n>] [--reconcile <jsonl>]', does: 'bind a canonical result (the RESULT FORMAT below); `--usage-usd` settles the attempt at what the reviewer actually cost (else it stays priced unknown at its reservation); on the submit that closes the phase, `--reconcile` takes your own pairing of findings both voices raised, one `{"outside_id","native_id","disposition","resolution"}` row each, over the structural one' },
{ cmd: 'gstack-autoplan submit --out <dir> --phase <p> --voice outside --unavailable|--failed "<reason>"', does: 'close the phase native-only when no outside voice can run' },
{ cmd: 'gstack-autoplan answer --out <dir> --gate-rev <n> --reply "<reply>" --by human|agent:<id> [--owner-reply <text>|-] [--rec-hash <h>]', does: 'answer the gate with the reply grammar below; an agent answer stores the owner\'s words verbatim' },
{ cmd: 'gstack-autoplan answer --out <dir> --gate-rev <n> --reply "<reply>" --by human|agent:<id> [--owner-reply <text>|-] [--words-by agent:<id>] [--rec-hash <h>]', does: 'answer the gate with the reply grammar below; an agent answer stores the owner\'s words verbatim, and passes --words-by agent:<id> whenever the words are its own (the default is owner)' },
{ cmd: 'gstack-autoplan recommend --out <dir> --gate-rev <n> --item <id> --option <k> --basis "<why>"', does: 'change one recommendation before relay; then render and relay again (a `rec` needs the new `--rec-hash`)' },
{ cmd: 'gstack-autoplan revise --out <dir> --plan <plan v2> --incorporates <decision ids>', does: 'bind the amended plan; reopens every phase whose reviewed plan changed, Eng last' },
{ cmd: 'gstack-autoplan revise --out <dir> --settle <decision id|d4>', does: 'discharge a packaging or ordering change with no plan edit; reopens nothing' },
{ cmd: 'gstack-autoplan revise --out <dir> --withdraw <decision id>', does: 'drop a pending amendment instead of applying it' },
{ cmd: 'gstack-autoplan revise --out <dir> --settle <decision id|d4> | --withdraw <decision id>', does: '--settle discharges a packaging or ordering change with no plan edit and reopens nothing; --withdraw drops a pending amendment instead of applying it' },
{ cmd: 'gstack-autoplan close --out <dir> --phase <p> --deferred "<reason>" --by human|agent:<id>', does: 'close a phase accepted-with-findings: its open findings go to tasks.jsonl, and Critical, urgent or user-challenge findings become gate items; also the recovery from `STOP: reason=budget_exhausted`' },
{ cmd: 'gstack-autoplan budget --out <dir> [--spend-cap <usd>] [--max-rounds <n>] --by human|agent:<id>', does: 'the owner\'s journaled cap raise; after a budget stop, `resume` continues the run' },
{ cmd: 'gstack-autoplan export --out <dir>', does: 'write the artifact set; then `gstack-artifact validate <dir>/run.json`' },
@@ -40,7 +40,7 @@ export const AUTOPLAN_DRIVER_COMMANDS: Array<{ cmd: string; does: string }> = [
export const GATE_REPLY_FORMS: Array<{ reply: string; means: string }> = [
{ reply: 'rec', means: 'every item takes its recommendation, plan changes included (each prints APPLY:); `all` and `do it` are aliases' },
{ reply: 'as-is', means: 'keep every direction; no plan change' },
{ reply: 'd3b uc1a', means: 'one <id><option> token per item you set; unanswered auto items take their default' },
{ reply: 'd3b uc1a', means: 'one <id><option> token per item you set; unanswered auto items take their default (a finding item keeps the plan as written; `p1b` applies it)' },
{ reply: 'rec except d3b', means: 'recommendations, with the listed overrides' },
];
+3 -4
View File
@@ -593,11 +593,10 @@ Show the screenshot to the user. Ask: "Does this feel right? Want to iterate on
If they want changes, regenerate the HTML with their feedback and re-render.
If they approve or say "good enough," proceed.
**Step 5: Include in design doc**
**Step 5: Save the sketch**
Reference the wireframe screenshot in the design doc's "Recommended Approach" section.
The screenshot file at \`<sketch-dir>/sketch.png\` (name the full path in the doc) can be referenced by downstream skills
(\`/plan-design-review\`, \`/design-review\`) to see what was originally envisioned.
Copy the temporary \`sketch.png\` and \`sketch.html\` to \`<PROJECT_DIR>/{user}-{branch}-sketch-{datetime}.*\`.
In "Recommended Approach" add \`Wireframe: projects/<slug>/<png name>\` (state-root relative), or \`Wireframe: none (<reason>)\`.
**Step 6: Outside design voices** (optional)
@@ -1,6 +1,6 @@
import type { TemplateContext } from '../types';
import { getHostConfig } from '../../../hosts/index';
import { runtimeRootPrelude } from '../runtime-root';
import { skillStartLines } from '../runtime-root';
/**
* Preamble bootstrap (token-reduction Phase 1).
@@ -41,12 +41,11 @@ function loadoutStartFlags(ctx: TemplateContext): string {
export function generatePreambleBash(ctx: TemplateContext): string {
const hostConfig = getHostConfig(ctx.host);
const runtimeRoot = hostConfig.usesEnvVars ? `${runtimeRootPrelude(ctx)}\n` : '';
const start = hostConfig.usesEnvVars ? skillStartLines(ctx) : null;
const runtimeRoot = start ? `${start.lookup}\n` : '';
const brainHealthFlag = ctx.host === 'gbrain' || ctx.host === 'hermes' ? ' --brain-health' : '';
const loadoutFlags = loadoutStartFlags(ctx);
const startCommand = hostConfig.usesEnvVars
? `"${ctx.paths.binDir}/gstack-skill-start"`
: `${ctx.paths.binDir}/gstack-skill-start`;
const startCommand = start ? start.command : `${ctx.paths.binDir}/gstack-skill-start`;
const entry = ['plan-design-review', 'plan-eng-review'].includes(ctx.skillName)
? `## Preamble (after scope gate)
+41 -5
View File
@@ -3,8 +3,9 @@
* bash finds gstack on Codex and every other `usesEnvVars` host.
*
* Those hosts run each fenced block in a fresh shell, so `GSTACK_ROOT`, `B`,
* `D` and `P` set by an earlier block are gone. The preamble's root resolution and
* the per-fence prelude come from `runtimeRootPrelude()`; `insertRuntimePreludes()`
* `D` and `P` set by an earlier block are gone. The preamble's skill-start fence
* comes from `skillStartLines()` and the per-fence prelude from
* `runtimeRootPrelude()`; `insertRuntimePreludes()`
* is the single post-render pass (gen-skill-docs, before rewriteInstallRoot)
* that puts it into every fence that uses these variables without assigning
* them. Claude output is returned unchanged.
@@ -13,6 +14,9 @@
* on a per-install render, the literal install root (no git call); otherwise the
* repo-local install, then the host's global root (`CODEX_HOME` on Codex). No
* root prints one line naming what was tried and the setup command, and exits.
* The skill-start fence differs (GST-882): an install-root render calls the
* literal install root's start script and ignores an exported `GSTACK_ROOT`, and
* with no root either render prints that line and lets the call fail.
*/
import { getHostConfig } from '../../hosts/index';
import { toShellPath, type TemplateContext } from './types';
@@ -37,13 +41,45 @@ const uses = (name: string) => new RegExp(`\\$\\{?${name}\\b`);
*/
function resolveRoot(ctx: TemplateContext): string {
const host = getHostConfig(ctx.host);
const fix = `Fix: ./setup --host ${host.name} from your gstack checkout; ./setup --status shows it.`;
const fix = installFix(ctx);
const exported = '[ -d "${GSTACK_ROOT:-/-}/bin" ]&&[ -d "$GSTACK_ROOT/lib" ]||';
if (ctx.installRoot) {
return `${exported}{ GSTACK_ROOT="${ctx.installRoot.replace(/\/+$/, '')}";[ -d "$GSTACK_ROOT/bin" ]||{ echo "gstack: no install found (tried $GSTACK_ROOT). ${fix}">&2;exit 1;};}`;
}
const global = host.name === 'codex' ? '${CODEX_HOME:-~/.codex}/skills/gstack' : `~/${host.globalRoot}`;
return `${exported}{ _r=$(git rev-parse --show-toplevel 2>/dev/null)/${host.localSkillRoot};[ -d "$_r/bin" ]||_r=${global};[ -d "$_r/bin" ]||{ echo "gstack: no install found (tried $_r). ${fix}">&2;exit 1;};GSTACK_ROOT=$_r;}`;
return `${exported}{ _r=$(git rev-parse --show-toplevel 2>/dev/null)/${host.localSkillRoot};[ -d "$_r/bin" ]||_r=${globalRoot(ctx)};[ -d "$_r/bin" ]||{ echo "gstack: no install found (tried $_r). ${fix}">&2;exit 1;};GSTACK_ROOT=$_r;}`;
}
const installFix = (ctx: TemplateContext) => `Fix: ./setup --host ${getHostConfig(ctx.host).name} from your gstack checkout; ./setup --status shows it.`;
const globalRoot = (ctx: TemplateContext) => {
const host = getHostConfig(ctx.host);
return host.name === 'codex' ? '${CODEX_HOME:-~/.codex}/skills/gstack' : `~/${host.globalRoot}`;
};
/**
* The skill-start fence on env-var hosts (GST-882). Hermes's Tirith blocks a
* start command it cannot resolve statically, a grouped fallback or a
* variable-selected path alike, and no allowlist entry clears that finding. An
* install-root render therefore names the literal start path; a checked-in
* render keeps resolveRoot's lookup order as plain lines. Either way a missing
* install prints the fix, and the call then fails.
*/
export function skillStartLines(ctx: TemplateContext): { lookup: string; command: string } {
const fix = installFix(ctx);
if (ctx.installRoot) {
const root = ctx.installRoot.replace(/\/+$/, '');
return {
lookup: `[ -x "${root}/bin/gstack-skill-start" ]||echo "gstack: no install found (tried ${root}). ${fix}">&2`,
command: `"${root}/bin/gstack-skill-start"`,
};
}
return {
lookup: [
`[ -d "\${GSTACK_ROOT:-/-}/lib" ]&&[ -d "$GSTACK_ROOT/bin" ]||GSTACK_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)/${getHostConfig(ctx.host).localSkillRoot}`,
`[ -d "$GSTACK_ROOT/bin" ]||GSTACK_ROOT=${globalRoot(ctx)}`,
`[ -d "$GSTACK_ROOT/bin" ]||echo "gstack: no install found (tried $GSTACK_ROOT). ${fix}">&2`,
].join('\n'),
command: '"$GSTACK_ROOT/bin/gstack-skill-start"',
};
}
const DERIVED: Record<string, string> = { BIN: 'bin', BROWSE: 'browse/dist', DESIGN: 'design/dist', MAKE_PDF: 'make-pdf/dist' };
+6 -40
View File
@@ -32,6 +32,10 @@ empty JSONL files array. Each task needs a concrete verification step.` : `Befor
build-actionable tasks. Each task derives from a specific finding — no padding.`}
${conditionalWrites ? `Always emit the markdown section. Write its JSONL artifact for \`/autoplan\` only when the ${storagePolicy} permits it; otherwise label the complete task output not persisted and do not claim an aggregation artifact exists.` : 'Emit the markdown section AND write a JSONL artifact that `/autoplan` can\naggregate across phases.'}
### Earlier lists on this branch
Before listing tasks, run \`~/.claude/skills/gstack/bin/gstack-tasks prior --phase ${phase}\` and reconcile what it prints, so a rerun never leaves two lists.
### Markdown section (always emit)
\`\`\`markdown
@@ -54,49 +58,11 @@ Rules:
### JSONL artifact (${conditionalWrites ? 'write when permitted, including zero tasks' : 'always write, even if zero tasks'})
\`/autoplan\` reads this file to aggregate across phases. Build each line with
\`jq -nc\` so titles and source findings containing quotes, newlines, or
backslashes serialize cleanly — never use hand-rolled \`echo\` / \`printf\`.
\`/autoplan\` aggregates this list. With your file-write tool, write one JSON object per line to \`.gstack/tmp/tasks-${phase}-rows.jsonl\` at the repo root (no free text in a command; no rows, no file): a task is \`{"id":"T1","priority":"P1","component":"","title":"","source_finding":"","effort_human":"~2h","effort_cc":"~15min","files":[]}\`, plus \`"carried_from":"<run_id>/<id>"\` when carried; a closed earlier task is \`{"prior":"<run_id>/<id>","disposition":"done|dropped","reason":""}\`. Then write the list (empty means "ran, no findings", distinct from "didn't run"):
\`\`\`bash
GSTACK_STATE_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get GSTACK_STATE_ROOT); : "\${GSTACK_STATE_ROOT:?gstack-paths failed; reinstall with ./setup or /gstack-upgrade}"
SLUG=$(~/.claude/skills/gstack/bin/gstack-slug --get SLUG 2>/dev/null)
TASKS_DIR="$GSTACK_STATE_ROOT/projects/\${SLUG:-unknown}"
mkdir -p "$TASKS_DIR"
TASKS_FILE="$TASKS_DIR/tasks-${phase}-$(date +%Y%m%d-%H%M%S).jsonl"
COMMIT=$(git rev-parse HEAD 2>/dev/null || echo unknown)
# The task record keeps the raw git branch; artifact filenames keep gstack-slug's sanitized BRANCH.
GIT_BRANCH=$(git branch --show-current 2>/dev/null || echo unknown)
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$"
# Repeat ONE jq invocation per task identified during this review.
# Substitute the placeholders inline with shell variables you set per task:
# TASK_ID (T1, T2, ...), PRIORITY (P1/P2/P3), COMPONENT, TITLE,
# SOURCE_FINDING, EFFORT_HUMAN, EFFORT_CC, FILES_JSON (a JSON array literal
# like '["browse/src/sanitize.ts","browse/src/server.ts"]').
jq -nc \\
--arg phase '${phase}' \\
--arg run_id "$RUN_ID" \\
--arg branch "$GIT_BRANCH" \\
--arg commit "$COMMIT" \\
--arg id "$TASK_ID" \\
--arg priority "$PRIORITY" \\
--arg component "$COMPONENT" \\
--arg effort_human "$EFFORT_HUMAN" \\
--arg effort_cc "$EFFORT_CC" \\
--arg title "$TITLE" \\
--arg source_finding "$SOURCE_FINDING" \\
--argjson files "$FILES_JSON" \\
'{phase:$phase, run_id:$run_id, branch:$branch, commit:$commit, id:$id, priority:$priority, component:$component, files:$files, effort_human:$effort_human, effort_cc:$effort_cc, title:$title, source_finding:$source_finding}' \\
>> "$TASKS_FILE"
~/.claude/skills/gstack/bin/gstack-tasks add --phase ${phase}
\`\`\`
If \`jq\` is not installed, fall back to skipping the JSONL write and warn
the user to install jq for autoplan aggregation. Never hand-roll JSONL.
${conditionalWrites ? 'When writes are permitted and zero tasks were identified, touch the JSONL file' : 'If zero tasks were identified in this review, still touch the JSONL file'}
(\`: > "$TASKS_FILE"\`) so the aggregator sees that the phase produced output
this run (an empty file means "ran, no findings" — distinct from "didn't run").
`;
};
+51 -12
View File
@@ -77,6 +77,11 @@
* bun run scripts/test-free-shards.ts --attribute-home # name real-home writers: each file alone, private HOME
* bun run scripts/test-free-shards.ts --ci-plan plan.json --shards 20
* bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json
* bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json --jobs 4
* # k sub-shards side by side inside one CI job (default 1 = one child, today's behavior;
* # scripts/lib/shard-jobs.ts). --jobs applies only to --ci-run: a positive whole number up
* # to the shard's file count, else FREE_JOBS_INVALID. GSTACK_FREE_JOBS sets the local
* # full-suite process count and never changes --ci-run; --jobs is its only knob.
* bun run scripts/test-free-shards.ts --ci-verify plan.json --results results/
*/
@@ -91,6 +96,10 @@ import { resolveStateRoot } from '../lib/state-root';
import { attributeFreeHomeWriters, guardFreeHome, sharedFreeHome, type FreeHomeGuardFactory } from './lib/free-home-guard';
import { appendStepSummary, ciHealthSummary, SEED_REFRESH_COMMAND, unseededWarning, windowsCurationLine } from './lib/free-ci-health';
import { curateWindowsSafe, type CurationResult } from './lib/windows-curation';
import {
assertSoloSeedSample, ciRunContext, parseJobsFlag, parseJunitCounts, readJunitReport, rejectJobsWithoutCiRun, runCiShard, SLOW_UNDER_LOAD, subShardRecordProblems,
validateJobsForShard, type ShardTestCounts, type SubShardRecord,
} from './lib/shard-jobs';
export { curateWindowsSafe, detectWindowsFragility, KNOWN_WINDOWS_INCOMPATIBLE, type CurationResult } from './lib/windows-curation';
import {
BunTestOutputClassifier,
@@ -447,6 +456,9 @@ export interface FreeCiResult {
revision: string;
outcome: FreeShardOutcome;
retry: FreeShardOutcome | null;
/** Present only at --jobs k > 1: k and one record per sub-shard. */
jobs?: number;
subShards?: SubShardRecord[];
}
/** Preserve the full-suite retry cap across independently running CI jobs. */
@@ -472,6 +484,8 @@ export function verifyFreeCiResults(plan: FreeCiPlan, results: FreeCiResult[]):
|| JSON.stringify(outcome.files) !== JSON.stringify(shard.files)) throw new Error('CI result identity, shard or file coverage mismatch');
seen.add(outcome.shard);
if (!hasCompleteCiSummary(outcome)) throw new Error('Missing or incomplete CI execution summary');
const subShardProblems = subShardRecordProblems(result);
if (subShardProblems.length) throw new Error(`CI shard ${outcome.shard} sub-shards rejected: ${subShardProblems.join('; ')}`);
if (outcome.status === 'passed') {
if (outcome.exitCode !== 0 || outcome.failingFiles.length || outcome.unattributedFailures || result.retry) throw new Error('Inconsistent passing CI result');
} else {
@@ -542,6 +556,8 @@ type CliOptions = {
result: string | null;
results: string | null;
filesFrom: string | null;
/** Set only by --jobs (with --ci-run); absent means k = 1. */
jobs?: number;
};
export function parseCliOptions(argv: string[]): CliOptions {
@@ -556,6 +572,7 @@ export function parseCliOptions(argv: string[]): CliOptions {
let wallTimeoutExplicit = false;
let quick = false;
let attributeHome = false;
let jobs: number | undefined;
const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results' | 'filesFrom', string | null> = {
ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, filesFrom: null,
};
@@ -579,6 +596,7 @@ export function parseCliOptions(argv: string[]): CliOptions {
'--result': pathFlag('--result', 'result'),
'--results': pathFlag('--results', 'results'),
'--files-from': pathFlag('--files-from', 'filesFrom'),
'--jobs': (next) => { jobs = parseJobsFlag(next()); },
'--shards': (next) => {
const value = next();
if (!value) throw new Error('Missing value for --shards');
@@ -602,8 +620,9 @@ export function parseCliOptions(argv: string[]): CliOptions {
if (ciModes > 1 || (ciModes && (quick || listOnly || dryRun || recordDurations || attributeHome))) throw new Error('CI modes cannot be combined with other selection modes');
if (paths.ciRun && (shardIndex === null || !paths.result)) throw new Error('--ci-run requires --shard and --result');
if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results');
if (jobs !== undefined && !paths.ciRun) rejectJobsWithoutCiRun();
if (quick && (recordDurations || windowsOnly || shardIndex !== null)) throw new Error('--quick cannot change recording, Windows or shard selection');
return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths };
return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }) };
}
const describeLines = (lines: string[]) => `${lines.slice(0, 5).join(', ')}${lines.length > 5 ? ` (+${lines.length - 5} more)` : ''}`;
@@ -1075,6 +1094,10 @@ export interface FreeShardOutcome {
unattributedFailures: number;
/** Named failures with their excerpts, so a flaky pass records what failed the first time. */
failures?: FreeRunFailure[];
/** Per-file test and expect() counts from bun's JUnit report (--ci-run only). */
counts?: ShardTestCounts;
/** Wedge suspects at a wall-timeout kill (FreeRunReport.inFlight). */
inFlight?: string[];
}
export interface ShardCommand {
@@ -1087,8 +1110,10 @@ export interface RunFreeShardOptions {
wallTimeoutMs?: number;
rootDir?: string;
env?: NodeJS.ProcessEnv;
/** Override the spawned command. Tests inject fake pass/fail/slow commands. */
commandFor?: (files: string[]) => ShardCommand;
/** Override the spawned command. Tests inject fake pass/fail/slow commands; `junitPath` is set with testCounts. */
commandFor?: (files: string[], junitPath?: string) => ShardCommand;
/** Ask bun for a JUnit report in the shard sandbox and record per-file counts on the outcome (--ci-run). */
testCounts?: boolean;
/** Suppress ALL child output from the console (tests). The classifier and the log file still see every byte. */
quiet?: boolean;
/** Forward the full child stream to the console (legacy firehose). Default: the quiet filtered console. */
@@ -1633,12 +1658,14 @@ export async function runFreeShard(
const shardLog = openShardLog(logPath, label, 0o600);
log(`[test:free] full log: ${logPath}`);
const { command, args } = options.commandFor
? options.commandFor(files)
: { command: process.execPath, args: buildShardArgs(files, { rootDir }) };
// realpath: the browser tracker refuses a state dir whose path resolves elsewhere.
const { stateDir, env } = createShardSandbox('gstack-free-shard-', options.env ?? process.env, { realpath: true });
// The JUnit reporter leaves the console stream byte-identical (Bun 1.4.2); it only adds per-file counts.
const junitPath = options.testCounts ? path.join(stateDir, 'junit.xml') : undefined;
const { command, args } = options.commandFor
? options.commandFor(files, junitPath)
: { command: process.execPath, args: [...buildShardArgs(files, { rootDir }), ...(junitPath ? ['--reporter=junit', `--reporter-outfile=${junitPath}`] : [])] };
let junitXml = null as string | null;
// CLI renders otherwise share the repo's .gstack/browse.json, where concurrent
// shards and prior daemons replace each other's state; override inherited state.
env.BROWSE_STATE_FILE = path.join(stateDir, '.gstack', 'browse.json');
@@ -1703,6 +1730,7 @@ export async function runFreeShard(
shardLog.write(diagnostic + '\n');
}
await new Promise<void>((resolve) => shardLog.stream.end(() => resolve()));
if (junitPath) junitXml = readJunitReport(junitPath);
try {
if (!cleanupError) fs.rmSync(stateDir, { recursive: true, force: true });
} catch {
@@ -1745,6 +1773,8 @@ export async function runFreeShard(
shard: shardNumber, files, status, exitCode, elapsedMs: Date.now() - startedAt, groupPid, failingFiles, unattributedFailures,
...(status === 'passed' ? {} : { failures: report.failures }),
summary: { testsRan: report.testsRan, filesRan: report.filesRan, sawTerminalSummary: report.sawTerminalSummary },
...(junitPath ? { counts: parseJunitCounts(junitXml, files, report.testsRan, rootDir) } : {}),
...(status === 'timed-out' && report.inFlight.length ? { inFlight: report.inFlight } : {}),
};
log(shardEpilogue(outcome, totalShards));
for (const line of buildRunEpilogue(status, report, outcome.elapsedMs, logPath)) log(line);
@@ -1799,6 +1829,7 @@ async function recordFreeTestDurations(files: string[], jobs: number, wallTimeou
const outcome = await runFreeShard([file], index + 1, files.length, {
wallTimeoutMs, quiet: true, homeGuard: sharedFreeHome,
});
assertSoloSeedSample(file, outcome);
durations[normalizeRelativePath(file)] = outcome.elapsedMs;
if (outcome.status !== 'passed') failed.push(file);
});
@@ -1944,14 +1975,22 @@ async function main(): Promise<number> {
}
const shard = plan.shards[options.shardIndex! - 1];
if (!shard || shard.shard !== options.shardIndex) throw new Error('CI shard index is outside the plan');
const outcome = await runFreeShard(shard.files, shard.shard, plan.shards.length, {
const jobs = options.jobs ?? 1;
validateJobsForShard(jobs, shard.files.length, shard.shard);
const durations = loadFreeTestDurations() ?? {};
const { result, exitCode } = await runCiShard({
plan, shard, revision, jobs, verbose: options.verbose, context: ciRunContext(options.ciRun!),
wallTimeoutMs: options.wallTimeoutExplicit ? options.wallTimeoutMs
: wallTimeoutForPackedShard(shard.predictedMs, options.wallTimeoutMs, shard.files.length),
verbose: options.verbose,
}, {
runShard: (files, shardNumber, totalShards, runOptions) => runFreeShard(files, shardNumber, totalShards, runOptions),
retry: outcomes => retryFailedFreeFiles(outcomes, plan.shards.length, options),
pack: (files, count) => packShardsByDuration(files, count, durations).shards,
isSerial: file => file in TREE_MUTATING || file in SLOW_UNDER_LOAD,
terminationRequested: isTerminationRequested,
});
const retried = await retryFailedFreeFiles([outcome], plan.shards.length, options);
writeJson(options.result!, { planId: plan.id, revision, outcome, retry: retried.retry } satisfies FreeCiResult);
return retried.exitCode;
writeJson(options.result!, result);
return exitCode;
}
let files = allFiles;
+14 -8
View File
@@ -128,12 +128,18 @@ export function flakeCheck(window: FlakeWindow | Error): Check {
fix: 'Fix the race at source (docs/TESTING_INTERNALS.md#flake-ledger); never raise the timeout or add a retry. Reproduce with `bun test <file>` under load.' };
if (window instanceof Error) return { ...base, status: 'unavailable', detail: `unavailable: ${unavailableReason(window)}` };
const failing = window.files.filter(file => file.failing);
const left = flakeWindowNotes(window);
if (!window.enforceable) {
return { ...base, status: 'pass', detail: `report-only: ${window.runs} main run(s) since ${window.since.slice(0, 10)} (< 20 needed to enforce)` };
return { ...base, status: 'pass', detail: `report-only: ${window.runs} main run(s) since ${window.since.slice(0, 10)} (< 20 needed to enforce)${left}` };
}
if (failing.length === 0) return { ...base, status: 'pass', detail: `no file above 5% over ${window.runs} main runs` };
if (failing.length === 0) return { ...base, status: 'pass', detail: `no file above 5% over ${window.runs} main runs${left}` };
return { ...base, status: 'fail',
detail: failing.map(file => `${file.file}: flaky in ${file.flakyRuns}/${window.runs} main runs (${pct(file.rate)} > 5%)`).join('; ') };
detail: failing.map(file => `${file.file}: flaky in ${file.flakyRuns}/${window.runs} main runs (${pct(file.rate)} > 5%)`).join('; ') + left };
}
/** The reused main pushes a flake window left out and the runs it could not check, or ''. */
function flakeWindowNotes(window: FlakeWindow): string {
return `${window.skipped ? `; ${window.skipped} reused main push(es) left out` : ''}${window.note ? `; ${window.note}` : ''}`;
}
export function unseededCheck(root: string): Check {
@@ -286,19 +292,19 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
metrics.push(...censusMetrics(inputs, artifactDirs));
const flakes = attempt(() => readCiFlakeLedgers({ client, repo, sinceDays, now, cacheDir }));
metrics.push(metric('free-flaky-pass-rate', 'Free-suite flaky-pass rate', 'share of main free-tests runs (success or failure) with at least one flaky-pass ledger entry; per-file rate = runs with that file / runs', () => {
metrics.push(metric('free-flaky-pass-rate', 'Free-suite flaky-pass rate', 'share of main free-tests runs (success or failure) with at least one flaky-pass ledger entry; per-file rate = runs with that file / runs; a main push that reused its pull request\'s run (no artifacts, `reuse` job green) is left out', () => {
const window = need(flakes);
return { value: `${pct(window.runs ? window.flakyRuns / window.runs : 0)} (${window.flakyRuns}/${window.runs} main runs since ${window.since.slice(0, 10)} needed the retry pass; ${window.files.length} file(s))`,
return { value: `${pct(window.runs ? window.flakyRuns / window.runs : 0)} (${window.flakyRuns}/${window.runs} main runs since ${window.since.slice(0, 10)} needed the retry pass; ${window.files.length} file(s)${flakeWindowNotes(window)})`,
detail: window.files.slice(0, 15).map(file => `${file.file}: ${file.flakyRuns}/${window.runs} runs (${pct(file.rate)})${file.failing ? ' FAIL > 5%' : ''}`) };
}));
metrics.push(metric('free-wall', 'Free-suite and Windows wall', 'p50 of run_started_at to updated_at for successful main runs of free-tests.yml and successful pull_request runs of windows-free-tests.yml (its workflow_dispatch duration recordings excluded)', () => {
metrics.push(metric('free-wall', 'Free-suite and Windows wall', 'p50 of run_started_at to updated_at for successful main runs of free-tests.yml and successful pull_request runs of windows-smoke.yml, the required Windows check (its workflow_dispatch runs excluded)', () => {
const walls = (workflow: string, filter: { branch?: string; event?: string }) =>
listWorkflowRuns(client, repo, workflow, { since, ...filter }).filter(run => run.conclusion === 'success').map(run => span(run.startedAt, run.updatedAt)!);
const free = walls('free-tests.yml', { branch: 'main' });
const windows = walls('windows-free-tests.yml', { event: 'pull_request' });
const windows = walls('windows-smoke.yml', { event: 'pull_request' });
const windowsValue = windows.length ? `${minutes(percentile(windows, 0.5))} (${windows.length} runs)` : 'no successful PR runs in window';
return { value: `free-tests ${minutes(percentile(free, 0.5))} (${free.length} runs); windows-free-tests (successful PR runs) ${windowsValue}` };
return { value: `free-tests ${minutes(percentile(free, 0.5))} (${free.length} runs); windows-smoke (successful PR runs) ${windowsValue}` };
}));
metrics.push(metric('free-serial', 'Free-suite serial time', 'sum of scripts/free-test-durations.json (the committed seed)', () => {
+2
View File
@@ -237,6 +237,7 @@ export const PAID_WORKFLOW_FILES = [
export const FREE_ONLY_PR_FILES = [
'scripts/test-free-shards.ts',
'scripts/lib/free-home-guard.ts', // Imported only by the free shard runner.
'scripts/lib/shard-jobs.ts', // Free runner's --ci-run --jobs sub-shards; ship-measure loads it through test-free-shards.ts, so its touchfiles list it.
'test/helpers/auq-parallel-worker.ts',
'test/helpers/stored-zip.ts', // Read only by the free test-health report tests.
'scripts/free-test-durations.json',
@@ -294,6 +295,7 @@ export const FREE_ONLY_PR_FILES = [
'.github/workflows/windows-free-tests.yml',
'.github/workflows/windows-pregate-e2e.yml',
'.github/workflows/windows-setup-e2e.yml',
'.github/workflows/windows-smoke.yml',
] as const;
/**
+1 -1
View File
@@ -241,6 +241,7 @@
"test/helpers/qa-functional-evidence.ts\tTS7006\tParameter 'row' implicitly has an 'any' type.": 2,
"test/helpers/session-runner.ts\tTS2352\tConversion of type 'ReadableStream<any>' to type 'ReadableStream<Uint8Array<ArrayBufferLike>>' may be a mistake because neither type sufficiently overlaps with the other. If this was intentional, convert the expression to 'unknown' first. Types of property 'getReader' are incompatible. Type '{ (): ReadableStreamDefaultReader<any>; (options: { mode: \"byob\"; }): ReadableStreamBYOBReader; (options?: ReadableStreamGetReaderOptions | undefined): ReadableStreamReader<...>; }' is not comparable to type '{ (): ReadableStreamDefaultReader<Uint8Array<ArrayBufferLike>>; (options: { mode: \"byob\"; }): ReadableStreamBYOBReader; (options?: ReadableStreamGetReaderOptions | undefined): ReadableStreamReader<...>; }'. Target signature provides too few arguments. Expected 1 or more, but got 0.": 1,
"test/helpers/setup-gbrain-sandbox.ts\tTS2345\tArgument of type '(event: any) => { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }[] | { message: { content: any; id: any; role: any; }; parent_tool_use_id: any; session_id: any; type: any; }[]' is not assignable to parameter of type '(this: undefined, value: unknown, index: number, array: unknown[]) => readonly { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }[] | { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }'. Type '{ claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }[] | { message: { content: any; id: any; role: any; }; parent_tool_use_id: any; session_id: any; type: any; }[]' is not assignable to type 'readonly { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }[] | { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }'. Type '{ message: { content: any; id: any; role: any; }; parent_tool_use_id: any; session_id: any; type: any; }[]' is not assignable to type 'readonly { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }[] | { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }'. Type '{ message: { content: any; id: any; role: any; }; parent_tool_use_id: any; session_id: any; type: any; }[]' is not assignable to type 'readonly { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }[]'. Type '{ message: { content: any; id: any; role: any; }; parent_tool_use_id: any; session_id: any; type: any; }' is missing the following properties from type '{ claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }': subtype, cwd, model, tools, claude_code_version": 1,
"test/helpers/shared-libs-checker-evidence.ts\tTS18048\t'current' is possibly 'undefined'.": 1,
"test/helpers/shared-libs-eval-fixture.ts\tTS7006\tParameter 'candidate' implicitly has an 'any' type.": 2,
"test/helpers/shared-libs-path-fixture.ts\tTS2352\tConversion of type '{ env: { GSTACK_HOME: string; }; repo: string; root: string; state: string; }' to type 'SharedLibsFixture' may be a mistake because neither type sufficiently overlaps with the other. If this was intentional, convert the expression to 'unknown' first. Type '{ env: { GSTACK_HOME: string; }; repo: string; root: string; state: string; }' is missing the following properties from type 'SharedLibsFixture': bin, trace, hookTrace, tip": 1,
"test/helpers/shared-libs-plan-actor.ts\tTS18046\t'questions' is of type 'unknown'.": 1,
@@ -431,7 +432,6 @@
"test/setup-gbrain-path4-caller.test.ts\tTS2339\tProperty 'unregistered' does not exist on type '{ 'no-init': number; 'no-install': number; 'no-registration': number; 'no-verifier': number; }'.": 1,
"test/setup-gbrain-path4-caller.test.ts\tTS2339\tProperty 'wrong-engine' does not exist on type '{ 'no-init': number; 'no-install': number; 'no-registration': number; 'no-verifier': number; }'.": 1,
"test/setup-gbrain-remote-caller.test.ts\tTS18048\t'lateDecision' is possibly 'undefined'.": 1,
"test/shared-libs-checker-interface-evidence.test.ts\tTS18048\t'current' is possibly 'undefined'.": 1,
"test/shared-libs-fixture.test.ts\tTS18046\t'packet' is of type 'unknown'.": 4,
"test/shared-libs-fixture.test.ts\tTS2345\tArgument of type '(options: { label: string; } | { description: string; label: string; } | { description: string; label: string; }) => Promise<void>' is not assignable to parameter of type '(...args: [{ label: string; } | { description: string; label: string; } | { description: string; label: string; }, done: (err?: unknown) => void] | [{ label: string; } | { description: string; label: string; } | { ...; }, { ...; } | undefined, done: (err?: unknown) => void]) => void | Promise<...>'. Types of parameters 'options' and 'args' are incompatible. Type '[{ label: string; } | { description: string; label: string; } | { description: string; label: string; }, done: (err?: unknown) => void] | [{ label: string; } | { description: string; label: string; } | { description: string; label: string; }, { ...; } | undefined, done: (err?: unknown) => void]' is not assignable to type '[options: { label: string; } | { description: string; label: string; } | { description: string; label: string; }]'. Type '[{ label: string; } | { description: string; label: string; } | { description: string; label: string; }, done: (err?: unknown) => void]' is not assignable to type '[options: { label: string; } | { description: string; label: string; } | { description: string; label: string; }]'. Source has 2 element(s) but target allows only 1.": 1,
"test/shared-libs-fixture.test.ts\tTS2345\tArgument of type '(options: { label: string; } | { description: string; label: string; } | { label: string; preview: string; } | { description: string; label: string; preview: string; } | { label: string; } | { description: string; label: string; } | { ...; }) => Promise<...>' is not assignable to parameter of type '(...args: [{ label: string; } | { description: string; label: string; } | { label: string; preview: string; } | { description: string; label: string; preview: string; } | { label: string; } | { description: string; label: string; } | { ...; }, done: (err?: unknown) => void] | [...]) => void | Promise<...>'. Types of parameters 'options' and 'args' are incompatible. Type '[{ label: string; } | { description: string; label: string; } | { label: string; preview: string; } | { description: string; label: string; preview: string; } | { label: string; } | { description: string; label: string; } | { ...; }, done: (err?: unknown) => void] | [...]' is not assignable to type '[options: { label: string; } | { description: string; label: string; } | { label: string; preview: string; } | { description: string; label: string; preview: string; } | { label: string; } | { description: string; label: string; } | { ...; }]'. Type '[{ label: string; } | { description: string; label: string; } | { label: string; preview: string; } | { description: string; label: string; preview: string; } | { label: string; } | { description: string; label: string; } | { ...; }, done: (err?: unknown) => void]' is not assignable to type '[options: { label: string; } | { description: string; label: string; } | { label: string; preview: string; } | { description: string; label: string; preview: string; } | { label: string; } | { description: string; label: string; } | { ...; }]'. Source has 2 element(s) but target allows only 1.": 1,
+36 -5
View File
@@ -27,10 +27,18 @@
# teardown) times the size's price. Without a key it runs the same files
# locally in parallel shards and reports spent=0.
#
# Reattach: the tree and the planned set's digest go into ubi-runner's run key
# (--key-input), so a rerun after the local runner died (a machine stop, a
# SIGKILL) reattaches to the VM still running this gate instead of paying for a
# second one. Its minutes since created_at count against the cap: the wall
# budget shrinks by them (PREGATE_REMOTE_CAP_SPENT when none is left), and
# spent= bills from the original create.
#
# Exit codes: 0 pass (or a paid-only selection, tests=0); 1 test failures (or
# a rejected file list); 2 missing input; 3 tree or plan mismatch (or the tree
# moved during the run); 4 infrastructure failure (create, setup, SSH, stop
# deadline); 5 the spend cap or pregate's deadline refuses the run. Every refusal line starts with a
# deadline, another gate on this machine owning the VM to reattach to); 5 the
# spend cap or pregate's deadline refuses the run. Every refusal line starts with a
# PREGATE_REMOTE_* reason code (docs/pregate.md#remote-gate). Nothing is
# uploaded or spent before the local tree check.
set -euo pipefail
@@ -147,8 +155,6 @@ if [ -n "$outer_s" ]; then
fi
if [ "$outer_stop" -lt "$stop_after" ]; then stop_after=$outer_stop stop_why="$outer_from ${outer_s}s minus 120 s teardown reserve and 15 s margin"; fi
fi
say "$count files on one $size VM; worst case \$$(usd "$wall_min") under the cap \$$cap; the VM is told to stop after ${stop_after}s ($stop_why)"
digest=$(tr -d '\r' <"$files" | grep . | LC_ALL=C sort | sha256sum | cut -d' ' -f1)
prelude="$logs/vm-setup.sh"
{
@@ -183,6 +189,28 @@ args=(run -s "$size" --src "$root" --setup "$prelude"
--pull "/tmp/gstack-free-test-*:$logs"
--pull "work/$(basename "$root")/.context/free-test-logs:$logs")
[ -z "${GSTACK_FREE_JOBS:-}" ] || args+=(--env "GSTACK_FREE_JOBS=$GSTACK_FREE_JOBS")
args+=(--key-input "pregate-remote tree=$tree plan=$digest")
# A rerun of this gate for the same tree and planned set has the same
# ubi-runner run key, so it reattaches to the VM a killed run left behind
# (`ubi-runner: reattached to <vm> created_at=<epoch>`). The minutes that VM
# has billed since created_at come off the cap's wall budget first.
prior_vm=""
attach=$(bash "$here/ubi-runner.sh" "${args[@]}" --probe -- "bash ~/.pregate-run.sh" 2>/dev/null \
| sed -n 's/^UBI_ATTACH_INFO: vm=\([^ ]*\) created_at=\([0-9]*\) key=[0-9a-f]* busy=\([01]\)$/\1 \2 \3/p' | tail -1) || attach=""
if [ -n "$attach" ]; then
read -r prior_vm prior_created prior_busy <<<"$attach"
[ "$prior_busy" = 0 ] || stop 4 PREGATE_REMOTE_ATTACH_BUSY "another gate on this machine still runs VM $prior_vm for this tree and planned set" "wait for it; this run created nothing and spent nothing"
prior_min=$(( ($(date +%s) - prior_created + 59) / 60 ))
left_s=$(( (wall_min - 3 - prior_min) * 60 ))
if [ "$left_s" -le 0 ]; then
stop 5 PREGATE_REMOTE_CAP_SPENT "VM $prior_vm, created ${prior_min} min ago for this tree and planned set, has already used the cap's $wall_min min wall (\$$(usd "$prior_min") of \$$cap)" \
"nothing reattached; destroy it: bash scripts/ubicloud/ubi-runner.sh down $prior_vm" "or rerun with a higher --spend-cap to let it finish: $rerun"
fi
if [ "$left_s" -lt "$stop_after" ]; then stop_after=$left_s stop_why="the cap's $wall_min min wall minus 3 min for teardown and $prior_min min already billed since $prior_vm was created"; fi
say "reattaching to VM $prior_vm, created ${prior_min} min ago by an earlier run of this gate for the same tree and planned set"
fi
say "$count files on one $size VM; worst case \$$(usd "$wall_min") under the cap \$$cap; the VM is told to stop after ${stop_after}s ($stop_why)"
# The VM run gets its own session (python3, which ubi-runner already needs, for
# setsid on Linux and macOS): pregate's --timeout kill signals this
@@ -220,7 +248,7 @@ while :; do
rm -f "$log.pid" "$log.timedout" "$log.left"
t0=$(date +%s)
vm_name="ubirun-$(bash "$here/ubi-runner.sh" owner)-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')-t$(( (stop_after + 900 + 3599) / 3600 ))"
say "VM $vm_name; if this run is killed before its teardown line, destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name"
[ -n "$prior_vm" ] || say "VM $vm_name; if this run is killed before its teardown line, destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name"
UBI_LEFT_FILE="$log.left" python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
bash -c "$vm_session" pregate-remote-vm "$stop_after" "$log" "$stream" "$here/ubi-runner.sh" "${args[@]}" -n "$vm_name" &
vmrun=$!
@@ -236,6 +264,9 @@ while :; do
tailer=""
t_end=$(date +%s)
t_create=$(at 'ubi-runner: creating ')
# A reattached VM is billed from its original create.
reattached=$(sed -n 's/^[0-9]* ubi-runner: reattached to [^ ]* created_at=\([0-9][0-9]*\) .*/\1/p' "$log" | tail -1)
[ -z "$reattached" ] || t_create=$reattached
if [ -n "$t_create" ] && ! grep -q 'ubi-runner: create failed' "$log"; then minutes=$(( (t_end - t_create + 59) / 60 )); billed=$(( billed + (minutes > 0 ? minutes : 1) )); fi
if [ "$rc" -ne 0 ] && [ "$attempt" -eq 1 ] && grep -q 'vCPU quota exhausted' "$log"; then
say "vCPU quota refused the VM; one retry in ${PREGATE_REMOTE_QUOTA_WAIT_S:-60} s"
@@ -279,7 +310,7 @@ if [ -n "$result" ] && [ "$rc" -ne 124 ]; then
echo "PREGATE_REMOTE: tree=$vm_tree tests=$(field tests "$result") failed=$(field failed "$result") spent=$spent"
exit $(( rc == 0 ? 0 : 1 ))
fi
vm=$(sed -n 's/^[0-9]* ubi-runner: creating \([^ ]*\) .*/\1/p' "$log" | tail -1)
vm=$(sed -n -e 's/^[0-9]* ubi-runner: creating \([^ ]*\) .*/\1/p' -e 's/^[0-9]* ubi-runner: reattached to \([^ ]*\) .*/\1/p' "$log" | tail -1)
down=$(grep -E 'ubi-runner: (destroyed|gone:|WARNING|FAILED to destroy)' "$log" | tail -1 | cut -d' ' -f2- || true)
cause=$(grep -E 'ubi-runner: ' "$log" | grep -vE 'ubi-runner: (destroyed|gone:|destroy requested|waiting for an in-flight)' | tail -1 | cut -d' ' -f2- || true)
[ "$rc" -ne 124 ] || cause="the VM was told to stop after ${stop_after}s ($stop_why) and torn down"
+697 -67
View File
@@ -33,8 +33,9 @@
# Teardown: `up` records each VM's state (location, create status) before it
# sends the create request, and `down` destroys and polls until the VM is
# confirmed gone or provably never existed, waiting out a create that may still
# be in flight. An interrupted `up` or `run` (EXIT, INT, TERM, HUP, QUIT) runs
# that `down`; further signals are ignored until it finishes. A create refused
# be in flight. An interrupted `up` or `run` (EXIT, INT, TERM, QUIT, and HUP
# until a remote phase is live) runs that `down`; further signals are ignored
# until it finishes. A create refused
# for vCPU quota prints the project's usage per owner.
#
# The owner-age sweep is off by default (the TTL reap below always runs). With UBI_GC_HOURS set to a positive
@@ -43,12 +44,13 @@
#
# TTL and reaper: a name this script generates ends in -t<hours> (UBI_TTL_HOURS,
# 1-999, default 3), the creator's own declaration of how long the VM may live.
# `reap` destroys every VM in the project whose TTL expired more than ten
# minutes ago, whichever owner made it, and this owner's own ubirun-<owner>-*
# names with no TTL once they are UBI_LEGACY_HOURS old (default 4; another
# owner's or an untagged TTL-less VM is never reaped); every `up` runs it first (UBI_REAP=0 skips it), so a runner
# killed by a sleeping machine leaves nothing behind for longer than its TTL.
# Names outside the ubirun- prefix are never reaped.
# `reap` destroys only this owner's VMs: its ubirun-<owner>-* names whose TTL
# expired more than ten minutes ago, and those with no TTL once they are
# UBI_LEGACY_HOURS old (default 4). Another owner's VM, an untagged one and
# names outside the ubirun- prefix are never reaped, expired or not; reap only
# counts other owners' expired VMs in its log (cloud deletes stay owner-scoped,
# GST-573). Every `up` runs it first (UBI_REAP=0 skips it), so a lane killed by
# a sleeping machine has its leftovers destroyed by its own next `up`.
#
# Leftover line: `run` and `verify` end with
# UBICLOUD: left=<0|1|unknown> owner=<owner> vm=<name> others=<n>
@@ -69,6 +71,33 @@
# UBI_RUN_TIMEOUT seconds (0, unlimited), exiting 124 with the last 40 lines,
# so a hung verify fails fast instead of holding the VM until its TTL.
#
# Detached phases and reattach: `run` and `verify` start their setup, command
# and tier phases on the VM detached from the ssh connection (setsid; stdout to
# ~/.ubirun-<phase>.out, stderr to .err, the session leader's pid in .pid, and
# the exit status in .exit, written by rename once the streams are closed) and
# stream both files back. Before creating a VM they compute a run key locally
# (the checkout's content tree, the command, the setup script's content, every
# --env entry, each --pass name with a sha256 of its value, size, location,
# image, storage, origin/HEAD, tiers and this script's own sha256) and look for
# a run record (<state dir>/<vm>/run) with the same key whose VM still exists:
# its runner alive (lease pid and start time) refuses with UBI_ATTACH_BUSY,
# otherwise the run reattaches, printing `reattached to <vm> created_at=<epoch>
# key=<k12>`, and continues from the first phase without an exit file. After
# unpack the VM recomputes the tree and refuses a mismatch (UBI_VM_TREE_MISMATCH).
# A leader gone with no exit file exits 125 (UBI_ATTACH_LOST). Signals: INT
# and TERM tear the VM down as before; HUP once a remote phase is live detaches
# (UBI_DETACHED, exit 129) and leaves the VM under its TTL; SIGKILL or a machine
# stop leaves the record and the VM, and the next run of the same command
# reattaches. `attach NAME` resumes a recorded run explicitly. Reattach never
# extends a TTL, and the reaper still applies.
#
# Tier checkpoints: a .gstack/ubicloud.json with `tiers` runs each tier as its
# own detached phase after setup, and records start/pass/fail per tier under
# UBI_CHECKPOINT_DIR/<run key>/ (write-once JSON plus the tier's log, published
# under a lock with a monotonic generation). A rerun reuses a pass younger than
# UBI_CHECKPOINT_MAX_AGE whose log re-hashes, so a resume on a fresh VM reruns
# only setup and the tiers without a pass.
#
# `verify` is the one entry point for a repository's full remote verify: it
# reads .gstack/ubicloud.json (setup, command, size, location, env, pass, pull)
# and runs it through `run`, so lanes share one committed definition instead of
@@ -100,6 +129,11 @@ PREFIX="ubirun"
if [ -n "${UBI_VM_RECORD:-}" ]; then VM_RECORD="$UBI_VM_RECORD"
elif [ -d "$HOME/.capy/work" ]; then VM_RECORD="$HOME/.capy/work/ubi-runner/vms.tsv"
else VM_RECORD="$STATE_ROOT/vms.tsv"; fi
if [ -n "${UBI_CHECKPOINT_DIR:-}" ]; then CHECKPOINT_DIR="$UBI_CHECKPOINT_DIR"
elif [ -d "$HOME/.capy/work" ]; then CHECKPOINT_DIR="$HOME/.capy/work/ubi-runner/checkpoints"
else CHECKPOINT_DIR="$STATE_ROOT/checkpoints"; fi
CHECKPOINT_MAX_AGE="${UBI_CHECKPOINT_MAX_AGE:-86400}"
REATTACH_MARGIN="${UBI_REATTACH_MARGIN:-900}"
die() { echo "ubi-runner: $*" >&2; exit 1; }
log() { echo "ubi-runner: $*" >&2; }
@@ -109,7 +143,7 @@ for bin in curl python3 ssh ssh-keygen tar; do
done
UBICLOUD_API_KEY="${UBICLOUD_API_KEY:-${UBICLOUD_API_TOKEN:-}}"
[[ "$GC_HOURS" =~ ^[0-9]+$ ]] || die "UBI_GC_HOURS must be a whole number of hours (unset or 0 disables the sweep)"
for v in IDLE_TIMEOUT RUN_TIMEOUT HEARTBEAT QUOTA_WAIT; do
for v in IDLE_TIMEOUT RUN_TIMEOUT HEARTBEAT QUOTA_WAIT CHECKPOINT_MAX_AGE REATTACH_MARGIN; do
[[ "${!v}" =~ ^[0-9]+$ ]] || die "UBI_$v must be a whole number of seconds (0 disables it)"
done
[ -z "$QUOTA_VCPUS" ] || [[ "$QUOTA_VCPUS" =~ ^[0-9]+$ ]] || die "UBI_QUOTA_VCPUS must be a whole number of vCPUs"
@@ -296,6 +330,410 @@ watched() {
return "$rc"
}
proc_start() { ps -o lstart= -p "$1" 2>/dev/null | tr -s ' ' || true; }
fsize() { if [ -f "$1" ]; then wc -c <"$1" | tr -d ' '; else echo 0; fi; }
sha256() { if command -v sha256sum >/dev/null 2>&1; then sha256sum | cut -d' ' -f1; else shasum -a 256 | cut -d' ' -f1; fi; }
# TREE_SH: print the content tree of the current directory, the same way here
# and on the VM: at a checkout's root, `git add -A` into a copy of its index
# (tracked plus untracked files that are not ignored, the files pack uploads);
# elsewhere, the same into a scratch repository over the directory.
TREE_SH='idx=$(mktemp -d "${TMPDIR:-/tmp}/ubirun-tree.XXXXXX") || exit 1
if [ "$(git rev-parse --show-toplevel 2>/dev/null)" = "$(pwd -P)" ]; then
gi=$(git rev-parse --git-path index)
if [ "$(git config --bool core.splitIndex)" != true ] && [ -f "$gi" ]; then cp "$gi" "$idx/index"; fi
else
git init -q --bare "$idx/git" && export GIT_DIR="$idx/git" GIT_WORK_TREE=.
fi
GIT_INDEX_FILE="$idx/index" git add -A >/dev/null 2>&1 && GIT_INDEX_FILE="$idx/index" git write-tree
rc=$?; rm -rf "$idx"; exit $rc'
# vm_tree NAME QDEST: the content tree of QDEST on the VM.
vm_tree() {
remote "$1" "command -v git >/dev/null || { sudo apt-get update -qq && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git; } >/dev/null 2>&1; cd $2 && bash -c $(sq "$TREE_SH")" </dev/null
}
base_sha() { git -C "$1" rev-parse -q --verify 'refs/remotes/origin/HEAD^{commit}' 2>/dev/null || echo none; }
# phase_status NAME PHASE: "exit <rc>" once the phase wrote its exit file,
# "alive" while its session leader runs, "lost" when the leader is gone with no
# exit file, "none" when it never started.
phase_status() {
local p="~/.ubirun-$2"
remote "$1" "if [ -f $p.exit ]; then echo exit \$(cat $p.exit); elif [ -s $p.pid ] && kill -0 \$(cat $p.pid) 2>/dev/null; then echo alive; elif [ -s $p.pid ]; then echo lost; else echo none; fi" </dev/null
}
# phase_start NAME PHASE INNER: start INNER on the VM in its own session,
# detached from this ssh connection. The leader records its pid, runs INNER with
# stdout and stderr in files, and writes the exit status by rename only after
# INNER's streams are closed.
phase_start() {
local p="~/.ubirun-$2" inner
inner=$(sq "echo \$\$ >$p.pid.tmp && mv $p.pid.tmp $p.pid; ( $3 ) >$p.out 2>$p.err </dev/null; echo \$? >$p.exit.tmp && mv $p.exit.tmp $p.exit")
remote "$1" "rm -f $p.exit $p.pid; : >$p.out; : >$p.err; if command -v setsid >/dev/null 2>&1; then setsid -f bash -c $inner </dev/null >/dev/null 2>&1; else nohup bash -c $inner </dev/null >/dev/null 2>&1 & fi" </dev/null >/dev/null
}
# stream_cmd PHASE OUT_OFFSET ERR_OFFSET: follow both output files from the
# given byte offsets until the session leader exits.
stream_cmd() {
local p="~/.ubirun-$1"
printf '%s' "i=0; while [ ! -s $p.pid ] && [ ! -f $p.exit ] && [ \$i -lt 600 ]; do sleep 0.1; i=\$((i + 1)); done; pid=\$(cat $p.pid 2>/dev/null); f=; [ -z \"\$pid\" ] || f=\"-f --pid=\$pid\"; tail -c +$(( $3 + 1 )) \$f $p.err >&2 & tail -c +$(( $2 + 1 )) \$f $p.out; wait"
}
PHASE_LIVE="" STREAM_PID="" RUN_KEY="" TREE=""
# on_hup: once a remote phase is live, HUP detaches: the VM keeps running under
# its TTL and the next run of the same command reattaches. Before that, HUP
# tears down like INT and TERM.
on_hup() {
[ -n "$PHASE_LIVE" ] || exit 130
[ -z "$STREAM_PID" ] || kill -TERM -- "-$STREAM_PID" 2>/dev/null || true
trap - EXIT
rm -f "$(state_dir "$RUN_VM")/lease"
leftover "left=1 owner=$OWNER vm=$RUN_VM detached=1"
echo "UBI_DETACHED: vm=$RUN_VM key=${RUN_KEY:0:12} reattach: rerun the same command" >&2
exit 129
}
# detached NAME PHASE INNER: run INNER as a detached phase (or reattach to it),
# streaming its output to this process and to the VM's state dir
# (<phase>.out, <phase>.err and the combined <phase>.log) under the same
# heartbeat and watchdog as watched. RUN_TIMEOUT counts from the phase's
# recorded start, not from a reattach. Returns the phase's exit status, 124
# after a watchdog kill, 125 when the session is gone without an exit status.
detached() {
local name=$1 phase=$2 inner=$3 dir st rc=0 tries=0 start now idle elapsed last_beat why old_traps spid sst oo oe
load "$name"
dir=$(state_dir "$name")
local lout="$dir/$phase.out" lerr="$dir/$phase.err" out="$dir/$phase.log" p="~/.ubirun-$phase"
st=none
if [ -f "$dir/phase.$phase.start" ]; then
st=$(phase_status "$name" "$phase") || st=""
fi
case $st in
none)
: >"$lout"; : >"$lerr"; : >"$out"
date +%s >"$dir/phase.$phase.start"
phase_start "$name" "$phase" "$inner" || { log "could not start $phase on $name"; return 255; } ;;
alive) log "$phase is still running on $name; following it" ;;
"exit "*) ;;
lost) log "UBI_ATTACH_LOST: vm=$name phase=$phase: its session is gone and left no exit status ($p.exit)"; return 125 ;;
*) log "could not read the state of $phase on $name"; return 255 ;;
esac
PHASE_LIVE=1
start=$(cat "$dir/phase.$phase.start")
# A runner killed on this machine leaves its stream client behind; it would
# append to the same local copies.
if [ -f "$dir/stream" ] && read -r spid sst <"$dir/stream" && [ -n "$spid" ] && [ "$(proc_start "$spid")" = "$sst" ]; then
kill -TERM -- "-$spid" 2>/dev/null || true
fi
oo=$(fsize "$lout") oe=$(fsize "$lerr")
if [ "$st" != none ]; then head -c "$oo" "$lout"; head -c "$oe" "$lerr" >&2; fi
old_traps=$(trap -p INT TERM)
trap '[ -z "$STREAM_PID" ] || kill -TERM -- "-$STREAM_PID" 2>/dev/null; exit 130' INT TERM
last_beat=$(date +%s)
while :; do
touch "$out"
# shellcheck disable=SC2046
( exec python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
ssh $(ssh_opts) "ubi@$IP" "umask 022; $(stream_cmd "$phase" "$oo" "$oe")" ) \
</dev/null > >(tee -a "$lout" | tee -a "$out") 2> >(tee -a "$lerr" | tee -a "$out" >&2) &
STREAM_PID=$!
echo "$STREAM_PID $(proc_start "$STREAM_PID")" >"$dir/stream"
why=""
while :; do
sleep 1
kill -0 "$STREAM_PID" 2>/dev/null || break
now=$(date +%s)
elapsed=$(( now - start ))
idle=$(( now - $(mtime "$out") ))
if [ "$IDLE_TIMEOUT" -gt 0 ] && [ "$idle" -ge "$IDLE_TIMEOUT" ]; then
why="printed nothing for ${idle}s (UBI_IDLE_TIMEOUT=$IDLE_TIMEOUT)"
elif [ "$RUN_TIMEOUT" -gt 0 ] && [ "$elapsed" -ge "$RUN_TIMEOUT" ]; then
why="ran ${elapsed}s (UBI_RUN_TIMEOUT=$RUN_TIMEOUT)"
fi
[ -z "$why" ] || break
if [ "$HEARTBEAT" -gt 0 ] && [ $(( now - last_beat )) -ge "$HEARTBEAT" ]; then
log "heartbeat: $phase ${elapsed}s elapsed, last output ${idle}s ago"
last_beat=$now
fi
done
if [ -n "$why" ]; then
log "watchdog: $phase $why; killing it. Last output:"
tail -n 40 "$out" >&2
kill -TERM -- "-$STREAM_PID" 2>/dev/null || kill "$STREAM_PID" 2>/dev/null || true
wait "$STREAM_PID" 2>/dev/null || true
STREAM_PID=""
# A phase that finished in the instant before the kill keeps its exit 0.
if [ "$(phase_status "$name" "$phase" 2>/dev/null)" = "exit 0" ]; then
log "watchdog: $phase finished before it could be killed; its exit 0 stands"
rc=0
else
remote_cancel "$name" "$p.pid"
rc=124
fi
break
fi
wait "$STREAM_PID" 2>/dev/null || true
STREAM_PID=""
st=$(phase_status "$name" "$phase") || st=""
case $st in
"exit "*) rc=${st#exit }; break ;;
lost|none) log "UBI_ATTACH_LOST: vm=$name phase=$phase: its session is gone and left no exit status ($p.exit)"; rc=125; break ;;
esac
tries=$(( tries + 1 ))
if [ "$tries" -ge 5 ]; then log "lost the connection to $phase on $name 5 times; giving up (the VM keeps it until teardown)"; rc=255; break; fi
log "the stream of $phase on $name dropped; following it again"
sleep "$POLL"
oo=$(fsize "$lout") oe=$(fsize "$lerr")
done
rm -f "$dir/stream"
trap - INT TERM; eval "$old_traps"
return "$rc"
}
# write_record DIR: the run record a later run of the same command reattaches
# by (key, tree, and this invocation for `attach NAME`). Remote files of each
# phase: ~/.ubirun-<phase>.{out,err,exit,pid} (setup, command, tier-<name>).
qwords() { local x; for x in "$@"; do printf '%q ' "$x"; done; }
write_record() {
{
printf 'RUN_KEY=%q\nTREE=%q\nPULL_ROOT=%q\nRESUME=%q\nRERUN_FAILED=%q\n' "$RUN_KEY" "$TREE" "$PULL_ROOT" "$RESUME" "$RERUN_FAILED"
printf 'RUN_ARGS=(%s)\n' "$(qwords "${RUN_ARGS[@]}")"
printf 'TIER_NAMES=(%s)\n' "$(qwords ${TIER_NAMES[@]+"${TIER_NAMES[@]}"})"
printf 'TIER_CMDS=(%s)\n' "$(qwords ${TIER_CMDS[@]+"${TIER_CMDS[@]}"})"
echo "# remote files: ~/.ubirun-<phase>.{out,err,exit,pid}; phase starts: phase.<phase>.start; lease: lease (pid start time)"
} >"$1/run.$$" && mv "$1/run.$$" "$1/run"
}
lease_take() { echo "$$ $(proc_start $$)" >"$1/lease.$$" && mv "$1/lease.$$" "$1/lease"; }
lease_busy() {
local pid st
[ -f "$1/lease" ] || return 1
read -r pid st <"$1/lease" || true
[ -n "$pid" ] && [ "$pid" != "$$" ] && kill -0 "$pid" 2>/dev/null && [ "$(proc_start "$pid")" = "$st" ]
}
# vm_alive NAME: the recorded VM still answers show. One confirmed gone is
# cleaned up (state dir and scratch record), so its record is never tried again.
vm_alive() {
local n=$1 out code
( load "$n"; [ "$CREATED" = yes ] && [ -n "$IP" ] ) 2>/dev/null || return 1
out=$(mktemp)
code=$(load "$n"; api "$out" vm "$LOCATION/$n" show)
rm -f "$out"
case $code in
200) return 0 ;;
404) log "run record for $n: the VM is gone; starting a new one"; cmd_down "$n" >/dev/null 2>&1 || true; return 1 ;;
*) return 1 ;;
esac
}
# ttl_short NAME SETUP: true when NAME's TTL (its created_at plus the name's
# -t<hours>, minus now) does not cover the run still to do plus
# REATTACH_MARGIN, with the numbers in TTL_WHY. The run still to do is the
# UBI_RUN_TIMEOUT bound of each phase that has not finished (setup when SETUP is
# set, then the command or every tier the plan does not reuse), counted from a
# started phase's recorded start; it is 0 when UBI_RUN_TIMEOUT is 0.
TTL_WHY=""
ttl_short() {
local name=$1 dir created left now need=0 p i phases=()
[[ "$name" =~ -t([0-9]{1,3})$ ]] || return 1
dir=$(state_dir "$name")
created=$(load "$name"; echo "$CREATE_AT")
now=$(date +%s)
left=$(( created + 10#${BASH_REMATCH[1]} * 3600 - now ))
[ -z "$2" ] || phases+=(setup)
if [ ${#TIER_NAMES[@]} -gt 0 ]; then
for i in "${!TIER_NAMES[@]}"; do [ "${TIER_ACT[i]}" = reuse ] || phases+=("tier-${TIER_NAMES[i]}"); done
else
phases+=(command)
fi
if [ "$RUN_TIMEOUT" -gt 0 ]; then
for p in "${phases[@]}"; do
if [ -f "$dir/phase.$p.start" ]; then i=$(( RUN_TIMEOUT - (now - $(cat "$dir/phase.$p.start")) )); [ "$i" -gt 0 ] || i=0
else i=$RUN_TIMEOUT; fi
need=$(( need + i ))
done
fi
TTL_WHY="ttl_left=${left}s needs=$(( need + REATTACH_MARGIN ))s (run still to do ${need}s + UBI_REATTACH_MARGIN=${REATTACH_MARGIN}s)"
[ "$left" -lt $(( need + REATTACH_MARGIN )) ]
}
# find_record: the VM of a run record with this RUN_KEY whose VM still exists.
find_record() {
local f n
for f in "$STATE_ROOT"/*/run; do
[ -f "$f" ] || continue
grep -qx "RUN_KEY=$RUN_KEY" "$f" || continue
n=$(basename "$(dirname "$f")")
if vm_alive "$n"; then echo "$n"; return 0; fi
done
}
# Tier checkpoints: write-once records <tier>.<epoch>.<pid>.json beside a copy
# of the tier's log, under CHECKPOINT_DIR/<run key>/, published under a lock
# with a monotonic generation; the newest generation per tier decides.
CKPT_PY='
import fcntl, hashlib, json, os, shutil, sys, time
op, d = sys.argv[1], sys.argv[2]
def records():
out = []
for f in (os.listdir(d) if os.path.isdir(d) else []):
if not f.endswith(".json") or f.startswith("."):
continue
try:
r = json.load(open(os.path.join(d, f)))
except (OSError, ValueError):
continue
if isinstance(r, dict) and isinstance(r.get("generation"), int):
r["_file"] = f
out.append(r)
return out
def newest(rs, tier):
rs = [r for r in rs if r.get("tier") == tier]
return max(rs, key=lambda r: r["generation"]) if rs else None
def sha(p):
h = hashlib.sha256()
with open(p, "rb") as fh:
for b in iter(lambda: fh.read(65536), b""):
h.update(b)
return h.hexdigest()
if op == "decide":
now, maxage, resume, rerun = int(sys.argv[3]), int(sys.argv[4]), sys.argv[5] == "1", sys.argv[6] == "1"
rs = records()
for tier in sys.argv[7:]:
r = newest(rs, tier)
f = r["_file"] if r else "-"
age = now - int(r.get("finished_at") or r.get("started_at") or now) if r else 0
dur = int(r.get("duration_s") or 0) if r else 0
code = str(r.get("exit", "-")) if r else "-"
log = os.path.join(d, str(r.get("log"))) if r and r.get("log") else ""
if not resume:
act, why = "run", "--no-resume"
elif r is None:
act, why = "run", "no record"
elif r.get("status") == "start":
act, why = "run", "interrupted (only a start record, %s)" % f
elif r.get("status") == "pass":
if age >= maxage:
act, why = "run", "pass %s is %ds old (UBI_CHECKPOINT_MAX_AGE=%d)" % (f, age, maxage)
elif not log or not os.path.isfile(log) or sha(log) != r.get("log_sha256"):
act, why = "run", "pass %s: its log does not re-hash" % f
else:
act, why = "reuse", "-"
elif r.get("status") == "fail":
act, why = ("rerun", "prior fail %s (exit %s; --rerun-failed)" % (f, code)) if rerun else ("blocked", "-")
else:
act, why = "run", "unreadable record %s" % f
print("\t".join([tier, act, f, str(age), str(dur), code, why]))
elif op == "publish":
tier, status, vm, started, finished, code, log, startgen, key, tree, pid = sys.argv[3:14]
os.makedirs(d, exist_ok=True)
with open(os.path.join(d, ".lock"), "a") as lk:
fcntl.flock(lk, fcntl.LOCK_EX)
rs = records()
gen = max([r["generation"] for r in rs] + [0]) + 1
if status != "start":
r = newest(rs, tier)
if r is None or str(r["generation"]) != startgen:
print("stale writer: its start record is generation %s, the newest for %s is %s" % (startgen, tier, r["generation"] if r else "none"), file=sys.stderr)
sys.exit(2)
rec = {"tier": tier, "status": status, "generation": gen, "vm": vm, "key": key, "tree": tree, "started_at": int(started)}
if status != "start":
rec.update(finished_at=int(finished), duration_s=int(finished) - int(started), exit=int(code))
epoch = int(time.time())
while os.path.exists(os.path.join(d, "%s.%d.%s.json" % (tier, epoch, pid))) or os.path.exists(os.path.join(d, "%s.%d.%s.log" % (tier, epoch, pid))):
epoch += 1
stem = "%s.%d.%s" % (tier, epoch, pid)
if status != "start":
shutil.copyfile(log, os.path.join(d, stem + ".log"))
rec["log"], rec["log_sha256"] = stem + ".log", sha(os.path.join(d, stem + ".log"))
tmp = os.path.join(d, ".%s.tmp" % stem)
with open(tmp, "w") as fh:
json.dump(rec, fh, sort_keys=True)
fh.write("\n")
os.link(tmp, os.path.join(d, stem + ".json"))
os.unlink(tmp)
print(gen)
'
# ckpt_publish TIER STATUS VM STARTED FINISHED EXIT LOG STARTGEN: print the new
# record's generation; fails (nothing published) for a stale writer.
ckpt_publish() { python3 -c "$CKPT_PY" publish "$CHECKPOINT_DIR/$RUN_KEY" "$@" "$RUN_KEY" "$TREE" "$$"; }
TIER_ACT=() TIER_TO_RUN=0 TIER_PASSED=0 TIER_REUSED="" TIER_FAILED="" TIER_PRIOR="" TIER_SAVED=0 TIER_BLOCKED="" TIER_BLOCKED_RC=1
# tier_plan: decide each tier from its newest checkpoint and print one
# UBI_VERIFY_RESUME line per tier before anything runs.
tier_plan() {
local plan t act f age dur code why
plan=$(python3 -c "$CKPT_PY" decide "$CHECKPOINT_DIR/$RUN_KEY" "$(date +%s)" "$CHECKPOINT_MAX_AGE" "$RESUME" "$RERUN_FAILED" "${TIER_NAMES[@]}") \
|| die "verify: could not read the tier checkpoints in $CHECKPOINT_DIR/$RUN_KEY"
while IFS=$'\t' read -r t act f age dur code why; do
case $act in
reuse)
echo "UBI_VERIFY_RESUME: tier=$t reuse $f (${age}s)" >&2
TIER_REUSED=${TIER_REUSED:+$TIER_REUSED,}$t TIER_SAVED=$(( TIER_SAVED + dur )) TIER_PASSED=$(( TIER_PASSED + 1 )) ;;
blocked)
echo "UBI_VERIFY_RESUME: tier=$t fail $f (${age}s, exit $code): not rerun without --rerun-failed" >&2
if [ -z "$TIER_BLOCKED" ]; then TIER_BLOCKED=$t; if [[ "$code" =~ ^[0-9]+$ ]]; then TIER_BLOCKED_RC=$code; fi; fi ;;
rerun)
echo "UBI_VERIFY_RESUME: tier=$t run: $why" >&2
TIER_PRIOR=${TIER_PRIOR:+$TIER_PRIOR,}$t:$code TIER_TO_RUN=$(( TIER_TO_RUN + 1 )) ;;
*)
echo "UBI_VERIFY_RESUME: tier=$t run: $why" >&2
TIER_TO_RUN=$(( TIER_TO_RUN + 1 )) ;;
esac
TIER_ACT+=("$act")
done <<<"$plan"
}
verify_line() {
echo "UBI_VERIFY: tree=${TREE:0:12} key=${RUN_KEY:0:12} tiers=${#TIER_NAMES[@]} passed=$TIER_PASSED reused=${TIER_REUSED:-none} failed=${TIER_FAILED:-none} prior_failures=${TIER_PRIOR:-none} saved_s=$TIER_SAVED"
}
# run_tiers NAME QDEST: run every tier the plan did not reuse, in order, each
# as its own detached phase; stop at the first failure and return its status.
# A pass is recorded only when the VM's tree after the tier equals the tree
# before it.
run_tiers() {
local name=$1 qdest=$2 dir i t phase before after trc started finished gen
dir=$(state_dir "$name")
for i in "${!TIER_NAMES[@]}"; do
[ "${TIER_ACT[i]}" != reuse ] || continue
t=${TIER_NAMES[i]} phase="tier-${TIER_NAMES[i]}"
if [ ! -f "$dir/phase.$phase.start" ]; then
before=$(vm_tree "$name" "$qdest") || before=unavailable
echo "$before" >"$dir/phase.$phase.tree"
if gen=$(ckpt_publish "$t" start "$name" "$(date +%s)" 0 0 "" 0); then echo "$gen" >"$dir/phase.$phase.gen"
else echo "UBI_CHECKPOINT_WRITE_FAILED: tier=$t start record (the tier still runs)" >&2; fi
fi
log "running tier $t on $name: ${TIER_CMDS[i]}"
trc=0
detached "$name" "$phase" "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "${TIER_CMDS[i]}")" || trc=$?
started=$(cat "$dir/phase.$phase.start" 2>/dev/null || date +%s) finished=$(date +%s)
log "tier $t exited $trc after $(( finished - started ))s"
case $trc in 125|255) TIER_FAILED=$t; return "$trc" ;; esac
gen=$(cat "$dir/phase.$phase.gen" 2>/dev/null || echo 0)
if [ "$trc" -ne 0 ]; then
ckpt_publish "$t" fail "$name" "$started" "$finished" "$trc" "$dir/$phase.log" "$gen" >/dev/null \
|| echo "UBI_CHECKPOINT_WRITE_FAILED: tier=$t fail record (the result stands)" >&2
TIER_FAILED=$t
return "$trc"
fi
TIER_PASSED=$(( TIER_PASSED + 1 ))
after=$(vm_tree "$name" "$qdest") || after=unavailable
before=$(cat "$dir/phase.$phase.tree" 2>/dev/null || echo unavailable)
if [ "$after" != "$before" ] || [ "$after" = unavailable ]; then
echo "UBI_VERIFY_TREE_MOVED: tier=$t before=${before:0:12} after=${after:0:12}: a tier must leave the source tree as it found it; no pass record" >&2
continue
fi
ckpt_publish "$t" pass "$name" "$started" "$finished" 0 "$dir/$phase.log" "$gen" >/dev/null \
|| echo "UBI_CHECKPOINT_WRITE_FAILED: tier=$t pass record (the result stands)" >&2
done
}
# vm_rows: "location name owner expires created" for every VM in the project;
# owner is "-" for untagged ubirun-<epoch>-* names and "." for VMs this runner
# did not name; expires is the epoch the name's -t<hours> TTL ends at, or "-";
@@ -303,6 +741,7 @@ watched() {
vm_rows() {
local loc name owner created expires
cli vm list -N -f location,name | while read -r loc name; do
[ -n "$name" ] || continue
owner=. created="" expires=-
if [[ "$name" =~ ^${PREFIX}-([a-z][a-z0-9]{0,11})-([0-9]{10})- ]]; then
owner=${BASH_REMATCH[1]} created=${BASH_REMATCH[2]}
@@ -316,23 +755,26 @@ vm_rows() {
done
}
# reap: destroy every VM whose name carries a TTL that ended more than
# REAP_GRACE seconds ago, whichever owner created it (the TTL is that owner's
# own declaration), and the caller's own TTL-less ubirun-<UBI_OWNER>-* names at
# least LEGACY_HOURS old. Another owner's TTL-less VM, an untagged one, and names
# outside the prefix are never touched: only a TTL lets reap judge another lane's VM.
# reap: destroy the caller's own VMs whose TTL ended more than REAP_GRACE
# seconds ago, and its TTL-less ubirun-<UBI_OWNER>-* names at least LEGACY_HOURS
# old. Every other VM is never touched, whatever its TTL says: a delete stays
# with the owner that created the VM. Other owners' expired VMs are only counted.
cmd_reap() {
local now rows loc name owner expires created n=0 failed=0
local now rows loc name owner expires created n=0 failed=0 others=0
[[ "$LEGACY_HOURS" =~ ^[1-9][0-9]{0,3}$ ]] || die "UBI_LEGACY_HOURS must be a whole number of hours from 1 to 9999 with no leading zero (how old a TTL-less ubirun-* VM must be before reap destroys it)"
now=$(date +%s)
rows=$(vm_rows) || { log "reap: could not list the project's VMs; nothing destroyed"; return 1; }
while read -r loc name owner expires created; do
[ -n "$name" ] || continue
if [ "$owner" != "$OWNER" ]; then
[ "$expires" = - ] || [ "$now" -lt $(( expires + REAP_GRACE )) ] || others=$(( others + 1 ))
continue
fi
if [ "$expires" != - ]; then
[ "$now" -ge $(( expires + REAP_GRACE )) ] || continue
log "reaping expired $loc/$name (owner $owner; its TTL ended $(( (now - expires) / 60 )) min ago)"
else
[ -n "${UBI_OWNER:-}" ] && [ "$owner" = "$OWNER" ] && [ "$created" != - ] || continue
[ -n "${UBI_OWNER:-}" ] && [ "$created" != - ] || continue
[ $(( now - 10#$created )) -ge $(( LEGACY_HOURS * 3600 )) ] || continue
log "reaping TTL-less $loc/$name (owner $owner; $(( (now - 10#$created) / 3600 ))h old, legacy sweep at ${LEGACY_HOURS}h)"
fi
@@ -340,6 +782,7 @@ cmd_reap() {
cmd_down "$name" -l "$loc" || { failed=$(( failed + 1 )); log "failed to reap $loc/$name"; }
done <<<"$rows"
[ "$n" -gt 0 ] || log "reap: no expired VMs"
[ "$others" -eq 0 ] || log "reap: left $others expired VMs of other owners; only their owners reap them"
[ "$failed" -eq 0 ] || { log "reap: $failed of $n expired VMs could not be destroyed"; return 1; }
}
@@ -367,21 +810,39 @@ cmd_list() {
}
# usage: VMs and vCPUs per owner, from each VM's size (standard-16 = 16 vCPUs).
# Exits 1 with no table when the VM list or any VM's size cannot be read, so a
# partial count is never taken for the project's usage. A VM destroyed between
# the list and its size read (404) no longer counts and is not a failure.
cmd_usage() {
local tmp loc name owner i=0
local tmp rows loc name owner f i=0 unread=0
rows=$(vm_rows) || { log "usage: could not list the project's VMs"; return 1; }
tmp=$(mktemp -d)
while read -r loc name owner _; do
[ -n "$name" ] || continue
i=$(( i + 1 ))
cli vm "$loc/$name" show 2>/dev/null | sed -n "s/^size: /$owner /p" >"$tmp/$i" &
{
case $(api "$tmp/$i.show" vm "$loc/$name" show) in
200) sed -n "s/^size: /$owner /p" "$tmp/$i.show" >"$tmp/$i" ;;
404) ;;
*) : >"$tmp/$i" ;;
esac
rm -f "$tmp/$i.show"
} &
[ $(( i % 8 )) -ne 0 ] || wait
done < <(vm_rows)
done <<<"$rows"
wait
cat "$tmp"/* 2>/dev/null | awk -v me="$OWNER" '
for f in "$tmp"/*; do [ ! -e "$f" ] || [ -s "$f" ] || unread=$(( unread + 1 )); done
if [ "$unread" -gt 0 ]; then
rm -rf "$tmp"
log "usage: could not read the size of $unread of $i VMs"
return 1
fi
find "$tmp" -type f -exec cat {} + | awk -v me="$OWNER" '
{ n = $2; sub(/^.*-/, "", n); o = $1 == "-" ? "(untagged)" : $1 == "." ? "(other)" : $1
vms[o]++; cpu[o] += n; total += n; count++ }
END {
printf "%-14s %4s %6s\n", "OWNER", "VMS", "VCPUS"
for (o in cpu) printf "%-14s %4d %6d%s\n", o, vms[o], cpu[o], o == me ? " (you)" : "" | "sort -k3,3nr"
for (o in cpu) printf "%-14s %4d %6d%s\n", o, vms[o], cpu[o], (o == me ? " (you)" : "") | "sort -k3,3nr"
close("sort -k3,3nr")
printf "%-14s %4d %6d\n", "total", count, total
}'
@@ -444,8 +905,17 @@ quota_preflight() {
# between them: a slow first usage read must not shave the printed budget.
remaining=$QUOTA_WAIT
while :; do
used=$(cmd_usage 2>/dev/null | awk '$1=="total" {print $3}')
[[ "$used" =~ ^[0-9]+$ ]] || return 0
if ! used=$(cmd_usage 2>/dev/null | awk '$1=="total" {print $3}') || ! [[ "$used" =~ ^[0-9]+$ ]]; then
if [ "$remaining" -le 0 ]; then
log "quota preflight: could not read the project's vCPU usage; the API's own answer decides"
return 0
fi
log "quota: could not read the project's vCPU usage; retrying, up to ${remaining}s more"
step=$(( remaining < 30 ? remaining : 30 ))
sleep "$step"
remaining=$(( remaining - step ))
continue
fi
[ $(( used + need )) -gt "$max" ] || return 0
if [ "$remaining" -le 0 ]; then
log "quota preflight refused $location/$name ($size): it needs $need vCPUs and the project's VMs already use $used of $max (ceiling from $source; other usage such as managed runners counts on top)"
@@ -637,7 +1107,12 @@ cmd_down() {
# pointer into the main repository, useless on the VM; the same snapshot
# replaces it, on the worktree's own branch and index. Not carried: submodule
# repositories (.git/modules) and a partial clone's promisor (the fetch pulls
# every missing blob first); gstack has neither.
# every missing blob first); gstack has neither. The archive is built in two
# passes inside the snapshot directory because macOS's bsdtar stops reading
# options at the first operand and resolves -T names after a -C, so no single
# tar command line works on both it and GNU tar (#3680). COPYFILE_DISABLE=1
# keeps Apple's tar from adding ._* AppleDouble entries for files with
# extended attributes, which the VM would see as untracked files.
cmd_pack() {
local src tmp gitdir branch head entry
src=$(cd "${1:-.}" && pwd)
@@ -668,9 +1143,10 @@ cmd_pack() {
# index's blobs that HEAD's tree lacks (staged, intent-to-add, the stages
# of an unmerged entry) are behind no ref, so they travel as a pack of
# their own; without them the VM's write-tree (gstack-wtree) cannot read a
# staged file.
# staged file. cat, not cp: git renames index.lock over the index, which
# cp reports as "replaced while being copied"; one open() reads one whole index.
if [ "$(git -C "$src" config --bool core.splitIndex 2>/dev/null)" != true ]; then
cp "$(cd "$src" && cd "$(dirname "$(git rev-parse --git-path index)")" && pwd)/index" "$tmp/.git/index"
cat "$(cd "$src" && cd "$(dirname "$(git rev-parse --git-path index)")" && pwd)/index" > "$tmp/.git/index"
# Read the copy, not the live index, so the pack matches it; a gitlink's
# sha (mode 160000) is a submodule commit this object store lacks.
GIT_INDEX_FILE="$tmp/.git/index" git -C "$src" ls-files -s | awk '$1 != "160000" {print $2}' | sort -u \
@@ -681,15 +1157,18 @@ cmd_pack() {
fi
(
cd "$src"
export COPYFILE_DISABLE=1
git ls-files -z -co --exclude-standard \
| while IFS= read -r -d '' f; do
if [ -e "$f" ] || [ -L "$f" ]; then printf '%s\0' "$f"; fi
done \
| tar --null -T - -c -C "$tmp" .git -zf -
| tar --null -T - -cf "$tmp/files.tar" \
&& tar -rf "$tmp/files.tar" -C "$tmp" .git \
&& gzip -c "$tmp/files.tar"
)
rm -rf "$tmp"; trap - EXIT
else
tar -C "$src" -czf - .
COPYFILE_DISABLE=1 tar -C "$src" -czf - .
fi
}
@@ -770,9 +1249,11 @@ run_teardown() {
[ "$rc" -ne 0 ] || exit 1
}
TIER_NAMES=() TIER_CMDS=() RESUME=1 RERUN_FAILED=0 FORCE_ATTACH="" RUN_ARGS=()
cmd_run() {
require_owner
local up_args=() src=. dest="" setup="" keep=0 pulls=() envs=()
local up_args=() src=. dest="" setup="" keep=0 pulls=() envs=() passes=() key_inputs=() probe=0
RUN_ARGS=("$@")
while [ $# -gt 0 ]; do
case $1 in
-S|--storage) check_storage "$2"; up_args+=("$1" "$2"); shift 2 ;;
@@ -781,60 +1262,161 @@ cmd_run() {
--dest) dest=$2; shift 2 ;;
--setup) setup=$2; shift 2 ;;
--env) envs+=("$2"); shift 2 ;;
--pass) pass_name "$2"; [ -n "${!2+x}" ] || die "--pass $2: not set locally"; envs+=("$2=${!2}"); shift 2 ;;
--pass) pass_name "$2"; [ -n "${!2+x}" ] || die "--pass $2: not set locally"; passes+=("$2"); shift 2 ;;
--pull) pulls+=("$2"); shift 2 ;;
--keep) keep=1; shift ;;
--key-input) key_inputs+=("$2"); shift 2 ;;
--probe) probe=1; shift ;;
--resume) RESUME=1; shift ;;
--no-resume) RESUME=0; shift ;;
--rerun-failed) RERUN_FAILED=1; shift ;;
--) shift; break ;;
*) die "run: unknown option $1" ;;
esac
done
[ $# -gt 0 ] || die "run: missing command after --"
[ $# -gt 0 ] || [ ${#TIER_NAMES[@]} -gt 0 ] || die "run: missing command after --"
[ -z "$setup" ] || [ -f "$setup" ] || die "setup script not found: $setup"
command -v git >/dev/null || die "git is required: the run key fingerprints the checkout with it"
src=$(cd "$src" && pwd)
[ -n "$dest" ] || dest="work/$(basename "$src")"
local i name=""
local i name="" size=$DEFAULT_SIZE location=$DEFAULT_LOCATION image=ubuntu-noble storage="" e n
for (( i = 0; i < ${#up_args[@]}; i += 2 )); do
case ${up_args[i]} in -n|--name) name=${up_args[i+1]} ;; esac
case ${up_args[i]} in
-n|--name) name=${up_args[i+1]} ;;
-s|--size) size=${up_args[i+1]} ;;
-l|--location) location=${up_args[i+1]} ;;
-b|--image) image=${up_args[i+1]} ;;
-S|--storage) storage=${up_args[i+1]} ;;
esac
done
# The run key: everything that decides what the VM runs, computed before any
# VM exists. Pass values enter only as their sha256.
if [ -z "$FORCE_ATTACH" ]; then
TREE=$(cd "$src" && bash -c "$TREE_SH") || die "could not fingerprint $src (git add -A into a temporary index failed)"
RUN_KEY=$({
echo "ubi-runner run key v1"
echo "tree $TREE"
printf 'command %q\n' "$*"
if [ -n "$setup" ]; then echo "setup $(sha256 <"$setup")"; else echo "setup none"; fi
for e in ${envs[@]+"${envs[@]}"}; do printf 'env %q\n' "$e"; done
for n in ${passes[@]+"${passes[@]}"}; do printf 'pass %s %s\n' "$n" "$(printf %s "${!n}" | sha256)"; done
printf 'vm %q %q %q %q\n' "$size" "$location" "$image" "$storage"
echo "base $(base_sha "$src")"
echo "runner $(sha256 <"${BASH_SOURCE[0]}")"
for e in ${key_inputs[@]+"${key_inputs[@]}"}; do printf 'input %q\n' "$e"; done
for i in "${!TIER_NAMES[@]}"; do printf 'tier %q %q\n' "${TIER_NAMES[i]}" "${TIER_CMDS[i]}"; done
} | sha256)
fi
[ ${#TIER_NAMES[@]} -eq 0 ] || tier_plan
local attach=""
if [ -n "$FORCE_ATTACH" ]; then
vm_alive "$FORCE_ATTACH" || die "attach: $FORCE_ATTACH no longer exists; rerun the original command to start fresh"
attach=$FORCE_ATTACH
else
attach=$(find_record)
fi
if [ "$probe" = 1 ]; then
if [ -z "$attach" ]; then echo "UBI_ATTACH_INFO: none key=${RUN_KEY:0:12}"
else
local busy=0
! lease_busy "$(state_dir "$attach")" || busy=1
if [ "$busy" = 0 ] && [ -z "$FORCE_ATTACH" ] && ttl_short "$attach" "$setup"; then
echo "UBI_ATTACH_INFO: none key=${RUN_KEY:0:12} ttl_short vm=$attach $TTL_WHY"
else
echo "UBI_ATTACH_INFO: vm=$attach created_at=$(load "$attach"; echo "$CREATE_AT") key=${RUN_KEY:0:12} busy=$busy"
fi
fi
return 0
fi
if [ ${#TIER_NAMES[@]} -gt 0 ] && { [ -n "$TIER_BLOCKED" ] || [ "$TIER_TO_RUN" -eq 0 ]; } && [ -n "$attach" ] && ! lease_busy "$(state_dir "$attach")"; then
log "verify: no tier needs $attach, which a dead runner left behind; destroying it"
cmd_down "$attach" >/dev/null || log "WARNING: failed to destroy $attach; run: $0 down $attach"
fi
if [ ${#TIER_NAMES[@]} -gt 0 ] && [ -n "$TIER_BLOCKED" ]; then
TIER_FAILED=$TIER_BLOCKED
verify_line
return "$TIER_BLOCKED_RC"
fi
if [ ${#TIER_NAMES[@]} -gt 0 ] && [ "$TIER_TO_RUN" -eq 0 ]; then
log "verify: every tier passed under this key; no VM needed"
verify_line
return 0
fi
if [ -n "$attach" ] && lease_busy "$(state_dir "$attach")"; then
die "UBI_ATTACH_BUSY: vm=$attach key=${RUN_KEY:0:12}: runner pid $(cut -d' ' -f1 <"$(state_dir "$attach")/lease") on this machine still owns it; wait for it or stop it, then rerun"
fi
if [ -n "$attach" ] && [ -z "$FORCE_ATTACH" ] && ttl_short "$attach" "$setup"; then
log "UBI_REATTACH_TTL_SHORT: vm=$attach key=${RUN_KEY:0:12} $TTL_WHY: starting a fresh VM (tier checkpoints carry over) and destroying $attach"
cmd_down "$attach" >/dev/null || log "WARNING: failed to destroy $attach; run: $0 down $attach"
attach=""
fi
[ -z "$attach" ] || name=$attach
[ -n "$name" ] || { name=$(new_name); up_args+=(-n "$name"); }
RUN_VM=$name
# Armed before the create request: an interrupted `up` still destroys its VM.
# A second signal must not cut teardown short, so the EXIT handler ignores them.
trap 'run_teardown "$?"' EXIT
trap 'exit 130' INT TERM HUP QUIT
cmd_up "${up_args[@]}" >/dev/null
if [[ "$name" =~ -t([0-9]+)$ ]]; then
log "$name carries a ${BASH_REMATCH[1]}h TTL (any later run of this script reaps it); if this process dies first: $0 down $name"
elif [[ "$name" =~ ^$PREFIX-$OWNER-[0-9]{10}- ]]; then
log "$name has no TTL (reap destroys it once it is ${LEGACY_HOURS}h old); if this process dies before its teardown line, destroy it with: $0 down $name"
trap 'exit 130' INT TERM QUIT
trap on_hup HUP
if [ -n "$attach" ]; then
load "$name"
log "reattached to $name created_at=$CREATE_AT key=${RUN_KEY:0:12}"
else
log "$name has no TTL and no reapable name, so nothing reaps it; if this process dies before its teardown line, destroy it with: $0 down $name"
cmd_up "${up_args[@]}" >/dev/null
trap 'exit 130' INT TERM QUIT
trap on_hup HUP
if [[ "$name" =~ -t([0-9]+)$ ]]; then
log "$name carries a ${BASH_REMATCH[1]}h TTL (this owner's next up or reap destroys it once it expires); if this process dies first: $0 down $name"
elif [[ "$name" =~ ^$PREFIX-$OWNER-[0-9]{10}- ]]; then
log "$name has no TTL (reap destroys it once it is ${LEGACY_HOURS}h old); if this process dies before its teardown line, destroy it with: $0 down $name"
else
log "$name has no TTL and no reapable name, so nothing reaps it; if this process dies before its teardown line, destroy it with: $0 down $name"
fi
fi
local dir
dir=$(state_dir "$name")
write_record "$dir"
lease_take "$dir"
if [ "$keep" = 1 ]; then
trap - EXIT
# A kept VM's finished run is not reattached by the next run; a detach (HUP) keeps the record.
trap 'rm -f "$(state_dir "$RUN_VM")/run"' EXIT
leftover "left=1 owner=$OWNER vm=$name kept=1"
log "--keep: leaving $name running; destroy with: $0 down $name"
fi
local e
for e in ${envs[@]+"${envs[@]}"}; do printf 'export %s=%q\n' "${e%%=*}" "${e#*=}"; done \
| remote "$name" 'umask 077 && cat > ~/.ubirun-env'
cmd_sync "$name" "$src" "$dest"
local qdest rc=0
local qdest rc=0 vmt
qdest=$(printf %q "$dest")
if [ ! -f "$dir/synced" ]; then
{ for e in ${envs[@]+"${envs[@]}"}; do printf 'export %s=%q\n' "${e%%=*}" "${e#*=}"; done
for n in ${passes[@]+"${passes[@]}"}; do printf 'export %s=%q\n' "$n" "${!n}"; done; } \
| remote "$name" 'umask 077 && cat > ~/.ubirun-env'
cmd_sync "$name" "$src" "$dest"
# A subdirectory of a checkout uploads the whole .git with only its own
# files, so its tree is compared VM to VM (tiers) but not to this side.
if [ -z "$(git -C "$src" rev-parse --show-prefix 2>/dev/null)" ]; then
vmt=$(vm_tree "$name" "$qdest") || vmt=unavailable
if [ "$vmt" != "$TREE" ]; then
log "UBI_VM_TREE_MISMATCH: vm=$name tree=${vmt:0:12} expected=${TREE:0:12}: the unpacked checkout is not the one this run was keyed on"
exit 3
fi
fi
[ -z "$setup" ] || remote "$name" 'cat > ~/.ubirun-setup.sh' <"$setup"
: >"$dir/synced"
fi
if [ -n "$setup" ]; then
remote "$name" 'cat > ~/.ubirun-setup.sh' <"$setup"
log "running setup $(basename "$setup")"
watched "$name" setup "cd $qdest && . ~/.ubirun-env && bash -l ~/.ubirun-setup.sh" || { rc=$?; log "setup failed (exit $rc)"; exit "$rc"; }
detached "$name" setup "cd $qdest && . ~/.ubirun-env && bash -l ~/.ubirun-setup.sh" || { rc=$?; log "setup failed (exit $rc)"; exit "$rc"; }
fi
local start
start=$(date +%s)
log "running on $name: $*"
watched "$name" command "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "$*")" || rc=$?
log "command exited $rc after $(( $(date +%s) - start ))s"
if [ ${#TIER_NAMES[@]} -gt 0 ]; then
run_tiers "$name" "$qdest" || rc=$?
else
log "running on $name: $*"
detached "$name" command "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "$*")" || rc=$?
log "command exited $rc after $(( $(date +%s) - $(cat "$dir/phase.command.start" 2>/dev/null || date +%s) ))s"
fi
local p
for p in ${pulls[@]+"${pulls[@]}"}; do
@@ -845,9 +1427,23 @@ cmd_run() {
fi
cmd_pull "$name" "${p%%:*}" "${p#*:}" || log "pull failed: $p"
done
[ ${#TIER_NAMES[@]} -eq 0 ] || verify_line
return "$rc"
}
# attach NAME: resume the run recorded for NAME on this machine (the same
# invocation, key and tiers), as a rerun of the same command would.
cmd_attach() {
local name=${1:-}
[ -n "$name" ] || die "attach: missing NAME"
valid_name "$name" || die "invalid VM name '$name'"
[ -f "$(state_dir "$name")/run" ] || die "attach: no run record for $name on this machine ($(state_dir "$name")/run); a machine replacement loses it"
# shellcheck disable=SC1091
. "$(state_dir "$name")/run"
FORCE_ATTACH=$name
cmd_run "${RUN_ARGS[@]}"
}
# verify [run opts] [--trust-config] [-- COMMAND]: run the repository's
# committed remote verify. .gstack/ubicloud.json under --src (default .)
# supplies setup, command, size, location, storage, image, env, pass and pull;
@@ -860,7 +1456,7 @@ cmd_run() {
# local variable it lists under pass is forwarded only when this command line
# also names it (--pass NAME) or grants every listed name (--trust-config).
cmd_verify() {
local src=. i j g n cfg cfg_setup cfg_cmd parsed err rc=0 trust=0 cfg_args=() cfg_pass=() user_pre=() user_cmd=() granted=() missing=()
local src=. i j g n cfg cfg_setup cfg_cmd parsed err rc=0 trust=0 cfg_args=() cfg_pass=() user_pre=() user_cmd=() granted=() missing=() cfg_tier_names=() cfg_tier_cmds=()
for (( i = 1; i <= $#; i++ )); do
[ "${!i}" = -- ] && break
if [ "${!i}" = --src ]; then j=$(( i + 1 )); src=${!j}; fi
@@ -869,7 +1465,7 @@ cmd_verify() {
[ -f "$cfg" ] || die "verify: no $cfg; commit one ({\"setup\": \"scripts/ubicloud/bootstrap.sh\", \"command\": \"bash scripts/ubicloud/verify.sh\"}) so every lane runs the same verify instead of its own wrapper"
err=$(mktemp)
parsed=$(python3 -c '
import json, os, shlex, sys
import json, os, re, shlex, sys
cfg, src = sys.argv[1], sys.argv[2]
try:
c = json.load(open(cfg))
@@ -895,7 +1491,25 @@ for item in c.get("pull", []):
if not sep or not remote:
print("pull entry %r must be REMOTE_GLOB:LOCAL_DIR" % (item,), file=sys.stderr); sys.exit(2)
args += ["--pull", q(remote + ":" + inside(local, "pull destination"))]
tiers = c.get("tiers")
names = []
if tiers is not None:
if "command" in c:
print("UBI_CONFIG_TIERS_AND_COMMAND: give either command or tiers, not both", file=sys.stderr); sys.exit(2)
if not isinstance(tiers, list) or not tiers:
print("UBI_CONFIG_TIER_NAME: tiers must be a non-empty list of {\"name\", \"command\"}", file=sys.stderr); sys.exit(2)
for t in tiers:
name = t.get("name") if isinstance(t, dict) else None
if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,31}", name):
print("UBI_CONFIG_TIER_NAME: tier name %r must match ^[a-z0-9][a-z0-9_-]{0,31}$" % (name,), file=sys.stderr); sys.exit(2)
if name in names:
print("UBI_CONFIG_TIER_NAME: tier name %r appears twice" % name, file=sys.stderr); sys.exit(2)
if not isinstance(t.get("command"), str) or not t["command"].strip():
print("UBI_CONFIG_TIER_NAME: tier %r needs a command" % name, file=sys.stderr); sys.exit(2)
names.append(name)
setup = c.get("setup", "")
print("cfg_tier_names=(" + " ".join(q(n) for n in names) + ")")
print("cfg_tier_cmds=(" + " ".join(q(t["command"]) for t in (tiers or [])) + ")")
print("cfg_args=(" + " ".join(args) + ")")
print("cfg_pass=(" + " ".join(q(x) for x in c.get("pass", [])) + ")")
print("cfg_setup=" + q(inside(setup, "setup") if setup else ""))
@@ -912,7 +1526,7 @@ print("cfg_cmd=" + q(c.get("command", "")))
case $1 in
--) shift; user_cmd=("$@"); break ;;
--trust-config) trust=1; shift ;;
--keep) user_pre+=("$1"); shift ;;
--keep|--resume|--no-resume|--rerun-failed|--probe) user_pre+=("$1"); shift ;;
--pass|-S|--storage|-s|--size|-l|--location|-n|--name|-b|--image|--src|--dest|--setup|--env|--pull)
[ $# -ge 2 ] || die "verify: $1 needs a value"
[ "$1" != --pass ] || granted+=("$2")
@@ -927,11 +1541,16 @@ print("cfg_cmd=" + q(c.get("command", "")))
missing+=("$n")
done
[ ${#missing[@]} -eq 0 ] || die "verify: $cfg asks to forward local ${missing[*]}; grant each on this command line with --pass NAME, or every name it lists with --trust-config"
[ ${#user_cmd[@]} -gt 0 ] || [ -n "$cfg_cmd" ] || die "verify: $cfg names no command and none was given after --"
[ ${#user_cmd[@]} -gt 0 ] || user_cmd=("$cfg_cmd")
# A trailing -- COMMAND replaces the configured tiers as it replaces command.
if [ ${#user_cmd[@]} -eq 0 ] && [ ${#cfg_tier_names[@]} -gt 0 ]; then
TIER_NAMES=("${cfg_tier_names[@]}") TIER_CMDS=("${cfg_tier_cmds[@]}")
else
[ ${#user_cmd[@]} -gt 0 ] || [ -n "$cfg_cmd" ] || die "verify: $cfg names no command and none was given after --"
[ ${#user_cmd[@]} -gt 0 ] || user_cmd=("$cfg_cmd")
fi
log "verify: $cfg"
PULL_ROOT=$(cd "$src" && pwd)
cmd_run ${cfg_args[@]+"${cfg_args[@]}"} ${cfg_setup:+--setup "$cfg_setup"} --src "$src" ${user_pre[@]+"${user_pre[@]}"} -- "${user_cmd[@]}"
cmd_run ${cfg_args[@]+"${cfg_args[@]}"} ${cfg_setup:+--setup "$cfg_setup"} --src "$src" ${user_pre[@]+"${user_pre[@]}"} -- ${user_cmd[@]+"${user_cmd[@]}"}
}
usage() {
@@ -939,13 +1558,17 @@ usage() {
usage: ubi-runner.sh <command> [args]
run [up opts] [--src DIR] [--dest PATH] [--setup FILE] [--env K=V]...
[--pass NAME]... [--pull REMOTE_GLOB:LOCAL_DIR]... [--keep] -- COMMAND
up + sync + setup + command + pull + destroy; exits with COMMAND's status
verify [run opts] [--trust-config] [-- COMMAND]
[--pass NAME]... [--pull REMOTE_GLOB:LOCAL_DIR]... [--keep] [--key-input S]... [--probe] -- COMMAND
up + sync + setup + command + pull + destroy; exits with COMMAND's status.
A rerun of the same command (same run key) reattaches to its live VM;
--probe prints UBI_ATTACH_INFO (and the tier plan) and exits
verify [run opts] [--trust-config] [--resume|--no-resume] [--rerun-failed] [-- COMMAND]
run the repository's committed verify (.gstack/ubicloud.json under --src);
its setup script and every pull destination stay inside --src (checked
again at pull time), and its pass names are forwarded only when granted
here (--pass NAME, or all: --trust-config)
here (--pass NAME, or all: --trust-config). With tiers, a pass checkpoint
under the same run key is reused (resume is the default)
attach NAME resume the run recorded for NAME on this machine, as a rerun would
up [-s SIZE] [-l LOCATION] [-n NAME] [-S STORAGE_GIB] [-b IMAGE]
create a VM and wait for SSH; prints its name
(STORAGE_GIB: one of $ALLOWED_STORAGE)
@@ -961,10 +1584,15 @@ usage: ubi-runner.sh <command> [args]
owner print this caller's owner tag ($OWNER)
records VMs this machine created and has not confirmed gone ($VM_RECORD)
gc HOURS destroy this owner's VMs older than HOURS (never other owners')
reap destroy every VM whose -t<hours> TTL has ended (any owner) and this
owner's TTL-less ubirun-* VMs older than UBI_LEGACY_HOURS
reap destroy this owner's VMs whose -t<hours> TTL has ended and its
TTL-less ubirun-* VMs older than UBI_LEGACY_HOURS (never other owners')
cli ARGS... raw Ubicloud CLI passthrough (e.g. cli vm list)
signals: INT and TERM tear the VM down. HUP, once a remote phase is live, detaches
(UBI_DETACHED, exit 129; the VM runs on under its TTL). SIGKILL or a machine stop leaves
the run record and the VM; rerun the same command to reattach (UBI_ATTACH_BUSY while
another runner owns it; exit 125 UBI_ATTACH_LOST when the remote session left no status).
VMs are named $PREFIX-<owner>-<epoch>-<suffix>-t<ttl hours>; up, run and verify need
UBI_OWNER (e.g. your thread code); run and verify end with a UBICLOUD: left=<n> line.
defaults: size=$DEFAULT_SIZE location=$DEFAULT_LOCATION (env UBI_SIZE, UBI_LOCATION)
@@ -976,7 +1604,8 @@ env: UBI_TTL_HOURS (${TTL_HOURS}h in the name, 1-999), UBI_REAP (0: up skips the
UBI_QUOTA_WAIT (${QUOTA_WAIT}s to wait for quota room instead of refusing),
UBI_GC_HOURS (unset/0: no sweep on up), UBI_CREATE_GRACE (${CREATE_GRACE}s),
UBI_DOWN_TIMEOUT (${DOWN_TIMEOUT}s), UBI_API_TIMEOUT (${API_TIMEOUT}s), UBI_VM_RECORD (scratch VM-id file),
UBI_ALLOWED_STORAGE (the -S sizes Ubicloud accepts; default "320 640")
UBI_ALLOWED_STORAGE (the -S sizes Ubicloud accepts; default "320 640"),
UBI_CHECKPOINT_DIR (tier checkpoints; $CHECKPOINT_DIR), UBI_CHECKPOINT_MAX_AGE (${CHECKPOINT_MAX_AGE}s a pass is reused)
EOF
}
@@ -989,6 +1618,7 @@ esac
case $cmd in
run) cmd_run "$@" ;;
verify) cmd_verify "$@" ;;
attach) cmd_attach "$@" ;;
up) trap 'trap "" INT TERM HUP QUIT; [ -z "$UP_NAME" ] || [ -n "$UP_READY" ] || cmd_down "$UP_NAME" || log "WARNING: failed to destroy $UP_NAME; run: $0 down $UP_NAME"' EXIT
trap 'exit 130' INT TERM HUP QUIT
cmd_up "$@" ;;