mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-08 01:38:10 +01:00
v1.92.60.0 stamp: merge main d7154dba13 into #3730 (land)
This commit is contained in:
commit
c67821d3a6
86 files changed
+2563
-408
No files matched your search
@@ -259,7 +259,38 @@ export function readFlakeLedgerDir(dir: string): FlakeLedgerEntry[] {
|
||||
export const FLAKE_POLICY = { maxRate: 0.05, minRuns: 20 } as const;
|
||||
|
||||
export interface FlakyFile { file: string; flakyRuns: number; entries: number; rate: number; enforced: boolean; failing: boolean }
|
||||
export interface FlakeWindow { runs: number; flakyRuns: number; since: string; files: FlakyFile[]; enforceable: boolean }
|
||||
/** `skipped`: reused main pushes left out of the window (reuseChecker); `note`: runs that could not be checked. */
|
||||
export interface FlakeWindow { runs: number; flakyRuns: number; since: string; files: FlakyFile[]; enforceable: boolean; skipped?: number; note?: string }
|
||||
|
||||
/** The most job listings one reuseChecker spends. */
|
||||
export const REUSE_LOOKUPS_MAX = 40;
|
||||
|
||||
/**
|
||||
* Finds runs that reused their merged pull request's verified run (#3709): such
|
||||
* a run uploads no artifact and its `reuse` job succeeds, because every suite
|
||||
* job was skipped. It holds no flake evidence, so flake counts leave it out of
|
||||
* their runs and their denominator (#3725). A run with any artifact costs no
|
||||
* call. A run with none costs one job listing, at most REUSE_LOOKUPS_MAX per
|
||||
* checker; past the cap, or when the listing fails, the run counts as it did
|
||||
* before #3725 and note() says so.
|
||||
*/
|
||||
export function reuseChecker(client: GhClient, repo: string) {
|
||||
let lookups = 0;
|
||||
let unchecked = 0;
|
||||
let reason = '';
|
||||
return {
|
||||
reused(run: Pick<CiRun, 'id' | 'attempt'>, artifactNames: string[]): boolean {
|
||||
if (artifactNames.length) return false;
|
||||
if (lookups >= REUSE_LOOKUPS_MAX) { unchecked++; reason ||= `cap of ${REUSE_LOOKUPS_MAX} job listings reached`; return false; }
|
||||
lookups++;
|
||||
try {
|
||||
return listRunJobs(client, repo, run.id, run.attempt).some(job => job.name === 'reuse' && job.conclusion === 'success');
|
||||
} catch (error) { unchecked++; reason ||= unavailableReason(error); return false; }
|
||||
},
|
||||
note: (): string | undefined => (unchecked
|
||||
? `${unchecked} run(s) with no artifacts not checked for main-push reuse (${reason}); counted as before #3725` : undefined),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Rank files by the share of runs in which they needed the retry pass. A file
|
||||
@@ -286,9 +317,10 @@ export function rankFlakyFiles(runEntries: Map<number, FlakeLedgerEntry[]>, runC
|
||||
/**
|
||||
* Read the flake ledgers of main's completed free-tests runs: the runs created
|
||||
* in the last `sinceDays`, widened to the newest `minRuns` runs (within
|
||||
* `maxLookbackDays`) so the 5% rule has a window of at least 20 runs. A run
|
||||
* with no `flake-ledger-*` artifact had no flaky passes (the upload is skipped
|
||||
* when the ledger file does not exist).
|
||||
* `maxLookbackDays`) so the 5% rule has a window of at least 20 runs. Reused
|
||||
* main pushes (reuseChecker) are left out and older runs take their place. A
|
||||
* run with no `flake-ledger-*` artifact had no flaky passes (the upload is
|
||||
* skipped when the ledger file does not exist).
|
||||
*/
|
||||
export function readCiFlakeLedgers(opts: { client: GhClient; repo: string; sinceDays: number; now: number; cacheDir: string;
|
||||
minRuns?: number; maxLookbackDays?: number; workflow?: string }): FlakeWindow {
|
||||
@@ -297,16 +329,25 @@ export function readCiFlakeLedgers(opts: { client: GhClient; repo: string; since
|
||||
const all = listWorkflowRuns(opts.client, opts.repo, opts.workflow ?? 'free-tests.yml',
|
||||
{ since: iso(Math.max(opts.sinceDays, opts.maxLookbackDays ?? 60)), branch: 'main' })
|
||||
.filter(run => run.conclusion === 'success' || run.conclusion === 'failure');
|
||||
const recent = all.filter(run => run.createdAt >= iso(opts.sinceDays));
|
||||
const window = recent.length >= minRuns ? recent : all.slice(0, minRuns);
|
||||
const since = iso(opts.sinceDays);
|
||||
const recent = all.filter(run => run.createdAt >= since).length;
|
||||
const fetcher = historyFetcher(opts.client);
|
||||
const reuse = reuseChecker(opts.client, opts.repo);
|
||||
const window: CiRun[] = [];
|
||||
let skipped = 0;
|
||||
for (const [index, run] of all.entries()) {
|
||||
if (index >= recent && window.length >= minRuns) break;
|
||||
if (reuse.reused(run, fetcher.listArtifacts(opts.repo, run.id).map(artifact => artifact.name))) skipped++;
|
||||
else window.push(run);
|
||||
}
|
||||
const runEntries = new Map<number, FlakeLedgerEntry[]>();
|
||||
for (const run of window) {
|
||||
const dirs = downloadRunArtifacts({ repo: opts.repo, run, cacheDir: opts.cacheDir, fetcher, match: name => name.startsWith('flake-ledger-') });
|
||||
const entries = dirs.flatMap(readFlakeLedgerDir);
|
||||
if (entries.length) runEntries.set(run.id, entries);
|
||||
}
|
||||
return rankFlakyFiles(runEntries, window.length, window[window.length - 1]?.createdAt ?? iso(opts.sinceDays));
|
||||
const note = reuse.note();
|
||||
return { ...rankFlakyFiles(runEntries, window.length, window[window.length - 1]?.createdAt ?? since), skipped, ...(note ? { note } : {}) };
|
||||
}
|
||||
|
||||
/** Markdown table of one run's merged flake ledgers, for a job summary. */
|
||||
|
||||
@@ -315,6 +315,10 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
|
||||
file: 'test/ship-queue-train-land.test.ts',
|
||||
reason: 'drives train land against test/helpers/ship-queue-train-fixture.ts plus an extensionless node gh wrapper on a colon-joined PATH; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead, as for test/ship-queue-train.test.ts and test/ship-queue-land.test.ts (windows-free-shard 6, run 40824965988)',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-queue-land-lock.test.ts',
|
||||
reason: 'drives land and train land against test/helpers/ship-queue-train-fixture.ts plus an extensionless node gh wrapper on a colon-joined PATH; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead, as for test/ship-queue-train-land.test.ts and test/ship-queue-land.test.ts (windows-free-shard 6, run 40824965988)',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-queue-train.test.ts',
|
||||
reason: 'drives bin/gstack-ship-queue against the stateful #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead of the shim (windows-free-shard run 39152702056)',
|
||||
@@ -407,6 +411,7 @@ export const KNOWN_WINDOWS_SAFE: Array<{ file: string; reason: string }> = [
|
||||
{ file: 'test/gstack-wtree-windows-paths.test.ts', reason: '#3101: runs bin/gstack-wtree through explicit bash in an os.tmpdir linked worktree with forward-slash paths; never executes a shebang; the drive-letter simulation is skipIf win32' },
|
||||
{ file: 'test/setup-hook-syntax-gate.test.ts', reason: '#3088/#3100: runs bin/gstack-hook-check and the setup hook sections through explicit bash with forward-slash paths; never executes a shebang. Its Bun drive-path spelling is the Windows bug it guards' },
|
||||
{ file: 'test/bun-config-shell-bins.test.ts', reason: '#3415: runs the record bins through lib/script-invocation.ts (Git Bash on Windows) with MSYS path conversion off, the case it guards; its bun and cygpath stand-ins and simulated-OSTYPE cases are skipIf win32' },
|
||||
{ file: 'test/ship-receipt-session-kind.test.ts', reason: '#3541: runs gstack-ship-receipt write through lib/script-invocation.ts, which must reach the bash gstack-session-kind through Git Bash; os.tmpdir git fixture only' },
|
||||
{ file: 'test/path-spelling.test.ts', reason: 'sources bin/gstack-path-spelling.sh through explicit bash and checks lib/path-spelling.ts; on Windows it exercises the real Git Bash spellings' },
|
||||
{ file: 'test/timeline-stop-hook-stdin.test.ts', reason: '#3628: the hook runs through Bun argv with a stdin pipe held open; the return-within-budget timing must meet real Windows process startup' },
|
||||
{ file: 'test/state-root-parity.test.ts', reason: 'runs the bash twin and lib/state-root.ts over an env table with PATH empty; no shebang execution, raw-string comparison is platform-neutral' },
|
||||
|
||||
@@ -89,7 +89,7 @@ export const SELECTION_DATA_MODULES: readonly string[] = [
|
||||
// The build entry list: every compiled binary, which skills reach through their `--outfile` path instead.
|
||||
'scripts/build.sh',
|
||||
// The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes.
|
||||
'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', 'lib/result-codes-plan.ts',
|
||||
'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', 'lib/result-codes-plan.ts', 'lib/result-codes-autoplan.ts',
|
||||
// The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case).
|
||||
'test/helpers/exercised-commands.json',
|
||||
];
|
||||
|
||||
@@ -17,7 +17,7 @@ import type { ResolverFn } from './types';
|
||||
|
||||
/** One entry per command line the loop runs, as the field-wave plan specifies them. */
|
||||
export const AUTOPLAN_DRIVER_COMMANDS: Array<{ cmd: string; does: string }> = [
|
||||
{ cmd: 'gstack-autoplan next --out <dir> --plan <plan.md> [scope flags] [--light] [--spend-cap <usd>] [--max-rounds <n>] [--estimate-usd <n>] [--scaffold]', does: `first call: init the run, print PHASE:, SCOPE:, DRIVER: and one ATTEMPT: per voice; --max-rounds caps review rounds per phase (default ${DEFAULT_MAX_ROUNDS} unattended); a run with --spend-cap, in any session kind (or any unattended Capy run), reserves ${DEFAULT_HOST_ESTIMATE_USD.toFixed(2)} per voice for each attempt, so the cap binds; --estimate-usd <n> sets that reservation and the run keeps it for later phases` },
|
||||
{ cmd: 'gstack-autoplan next --out <dir> --plan <plan.md> [scope flags] [--light] [--spend-cap <usd>] [--max-rounds <n>] [--estimate-usd <n>] [--scaffold] [--ceo-only]', does: `first call: init the run, print PHASE:, SCOPE:, DRIVER: and one ATTEMPT: per voice; --max-rounds caps review rounds per phase (default ${DEFAULT_MAX_ROUNDS} unattended); a run with --spend-cap, in any session kind (or any unattended Capy run), reserves ${DEFAULT_HOST_ESTIMATE_USD.toFixed(2)} per voice for each attempt, so the cap binds; --estimate-usd <n> sets that reservation and the run keeps it for later phases; --ceo-only (command line only, never a scope marker) reviews CEO alone and ends \`status=reviewed\` with \`<dir>/ceo-review.md\` (\`REVIEWED:\`): no gate, and the gate verbs refuse it (\`CEO_ONLY_NO_GATE\`)` },
|
||||
{ cmd: 'gstack-autoplan run --out <dir> --plan <plan.md> --native-model <id> --outside-model <id> --native-runner api --runner api [--max-tokens <n>] [--spend-cap <usd>]', does: 'hosts without subagents: one command calls both models directly (ANTHROPIC_API_KEY, OPENAI_API_KEY) and loops next/submit to the gate; a response cut off at the output limit is reported truncated, so raise --max-tokens (default 16384 for both providers; OpenAI counts reasoning tokens toward it); each api call reserves its priced worst case (a model with no price reserves the per-voice default under a cap), a native call that went out but bound no result settles at its usage, or its full reservation when the usage is unknown, and a failed outside call stays counted at its reservation' },
|
||||
{ cmd: 'gstack-autoplan next --out <dir> [--redispatch <attempt>]', does: 'later calls: open the next phase or reprint open attempts; never redispatches on its own' },
|
||||
{ cmd: 'gstack-autoplan submit --out <dir> --phase <p> --voice native|outside --record <file.md> --model <id> [--attempt <id>]', does: 'bind a free-form review record (canonical fence, one findings table or list, or `No findings.`)' },
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { TemplateContext } from '../types';
|
||||
import { getHostConfig } from '../../../hosts/index';
|
||||
import { runtimeRootPrelude } from '../runtime-root';
|
||||
import { skillStartLines } from '../runtime-root';
|
||||
|
||||
/**
|
||||
* Preamble bootstrap (token-reduction Phase 1).
|
||||
@@ -25,11 +25,10 @@ import { runtimeRootPrelude } from '../runtime-root';
|
||||
*/
|
||||
export function generatePreambleBash(ctx: TemplateContext): string {
|
||||
const hostConfig = getHostConfig(ctx.host);
|
||||
const runtimeRoot = hostConfig.usesEnvVars ? `${runtimeRootPrelude(ctx)}\n` : '';
|
||||
const start = hostConfig.usesEnvVars ? skillStartLines(ctx) : null;
|
||||
const runtimeRoot = start ? `${start.lookup}\n` : '';
|
||||
const brainHealthFlag = ctx.host === 'gbrain' || ctx.host === 'hermes' ? ' --brain-health' : '';
|
||||
const startCommand = hostConfig.usesEnvVars
|
||||
? `"${ctx.paths.binDir}/gstack-skill-start"`
|
||||
: `${ctx.paths.binDir}/gstack-skill-start`;
|
||||
const startCommand = start ? start.command : `${ctx.paths.binDir}/gstack-skill-start`;
|
||||
|
||||
const entry = ['plan-design-review', 'plan-eng-review'].includes(ctx.skillName)
|
||||
? `## Preamble (after scope gate)
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
* bash finds gstack on Codex and every other `usesEnvVars` host.
|
||||
*
|
||||
* Those hosts run each fenced block in a fresh shell, so `GSTACK_ROOT`, `B`,
|
||||
* `D` and `P` set by an earlier block are gone. The preamble's root resolution and
|
||||
* the per-fence prelude come from `runtimeRootPrelude()`; `insertRuntimePreludes()`
|
||||
* `D` and `P` set by an earlier block are gone. The preamble's skill-start fence
|
||||
* comes from `skillStartLines()` and the per-fence prelude from
|
||||
* `runtimeRootPrelude()`; `insertRuntimePreludes()`
|
||||
* is the single post-render pass (gen-skill-docs, before rewriteInstallRoot)
|
||||
* that puts it into every fence that uses these variables without assigning
|
||||
* them. Claude output is returned unchanged.
|
||||
@@ -13,6 +14,9 @@
|
||||
* on a per-install render, the literal install root (no git call); otherwise the
|
||||
* repo-local install, then the host's global root (`CODEX_HOME` on Codex). No
|
||||
* root prints one line naming what was tried and the setup command, and exits.
|
||||
* The skill-start fence differs (GST-882): an install-root render calls the
|
||||
* literal install root's start script and ignores an exported `GSTACK_ROOT`, and
|
||||
* with no root either render prints that line and lets the call fail.
|
||||
*/
|
||||
import { getHostConfig } from '../../hosts/index';
|
||||
import { toShellPath, type TemplateContext } from './types';
|
||||
@@ -37,13 +41,45 @@ const uses = (name: string) => new RegExp(`\\$\\{?${name}\\b`);
|
||||
*/
|
||||
function resolveRoot(ctx: TemplateContext): string {
|
||||
const host = getHostConfig(ctx.host);
|
||||
const fix = `Fix: ./setup --host ${host.name} from your gstack checkout; ./setup --status shows it.`;
|
||||
const fix = installFix(ctx);
|
||||
const exported = '[ -d "${GSTACK_ROOT:-/-}/bin" ]&&[ -d "$GSTACK_ROOT/lib" ]||';
|
||||
if (ctx.installRoot) {
|
||||
return `${exported}{ GSTACK_ROOT="${ctx.installRoot.replace(/\/+$/, '')}";[ -d "$GSTACK_ROOT/bin" ]||{ echo "gstack: no install found (tried $GSTACK_ROOT). ${fix}">&2;exit 1;};}`;
|
||||
}
|
||||
const global = host.name === 'codex' ? '${CODEX_HOME:-~/.codex}/skills/gstack' : `~/${host.globalRoot}`;
|
||||
return `${exported}{ _r=$(git rev-parse --show-toplevel 2>/dev/null)/${host.localSkillRoot};[ -d "$_r/bin" ]||_r=${global};[ -d "$_r/bin" ]||{ echo "gstack: no install found (tried $_r). ${fix}">&2;exit 1;};GSTACK_ROOT=$_r;}`;
|
||||
return `${exported}{ _r=$(git rev-parse --show-toplevel 2>/dev/null)/${host.localSkillRoot};[ -d "$_r/bin" ]||_r=${globalRoot(ctx)};[ -d "$_r/bin" ]||{ echo "gstack: no install found (tried $_r). ${fix}">&2;exit 1;};GSTACK_ROOT=$_r;}`;
|
||||
}
|
||||
|
||||
const installFix = (ctx: TemplateContext) => `Fix: ./setup --host ${getHostConfig(ctx.host).name} from your gstack checkout; ./setup --status shows it.`;
|
||||
const globalRoot = (ctx: TemplateContext) => {
|
||||
const host = getHostConfig(ctx.host);
|
||||
return host.name === 'codex' ? '${CODEX_HOME:-~/.codex}/skills/gstack' : `~/${host.globalRoot}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* The skill-start fence on env-var hosts (GST-882). Hermes's Tirith blocks a
|
||||
* start command it cannot resolve statically, a grouped fallback or a
|
||||
* variable-selected path alike, and no allowlist entry clears that finding. An
|
||||
* install-root render therefore names the literal start path; a checked-in
|
||||
* render keeps resolveRoot's lookup order as plain lines. Either way a missing
|
||||
* install prints the fix, and the call then fails.
|
||||
*/
|
||||
export function skillStartLines(ctx: TemplateContext): { lookup: string; command: string } {
|
||||
const fix = installFix(ctx);
|
||||
if (ctx.installRoot) {
|
||||
const root = ctx.installRoot.replace(/\/+$/, '');
|
||||
return {
|
||||
lookup: `[ -x "${root}/bin/gstack-skill-start" ]||echo "gstack: no install found (tried ${root}). ${fix}">&2`,
|
||||
command: `"${root}/bin/gstack-skill-start"`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
lookup: [
|
||||
`[ -d "\${GSTACK_ROOT:-/-}/lib" ]&&[ -d "$GSTACK_ROOT/bin" ]||GSTACK_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)/${getHostConfig(ctx.host).localSkillRoot}`,
|
||||
`[ -d "$GSTACK_ROOT/bin" ]||GSTACK_ROOT=${globalRoot(ctx)}`,
|
||||
`[ -d "$GSTACK_ROOT/bin" ]||echo "gstack: no install found (tried $GSTACK_ROOT). ${fix}">&2`,
|
||||
].join('\n'),
|
||||
command: '"$GSTACK_ROOT/bin/gstack-skill-start"',
|
||||
};
|
||||
}
|
||||
|
||||
const DERIVED: Record<string, string> = { BIN: 'bin', BROWSE: 'browse/dist', DESIGN: 'design/dist', MAKE_PDF: 'make-pdf/dist' };
|
||||
|
||||
@@ -128,12 +128,18 @@ export function flakeCheck(window: FlakeWindow | Error): Check {
|
||||
fix: 'Fix the race at source (docs/TESTING_INTERNALS.md#flake-ledger); never raise the timeout or add a retry. Reproduce with `bun test <file>` under load.' };
|
||||
if (window instanceof Error) return { ...base, status: 'unavailable', detail: `unavailable: ${unavailableReason(window)}` };
|
||||
const failing = window.files.filter(file => file.failing);
|
||||
const left = flakeWindowNotes(window);
|
||||
if (!window.enforceable) {
|
||||
return { ...base, status: 'pass', detail: `report-only: ${window.runs} main run(s) since ${window.since.slice(0, 10)} (< 20 needed to enforce)` };
|
||||
return { ...base, status: 'pass', detail: `report-only: ${window.runs} main run(s) since ${window.since.slice(0, 10)} (< 20 needed to enforce)${left}` };
|
||||
}
|
||||
if (failing.length === 0) return { ...base, status: 'pass', detail: `no file above 5% over ${window.runs} main runs` };
|
||||
if (failing.length === 0) return { ...base, status: 'pass', detail: `no file above 5% over ${window.runs} main runs${left}` };
|
||||
return { ...base, status: 'fail',
|
||||
detail: failing.map(file => `${file.file}: flaky in ${file.flakyRuns}/${window.runs} main runs (${pct(file.rate)} > 5%)`).join('; ') };
|
||||
detail: failing.map(file => `${file.file}: flaky in ${file.flakyRuns}/${window.runs} main runs (${pct(file.rate)} > 5%)`).join('; ') + left };
|
||||
}
|
||||
|
||||
/** The reused main pushes a flake window left out and the runs it could not check, or ''. */
|
||||
function flakeWindowNotes(window: FlakeWindow): string {
|
||||
return `${window.skipped ? `; ${window.skipped} reused main push(es) left out` : ''}${window.note ? `; ${window.note}` : ''}`;
|
||||
}
|
||||
|
||||
export function unseededCheck(root: string): Check {
|
||||
@@ -286,9 +292,9 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
|
||||
metrics.push(...censusMetrics(inputs, artifactDirs));
|
||||
|
||||
const flakes = attempt(() => readCiFlakeLedgers({ client, repo, sinceDays, now, cacheDir }));
|
||||
metrics.push(metric('free-flaky-pass-rate', 'Free-suite flaky-pass rate', 'share of main free-tests runs (success or failure) with at least one flaky-pass ledger entry; per-file rate = runs with that file / runs', () => {
|
||||
metrics.push(metric('free-flaky-pass-rate', 'Free-suite flaky-pass rate', 'share of main free-tests runs (success or failure) with at least one flaky-pass ledger entry; per-file rate = runs with that file / runs; a main push that reused its pull request\'s run (no artifacts, `reuse` job green) is left out', () => {
|
||||
const window = need(flakes);
|
||||
return { value: `${pct(window.runs ? window.flakyRuns / window.runs : 0)} (${window.flakyRuns}/${window.runs} main runs since ${window.since.slice(0, 10)} needed the retry pass; ${window.files.length} file(s))`,
|
||||
return { value: `${pct(window.runs ? window.flakyRuns / window.runs : 0)} (${window.flakyRuns}/${window.runs} main runs since ${window.since.slice(0, 10)} needed the retry pass; ${window.files.length} file(s)${flakeWindowNotes(window)})`,
|
||||
detail: window.files.slice(0, 15).map(file => `${file.file}: ${file.flakyRuns}/${window.runs} runs (${pct(file.rate)})${file.failing ? ' FAIL > 5%' : ''}`) };
|
||||
}));
|
||||
|
||||
|
||||
@@ -44,12 +44,13 @@
|
||||
#
|
||||
# TTL and reaper: a name this script generates ends in -t<hours> (UBI_TTL_HOURS,
|
||||
# 1-999, default 3), the creator's own declaration of how long the VM may live.
|
||||
# `reap` destroys every VM in the project whose TTL expired more than ten
|
||||
# minutes ago, whichever owner made it, and this owner's own ubirun-<owner>-*
|
||||
# names with no TTL once they are UBI_LEGACY_HOURS old (default 4; another
|
||||
# owner's or an untagged TTL-less VM is never reaped); every `up` runs it first (UBI_REAP=0 skips it), so a runner
|
||||
# killed by a sleeping machine leaves nothing behind for longer than its TTL.
|
||||
# Names outside the ubirun- prefix are never reaped.
|
||||
# `reap` destroys only this owner's VMs: its ubirun-<owner>-* names whose TTL
|
||||
# expired more than ten minutes ago, and those with no TTL once they are
|
||||
# UBI_LEGACY_HOURS old (default 4). Another owner's VM, an untagged one and
|
||||
# names outside the ubirun- prefix are never reaped, expired or not; reap only
|
||||
# counts other owners' expired VMs in its log (cloud deletes stay owner-scoped,
|
||||
# GST-573). Every `up` runs it first (UBI_REAP=0 skips it), so a lane killed by
|
||||
# a sleeping machine has its leftovers destroyed by its own next `up`.
|
||||
#
|
||||
# Leftover line: `run` and `verify` end with
|
||||
# UBICLOUD: left=<0|1|unknown> owner=<owner> vm=<name> others=<n>
|
||||
@@ -754,23 +755,26 @@ vm_rows() {
|
||||
done
|
||||
}
|
||||
|
||||
# reap: destroy every VM whose name carries a TTL that ended more than
|
||||
# REAP_GRACE seconds ago, whichever owner created it (the TTL is that owner's
|
||||
# own declaration), and the caller's own TTL-less ubirun-<UBI_OWNER>-* names at
|
||||
# least LEGACY_HOURS old. Another owner's TTL-less VM, an untagged one, and names
|
||||
# outside the prefix are never touched: only a TTL lets reap judge another lane's VM.
|
||||
# reap: destroy the caller's own VMs whose TTL ended more than REAP_GRACE
|
||||
# seconds ago, and its TTL-less ubirun-<UBI_OWNER>-* names at least LEGACY_HOURS
|
||||
# old. Every other VM is never touched, whatever its TTL says: a delete stays
|
||||
# with the owner that created the VM. Other owners' expired VMs are only counted.
|
||||
cmd_reap() {
|
||||
local now rows loc name owner expires created n=0 failed=0
|
||||
local now rows loc name owner expires created n=0 failed=0 others=0
|
||||
[[ "$LEGACY_HOURS" =~ ^[1-9][0-9]{0,3}$ ]] || die "UBI_LEGACY_HOURS must be a whole number of hours from 1 to 9999 with no leading zero (how old a TTL-less ubirun-* VM must be before reap destroys it)"
|
||||
now=$(date +%s)
|
||||
rows=$(vm_rows) || { log "reap: could not list the project's VMs; nothing destroyed"; return 1; }
|
||||
while read -r loc name owner expires created; do
|
||||
[ -n "$name" ] || continue
|
||||
if [ "$owner" != "$OWNER" ]; then
|
||||
[ "$expires" = - ] || [ "$now" -lt $(( expires + REAP_GRACE )) ] || others=$(( others + 1 ))
|
||||
continue
|
||||
fi
|
||||
if [ "$expires" != - ]; then
|
||||
[ "$now" -ge $(( expires + REAP_GRACE )) ] || continue
|
||||
log "reaping expired $loc/$name (owner $owner; its TTL ended $(( (now - expires) / 60 )) min ago)"
|
||||
else
|
||||
[ -n "${UBI_OWNER:-}" ] && [ "$owner" = "$OWNER" ] && [ "$created" != - ] || continue
|
||||
[ -n "${UBI_OWNER:-}" ] && [ "$created" != - ] || continue
|
||||
[ $(( now - 10#$created )) -ge $(( LEGACY_HOURS * 3600 )) ] || continue
|
||||
log "reaping TTL-less $loc/$name (owner $owner; $(( (now - 10#$created) / 3600 ))h old, legacy sweep at ${LEGACY_HOURS}h)"
|
||||
fi
|
||||
@@ -778,6 +782,7 @@ cmd_reap() {
|
||||
cmd_down "$name" -l "$loc" || { failed=$(( failed + 1 )); log "failed to reap $loc/$name"; }
|
||||
done <<<"$rows"
|
||||
[ "$n" -gt 0 ] || log "reap: no expired VMs"
|
||||
[ "$others" -eq 0 ] || log "reap: left $others expired VMs of other owners; only their owners reap them"
|
||||
[ "$failed" -eq 0 ] || { log "reap: $failed of $n expired VMs could not be destroyed"; return 1; }
|
||||
}
|
||||
|
||||
@@ -1363,7 +1368,7 @@ cmd_run() {
|
||||
trap 'exit 130' INT TERM QUIT
|
||||
trap on_hup HUP
|
||||
if [[ "$name" =~ -t([0-9]+)$ ]]; then
|
||||
log "$name carries a ${BASH_REMATCH[1]}h TTL (any later run of this script reaps it); if this process dies first: $0 down $name"
|
||||
log "$name carries a ${BASH_REMATCH[1]}h TTL (this owner's next up or reap destroys it once it expires); if this process dies first: $0 down $name"
|
||||
elif [[ "$name" =~ ^$PREFIX-$OWNER-[0-9]{10}- ]]; then
|
||||
log "$name has no TTL (reap destroys it once it is ${LEGACY_HOURS}h old); if this process dies before its teardown line, destroy it with: $0 down $name"
|
||||
else
|
||||
@@ -1579,8 +1584,8 @@ usage: ubi-runner.sh <command> [args]
|
||||
owner print this caller's owner tag ($OWNER)
|
||||
records VMs this machine created and has not confirmed gone ($VM_RECORD)
|
||||
gc HOURS destroy this owner's VMs older than HOURS (never other owners')
|
||||
reap destroy every VM whose -t<hours> TTL has ended (any owner) and this
|
||||
owner's TTL-less ubirun-* VMs older than UBI_LEGACY_HOURS
|
||||
reap destroy this owner's VMs whose -t<hours> TTL has ended and its
|
||||
TTL-less ubirun-* VMs older than UBI_LEGACY_HOURS (never other owners')
|
||||
cli ARGS... raw Ubicloud CLI passthrough (e.g. cli vm list)
|
||||
|
||||
signals: INT and TERM tear the VM down. HUP, once a remote phase is live, detaches
|
||||
|
||||
Reference in new issue
Block a user