mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-06 00:44:07 +01:00
Merge origin/main (v1.91.267.0) into absorb-tstack-2
GSTA-36 rule-19 grant 2026-10-27. CHANGELOG via mergeChangelogs; lib/gstack-post.ts and its test keep both --replace (#3483) and --attribute; catalog ratchet re-measured on top of design-motion: 1,314 -> 1,359 (5,427 bytes = 1,357; modernize +178); context-budget.json modernize 9466 and alwaysOnTotal 7880 recaptured; bin/gstack-doctor-components.sh regenerated. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
commit
4cd14f90ae
145 files changed
+11148
-558
No files matched your search
@@ -295,6 +295,10 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
|
||||
file: 'test/ship-queue-train-cost.test.ts',
|
||||
reason: 'drives bin/gstack-ship-queue train verdict against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead of the shim (windows-free-shard 2 on #3478 at d5e6a7d)',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-queue-train-ops.test.ts',
|
||||
reason: 'drives bin/gstack-ship-queue (screen verdicts, scratch-merge screens, prune --stale, watch, forget, verify --post-merge, ready --dispatch-ci, --stack-tests) against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-policy-compat.test.ts',
|
||||
reason: 'spawns bin/gstack-ship-policy, bin/gstack-restamp and bin/gstack-tree-receipt (shebangs) through test/helpers/restamp-fixture.ts',
|
||||
|
||||
+85
-35
@@ -7,17 +7,27 @@
|
||||
* hotfix (--ref <sha on main>). Dry runs print the plan and change nothing.
|
||||
*
|
||||
* bun scripts/release-cut.ts --repo owner/name [--ref <sha>] [--dry-run] [--summary <file>]
|
||||
* [--required-check <name>] [--backstop-check <name>] [--notes-ref <ref>]
|
||||
*
|
||||
* The checkout needs origin's full history (fetch-depth 0). Green, for a
|
||||
* commit on main's first-parent history, means: its free-tests check run
|
||||
* succeeded (the status main requires), its main-backstop check run succeeded
|
||||
* (evals-main-backstop.yml publishes it: success only when a green train
|
||||
* record covers the tree or the backstop gated it green; a push main moved
|
||||
* past before it was gated is neutral, never green), every check run on it
|
||||
* has completed and none failed, and refs/notes/gstack-trains holds no active
|
||||
* freeze. The newest such commit in the window (7 days, 200 first-parent
|
||||
* commits) is the candidate; a commit with a check run still going is
|
||||
* skipped, not waited for.
|
||||
* commit on main's first-parent history, means: its required check run
|
||||
* succeeded (--required-check, default free-tests: the status main requires),
|
||||
* it was gated (see below), every check run on it has completed and none
|
||||
* failed, and the trains notes ref (--notes-ref, default
|
||||
* refs/notes/gstack-trains) holds no active freeze. The newest such commit in
|
||||
* the window (7 days, 200 first-parent commits) is the candidate; a commit
|
||||
* with a check run still going is skipped, not waited for.
|
||||
*
|
||||
* Gated means one of two kinds of evidence. Either the backstop check run
|
||||
* (--backstop-check, default main-backstop) succeeded: evals-main-backstop.yml
|
||||
* publishes it, success only when a green train record covers the tree or the
|
||||
* backstop gated it green, neutral when main moved past the push before it was
|
||||
* gated. Or a train record in the notes ref (kind train, state green, merging
|
||||
* or merged) lists the commit's sha or its tree in merged[]: those entries are
|
||||
* the main commits `gstack-ship-queue verify --train` confirmed, so a neutral
|
||||
* or absent backstop run on such a commit is not held against it, and the
|
||||
* CANDIDATE line says "gated by train <nonce>". A red backstop run stays red
|
||||
* either way. A commit with neither is not green.
|
||||
*
|
||||
* stable moves only forward: a candidate behind the current stable is refused
|
||||
* (REFUSED non-fast-forward), as is a tag v<VERSION> that already names another
|
||||
@@ -66,43 +76,74 @@ export function compareVersions(a: string, b: string): number {
|
||||
return 0;
|
||||
}
|
||||
|
||||
export type Verdict = { green: true } | { green: false; why: string; pending: boolean };
|
||||
/** Green names what gated the commit: the backstop check's name, or `train <nonce>`. Pending means a check run on it is still going. */
|
||||
export type Verdict = { green: true; via: string } | { green: false; why: string; pending: boolean };
|
||||
interface CheckRun { name: string; status: string; conclusion: string | null }
|
||||
|
||||
/** Why a commit is not releasable, or green. Pending means a check run on it is still going. */
|
||||
export function gateVerdict(gh: Gh, repo: string, sha: string): Verdict {
|
||||
/** Main commits a green, merging or merged train record lists in merged[], by sha and by tree, each naming the train. */
|
||||
export interface TrainEvidence { bySha: Map<string, string>; byTree: Map<string, string> }
|
||||
export interface GateOptions { requiredCheck?: string; backstopCheck?: string; evidence?: TrainEvidence; treeOf?: (sha: string) => string | null }
|
||||
|
||||
/** Why a commit is not releasable, or green and through which evidence. */
|
||||
export function gateVerdict(gh: Gh, repo: string, sha: string, o: GateOptions = {}): Verdict {
|
||||
const required = o.requiredCheck ?? REQUIRED_CHECK;
|
||||
const backstop = o.backstopCheck ?? BACKSTOP_CHECK;
|
||||
const paged = parseJson<unknown>(gh(['api', '--paginate', '--slurp', `repos/${repo}/commits/${sha}/check-runs?per_page=100`]));
|
||||
if (!paged) return { green: false, why: 'cannot read its check runs', pending: false };
|
||||
const pages = (Array.isArray(paged) ? paged : [paged]) as Array<{ check_runs?: CheckRun[] }>;
|
||||
const runs = pages.flatMap(p => p.check_runs ?? []);
|
||||
for (const name of [REQUIRED_CHECK, BACKSTOP_CHECK]) {
|
||||
let via = backstop;
|
||||
for (const name of [required, backstop]) {
|
||||
const named = runs.filter(c => c.name === name);
|
||||
if (named.length === 0) return { green: false, why: `no ${name} check run`, pending: false };
|
||||
if (named.some(c => c.status !== 'completed')) return { green: false, why: `${name} is still running`, pending: true };
|
||||
if (!named.some(c => c.conclusion === 'success')) return { green: false, why: `${name} concluded ${named.map(c => c.conclusion).join(',')}`, pending: false };
|
||||
if (named.some(c => c.conclusion === 'success')) continue;
|
||||
const train = name === backstop && name !== required && o.evidence ? o.evidence.bySha.get(sha) ?? o.evidence.byTree.get(o.treeOf?.(sha) ?? '') : undefined;
|
||||
if (train) { via = `train ${train}`; continue; }
|
||||
if (named.length === 0) return { green: false, why: `no ${name} check run`, pending: false };
|
||||
return { green: false, why: `${name} concluded ${named.map(c => c.conclusion).join(',')}`, pending: false };
|
||||
}
|
||||
const red = runs.filter(c => c.status === 'completed' && RED.has(c.conclusion ?? ''));
|
||||
if (red.length > 0) return { green: false, why: `red check runs: ${[...new Set(red.map(c => c.name))].join(', ')}`, pending: false };
|
||||
const running = runs.filter(c => c.status !== 'completed');
|
||||
if (running.length > 0) return { green: false, why: `check runs still going: ${[...new Set(running.map(c => c.name))].join(', ')}`, pending: true };
|
||||
return { green: true };
|
||||
return { green: true, via };
|
||||
}
|
||||
|
||||
/** An active freeze record (kind freeze, cleared_at null) in the trains notes ref, after fetching it. */
|
||||
export function activeFreeze(git: Git): string | null {
|
||||
const f = git(['fetch', '--no-tags', '-q', 'origin', `+${NOTES_REF}:${NOTES_LOCAL}`]);
|
||||
if (!f.ok) return /couldn't find remote ref|no such ref/i.test(f.out) ? null : `cannot fetch ${NOTES_REF}: ${f.out.slice(0, 160)}`;
|
||||
export interface TrainNotes { error: string | null; freeze: string | null; evidence: TrainEvidence }
|
||||
interface NoteRecord { kind?: string; state?: string; nonce?: string; merged?: Array<{ sha?: string; tree?: string }>; cleared_at?: string | null; reason?: string; train?: string }
|
||||
|
||||
/**
|
||||
* One read of the trains notes ref (fetched from origin): the active freeze
|
||||
* record (kind freeze, cleared_at null) and the train evidence (lib/
|
||||
* ship-queue-train-record.ts TrainRecord: kind train, state green, merging or
|
||||
* merged, merged[].sha and .tree). A fetch failure other than a missing ref is
|
||||
* an error; a missing ref holds nothing.
|
||||
*/
|
||||
export function readTrainNotes(git: Git, notesRef = NOTES_REF): TrainNotes {
|
||||
const notes: TrainNotes = { error: null, freeze: null, evidence: { bySha: new Map(), byTree: new Map() } };
|
||||
const f = git(['fetch', '--no-tags', '-q', 'origin', `+${notesRef}:${NOTES_LOCAL}`]);
|
||||
if (!f.ok) {
|
||||
if (!/couldn't find remote ref|no such ref/i.test(f.out)) notes.error = `cannot fetch ${notesRef}: ${f.out.slice(0, 160)}`;
|
||||
return notes;
|
||||
}
|
||||
const listed = git(['notes', `--ref=${NOTES_LOCAL}`, 'list']);
|
||||
if (!listed.ok || !listed.out.trim()) return null;
|
||||
if (!listed.ok || !listed.out.trim()) return notes;
|
||||
for (const line of listed.out.split('\n')) {
|
||||
const blob = line.split(' ')[0];
|
||||
if (!blob) continue;
|
||||
try {
|
||||
const r = JSON.parse(git(['cat-file', 'blob', blob]).out) as { kind?: string; cleared_at?: string | null; reason?: string; train?: string };
|
||||
if (r.kind === 'freeze' && !r.cleared_at) return `queue frozen since train ${r.train ?? '?'}: ${r.reason ?? 'red backstop'}`;
|
||||
} catch { /* not a record */ }
|
||||
let parsed: unknown;
|
||||
try { parsed = JSON.parse(git(['cat-file', 'blob', blob]).out); } catch { continue; }
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) continue;
|
||||
const r = parsed as NoteRecord;
|
||||
if (r.kind === 'freeze' && !r.cleared_at) notes.freeze = `queue frozen since train ${r.train ?? '?'}: ${r.reason ?? 'red backstop'}`;
|
||||
if (r.kind !== 'train' || typeof r.nonce !== 'string' || !r.nonce || !['green', 'merging', 'merged'].includes(r.state ?? '')) continue;
|
||||
for (const m of Array.isArray(r.merged) ? r.merged : []) {
|
||||
if (!m || typeof m !== 'object') continue;
|
||||
if (typeof m.sha === 'string' && m.sha) notes.evidence.bySha.set(m.sha, r.nonce);
|
||||
if (typeof m.tree === 'string' && m.tree) notes.evidence.byTree.set(m.tree, r.nonce);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
return notes;
|
||||
}
|
||||
|
||||
export interface ChangelogEntry { version: string; date: string; headline: string }
|
||||
@@ -158,13 +199,16 @@ export function releaseNotes(n: NotesInput): string {
|
||||
}
|
||||
|
||||
export interface Plan {
|
||||
sha: string; version: string; tag: string; date: string; hotfix: boolean;
|
||||
sha: string; version: string; tag: string; date: string; hotfix: boolean; via: string;
|
||||
prevStable: string | null; prevVersion: string | null; prevTag: string | null;
|
||||
tagExists: 'no' | 'same' | 'other'; stableAt: boolean; notes: string; skipped: string[];
|
||||
}
|
||||
export type PlanResult = { ok: true; plan: Plan } | { ok: false; code: 'NOTHING_NEW' | 'NO_GREEN_COMMIT' | 'REFUSED'; message: string; skipped?: string[] };
|
||||
|
||||
export interface PlanOptions { repo: string; ref?: string; git: Git; gh: Gh; now?: () => number; windowDays?: number; windowCommits?: number }
|
||||
export interface PlanOptions {
|
||||
repo: string; ref?: string; git: Git; gh: Gh; now?: () => number; windowDays?: number; windowCommits?: number;
|
||||
requiredCheck?: string; backstopCheck?: string; notesRef?: string;
|
||||
}
|
||||
|
||||
export function planRelease(o: PlanOptions): PlanResult {
|
||||
const { git, gh, repo } = o;
|
||||
@@ -174,8 +218,13 @@ export function planRelease(o: PlanOptions): PlanResult {
|
||||
if (!mainHead.ok) return { ok: false, code: 'REFUSED', message: 'origin/main is not fetched: ' + mainHead.out };
|
||||
const stable = git(['rev-parse', '--verify', '-q', 'refs/remotes/origin/stable^{commit}']);
|
||||
const prevStable = stable.ok ? stable.out : null;
|
||||
const frozen = activeFreeze(git);
|
||||
const trains = readTrainNotes(git, o.notesRef);
|
||||
const frozen = trains.error ?? trains.freeze;
|
||||
if (frozen) return { ok: false, code: 'REFUSED', message: frozen };
|
||||
const gate: GateOptions = {
|
||||
requiredCheck: o.requiredCheck, backstopCheck: o.backstopCheck, evidence: trains.evidence,
|
||||
treeOf: (sha) => { const t = git(['rev-parse', sha + '^{tree}']); return t.ok ? t.out : null; },
|
||||
};
|
||||
|
||||
let candidates: string[];
|
||||
if (o.ref) {
|
||||
@@ -194,9 +243,10 @@ export function planRelease(o: PlanOptions): PlanResult {
|
||||
// stable itself stays a candidate so a cut that stopped half-way (tag and
|
||||
// stable pushed, release missing) is finished; nothing older is looked at.
|
||||
let sha: string | null = null;
|
||||
let via = '';
|
||||
for (const c of candidates) {
|
||||
const v = gateVerdict(gh, repo, c);
|
||||
if (v.green) { sha = c; break; }
|
||||
const v = gateVerdict(gh, repo, c, gate);
|
||||
if (v.green) { sha = c; via = v.via; break; }
|
||||
skipped.push(c.slice(0, 12) + ': ' + v.why);
|
||||
if (prevStable && c === prevStable) { skipped.push(c.slice(0, 12) + ': already stable; nothing newer is green'); break; }
|
||||
}
|
||||
@@ -240,7 +290,7 @@ export function planRelease(o: PlanOptions): PlanResult {
|
||||
const changelog = git(['show', sha + ':CHANGELOG.md']);
|
||||
const date = git(['show', '-s', '--format=%cs', sha]).out;
|
||||
const notes = releaseNotes({ repo, version, date, prevVersion, prevTag, entries: changelog.ok ? changelogEntries(changelog.out) : [], hotfix: Boolean(o.ref), first: !before });
|
||||
return { ok: true, plan: { sha, version, tag, date, hotfix: Boolean(o.ref), prevStable, prevVersion, prevTag, tagExists, stableAt: prevStable === sha, notes, skipped } };
|
||||
return { ok: true, plan: { sha, version, tag, date, hotfix: Boolean(o.ref), via, prevStable, prevVersion, prevTag, tagExists, stableAt: prevStable === sha, notes, skipped } };
|
||||
}
|
||||
|
||||
export interface CutResult { ok: boolean; lines: string[] }
|
||||
@@ -283,7 +333,7 @@ export function cutRelease(p: Plan, o: { repo: string; git: Git; gh: Gh; notesDi
|
||||
|
||||
export function describePlan(p: Plan, dryRun: boolean): string[] {
|
||||
const lines = [
|
||||
'CANDIDATE: ' + p.sha + ' (' + p.date + ')' + (p.hotfix ? ' [hotfix: --ref]' : ''),
|
||||
'CANDIDATE: ' + p.sha + ' (' + p.date + ')' + (p.hotfix ? ' [hotfix: --ref]' : '') + ' gated by ' + p.via,
|
||||
'VERSION: ' + p.version + ' -> tag ' + p.tag + (p.tagExists === 'same' ? ' (exists at this commit)' : ''),
|
||||
'STABLE: ' + (p.stableAt ? 'already at ' + p.sha.slice(0, 12) + ' (finishing an earlier cut)' : (p.prevStable ? p.prevStable.slice(0, 12) + ' (v' + (p.prevVersion ?? '?') + ')' : 'none yet') + ' -> ' + p.sha.slice(0, 12)),
|
||||
'NOTES: ' + p.notes.split('\n').filter(l => l.startsWith('- ')).length + ' version lines, ' + p.notes.length + ' chars',
|
||||
@@ -307,13 +357,13 @@ if (import.meta.main) {
|
||||
const repo = arg('--repo');
|
||||
const dryRun = process.argv.includes('--dry-run');
|
||||
const summary = arg('--summary');
|
||||
if (!repo) { console.error('usage: release-cut.ts --repo owner/name [--ref <sha>] [--dry-run] [--summary <file>]'); process.exit(2); }
|
||||
if (!repo) { console.error('usage: release-cut.ts --repo owner/name [--ref <sha>] [--dry-run] [--summary <file>] [--required-check <name>] [--backstop-check <name>] [--notes-ref <ref>]'); process.exit(2); }
|
||||
const git = gitIn(process.cwd());
|
||||
const emit = (lines: string[]) => {
|
||||
for (const l of lines) console.log(l);
|
||||
if (summary) fs.appendFileSync(summary, lines.map(l => '- ' + l).join('\n') + '\n');
|
||||
};
|
||||
const planned = planRelease({ repo, ref: arg('--ref'), git, gh: runGh });
|
||||
const planned = planRelease({ repo, ref: arg('--ref'), git, gh: runGh, requiredCheck: arg('--required-check'), backstopCheck: arg('--backstop-check'), notesRef: arg('--notes-ref') });
|
||||
if (!planned.ok) {
|
||||
emit([planned.code + ': ' + planned.message, ...(planned.skipped ?? []).map(s => 'SKIPPED: ' + s)]);
|
||||
// A dry run that finds nothing green has done its job (the skips say why);
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
*
|
||||
* Derived in part from pbakaus/impeccable (Apache-2.0), modified. See NOTICE.md.
|
||||
*/
|
||||
import { DESIGN_SLOP_CATALOG, BANNED_FONTS, type DesignSlopEntry } from '../../lib/design-catalog';
|
||||
import { RENDERED, BANNED_FONTS, type DesignSlopEntry } from '../../lib/design-catalog';
|
||||
import { SENTINEL, DETECT_EXIT_ECHO } from '../../lib/design-detect-contract';
|
||||
|
||||
export const DESIGN_CHECKLIST_HEADER =
|
||||
@@ -24,17 +24,17 @@ export const DESIGN_CHECKLIST_SLOP_HEADING = 'AI Slop Detection';
|
||||
|
||||
const TIER_ORDER: Record<DesignSlopEntry['confidence'], number> = { HIGH: 0, MEDIUM: 1, LOW: 2 };
|
||||
|
||||
/** Category 1: slop entries a code reader can grep for, plus the legacy blacklist lines. */
|
||||
/** Category 1: rendered (non-checkerOnly) slop entries a code reader can grep for, plus the legacy blacklist lines. */
|
||||
export function checklistSlopEntries(): DesignSlopEntry[] {
|
||||
// Array.prototype.sort is stable, so catalog order survives within a tier.
|
||||
return DESIGN_SLOP_CATALOG
|
||||
return RENDERED
|
||||
.filter(e => e.kind === 'slop' && (e.detect.includes('grep') || e.legacyBlacklist))
|
||||
.sort((a, b) => TIER_ORDER[a.confidence] - TIER_ORDER[b.confidence]);
|
||||
}
|
||||
|
||||
/** Catalog rules a review may fix without asking: mechanical CSS changes with HIGH confidence. */
|
||||
export function autoFixEntries(): DesignSlopEntry[] {
|
||||
return DESIGN_SLOP_CATALOG.filter(e => e.tier === 'auto-fix');
|
||||
return RENDERED.filter(e => e.tier === 'auto-fix');
|
||||
}
|
||||
|
||||
function endsWithPunctuation(s: string): boolean {
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* Craft resolvers shared by /design-review's craft audit and /plan-design-review.
|
||||
*
|
||||
* {{DESIGN_CRAFT_RULE:<lead>}} renders one paragraph of design-html's craft
|
||||
* section verbatim, found by its bold lead (`**Ship floor.**`), with the
|
||||
* bullets under it, so the audit holds a page to the rules /design-html builds
|
||||
* it to and the two cannot drift. A renamed lead fails generation.
|
||||
*
|
||||
* {{DESIGN_PERSUADE_CHECKS}} is the five conversion checks, phrased to hold for
|
||||
* a built page and for a plan of one. Adapted from Elaya's landing-page-design
|
||||
* and redesign-skill (MIT); see NOTICE.md.
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import type { ResolverFn } from './types';
|
||||
|
||||
const CRAFT_SECTION = path.resolve(import.meta.dir, '../../design-html/sections/craft.md.tmpl');
|
||||
|
||||
export const DESIGN_CRAFT_RULE: ResolverFn = (_ctx, args) => {
|
||||
const lead = args?.join(':');
|
||||
if (!lead) throw new Error('{{DESIGN_CRAFT_RULE:<lead>}} requires the bold lead of a craft paragraph');
|
||||
const matches = fs.readFileSync(CRAFT_SECTION, 'utf-8').split(/\n{2,}/).filter(p => p.startsWith(`**${lead}.**`) || p.startsWith(`**${lead},`));
|
||||
if (matches.length !== 1) throw new Error(`{{DESIGN_CRAFT_RULE:${lead}}}: ${matches.length} paragraphs starting "**${lead}.**" or "**${lead}," in design-html/sections/craft.md.tmpl; need exactly one`);
|
||||
return matches[0].trimEnd();
|
||||
};
|
||||
|
||||
export const DESIGN_PERSUADE_CHECKS: ResolverFn = () => `1. **One primary action**, named; nothing competes with it above the fold, and the closing CTA repeats it.
|
||||
2. **Named objections**: the visitor's top objections, each answered where it arises (FAQ of real questions, comparison, guarantee).
|
||||
3. **Proof beside each claim**: a supplied customer, quote or figure next to the claim it supports, never a metric row; an unsupported claim is cut or marked \`[Needs proof: …]\`.
|
||||
4. **A risk reversal** the business really offers (free plan, trial, no card, cancel anytime, refund); never an invented one.
|
||||
5. **An index/noindex decision**: an ad-only or time-bound page is \`noindex\`; an indexed one has a real title and description, and its FAQ ships as FAQPage JSON-LD.`;
|
||||
@@ -39,6 +39,7 @@ import { generateDxFramework, generateDxErrorCompact, generateDxErrorItems } fro
|
||||
import { generateGBrainContextLoad, generateGBrainSaveResults, generateBrainPreflight, generateBrainCacheRefresh, generateBrainWriteBack } from './gbrain';
|
||||
import { generateTasksSectionEmit, generateTasksSectionAggregate } from './tasks-section';
|
||||
import { SECTION, SECTION_INDEX } from './sections';
|
||||
import { DESIGN_CRAFT_RULE, DESIGN_PERSUADE_CHECKS } from './design-craft';
|
||||
import { generateRedactInvocationBlock } from './redact-doc';
|
||||
import { generateFreeTextFile } from './free-text-file';
|
||||
import { FOREGROUND_DISPATCH_NOTE, codexSelect } from './constants';
|
||||
@@ -109,6 +110,8 @@ export const RESOLVERS: Record<string, ResolverFn> = {
|
||||
DESIGN_DETECTOR: generateDesignDetector,
|
||||
DESIGN_MD_CHECK: generateDesignMdCheck,
|
||||
DESIGN_SLOP_BULLETS: generateDesignSlopBullets,
|
||||
DESIGN_CRAFT_RULE,
|
||||
DESIGN_PERSUADE_CHECKS,
|
||||
UX_PRINCIPLES: generateUXPrinciples,
|
||||
DESIGN_OUTSIDE_VOICES: generateDesignOutsideVoices,
|
||||
DESIGN_REVIEW_LITE: generateDesignReviewLite,
|
||||
|
||||
@@ -25,9 +25,10 @@ const ROOT = path.resolve(import.meta.dir, '..', '..');
|
||||
|
||||
export const QA_ASSET_BLOCKER = 'If missing or unreadable, report a QA setup blocker and its affected probes as blocked; continue other safe probes (independent functional/static checks). Missing/unreadable assets block required QA.';
|
||||
|
||||
/** Skills carved on every host: QA (portable assets) and those whose external
|
||||
* render would exceed SKILL_BYTE_CEILING inlined (C4, #2777). */
|
||||
const CARVED_ON_EVERY_HOST = ['qa', 'qa-only', 'ship', 'plan-ceo-review', 'office-hours'];
|
||||
/** Skills carved on every host: QA (portable assets), those whose external
|
||||
* render would exceed SKILL_BYTE_CEILING inlined (C4, #2777), and design-review,
|
||||
* whose on-demand craft audit stays out of every skeleton (GSTA-92). */
|
||||
const CARVED_ON_EVERY_HOST = ['qa', 'qa-only', 'ship', 'plan-ceo-review', 'office-hours', 'design-review'];
|
||||
|
||||
/** Hosts read at most this many UTF-8 bytes of one SKILL.md (~40K tokens). */
|
||||
export const SKILL_BYTE_CEILING = 160_000;
|
||||
|
||||
+378
-26
@@ -23,7 +23,7 @@
|
||||
# Needs only bash, curl, python3, ssh, ssh-keygen, and tar locally, so it works
|
||||
# from dev boxes, containers, and cloud sandboxes.
|
||||
#
|
||||
# Ownership: VMs are named ubirun-<owner>-<epoch>-<suffix>. <owner> is UBI_OWNER
|
||||
# Ownership: VMs are named ubirun-<owner>-<epoch>-<suffix>[-t<hours>]. <owner> is UBI_OWNER
|
||||
# (lowercased, letters and digits only, at most 12 characters, starting with a
|
||||
# letter) or a random per-machine id kept in $STATE_ROOT/owner-id. Ubicloud names
|
||||
# allow 63 characters of [a-z0-9-], so a full name stays under 40.
|
||||
@@ -38,6 +38,29 @@
|
||||
# Stale-VM sweeps are off by default. With UBI_GC_HOURS set to a positive
|
||||
# number, every `up` first destroys this owner's VMs older than that; `gc HOURS`
|
||||
# runs the same sweep on demand. Neither ever touches another owner's VMs.
|
||||
#
|
||||
# TTL and reaper: a name this script generates ends in -t<hours> (UBI_TTL_HOURS,
|
||||
# default 4; 0 omits it), the creator's own declaration of how long the VM may
|
||||
# live. `reap` destroys every VM in the project whose TTL expired more than ten
|
||||
# minutes ago, whichever owner made it, and every `up` runs it first (UBI_REAP=0
|
||||
# skips it), so a runner killed by a sleeping machine leaves nothing behind for
|
||||
# longer than its TTL. Names without a TTL are never reaped.
|
||||
#
|
||||
# Quota preflight: when the project's vCPU ceiling is known (UBI_QUOTA_VCPUS, or
|
||||
# the maximum a refusal printed, cached in $STATE_ROOT/quota-max), `up` adds the
|
||||
# size's vCPUs to the project's running VMs and refuses before any create when
|
||||
# that exceeds it; UBI_QUOTA_WAIT=<seconds> waits for room instead.
|
||||
#
|
||||
# Watchdog: `run`, `verify` and `ssh NAME COMMAND` stream the remote output
|
||||
# through a watchdog that logs a heartbeat every UBI_HEARTBEAT seconds (300)
|
||||
# and kills a command silent for UBI_IDLE_TIMEOUT seconds (1800) or older than
|
||||
# UBI_RUN_TIMEOUT seconds (0, unlimited), exiting 124 with the last 40 lines,
|
||||
# so a hung verify fails fast instead of holding the VM until its TTL.
|
||||
#
|
||||
# `verify` is the one entry point for a repository's full remote verify: it
|
||||
# reads .gstack/ubicloud.json (setup, command, size, location, env, pass, pull)
|
||||
# and runs it through `run`, so lanes share one committed definition instead of
|
||||
# per-lane wrappers.
|
||||
set -euo pipefail
|
||||
# Keep heredoc bodies on temp files, not the pipe window (test/heredoc-pipe-deadlock.test.ts).
|
||||
BASH_COMPAT=50
|
||||
@@ -52,6 +75,14 @@ CREATE_GRACE="${UBI_CREATE_GRACE:-180}"
|
||||
DOWN_TIMEOUT="${UBI_DOWN_TIMEOUT:-900}"
|
||||
POLL="${UBI_POLL_SECONDS:-5}"
|
||||
ALLOWED_STORAGE="${UBI_ALLOWED_STORAGE:-320 640}"
|
||||
TTL_HOURS="${UBI_TTL_HOURS:-4}"
|
||||
REAP="${UBI_REAP:-1}"
|
||||
REAP_GRACE=600
|
||||
IDLE_TIMEOUT="${UBI_IDLE_TIMEOUT:-1800}"
|
||||
RUN_TIMEOUT="${UBI_RUN_TIMEOUT:-0}"
|
||||
HEARTBEAT="${UBI_HEARTBEAT:-300}"
|
||||
QUOTA_VCPUS="${UBI_QUOTA_VCPUS:-}"
|
||||
QUOTA_WAIT="${UBI_QUOTA_WAIT:-0}"
|
||||
PREFIX="ubirun"
|
||||
if [ -n "${UBI_VM_RECORD:-}" ]; then VM_RECORD="$UBI_VM_RECORD"
|
||||
elif [ -d "$HOME/.capy/work" ]; then VM_RECORD="$HOME/.capy/work/ubi-runner/vms.tsv"
|
||||
@@ -65,6 +96,11 @@ for bin in curl python3 ssh ssh-keygen tar; do
|
||||
done
|
||||
UBICLOUD_API_KEY="${UBICLOUD_API_KEY:-${UBICLOUD_API_TOKEN:-}}"
|
||||
[[ "$GC_HOURS" =~ ^[0-9]+$ ]] || die "UBI_GC_HOURS must be a whole number of hours (unset or 0 disables the sweep)"
|
||||
[[ "$TTL_HOURS" =~ ^(0|[1-9][0-9]{0,2})$ ]] || die "UBI_TTL_HOURS must be a whole number of hours from 0 to 999 with no leading zero (0 names VMs without a TTL)"
|
||||
for v in IDLE_TIMEOUT RUN_TIMEOUT HEARTBEAT QUOTA_WAIT; do
|
||||
[[ "${!v}" =~ ^[0-9]+$ ]] || die "UBI_$v must be a whole number of seconds (0 disables it)"
|
||||
done
|
||||
[ -z "$QUOTA_VCPUS" ] || [[ "$QUOTA_VCPUS" =~ ^[0-9]+$ ]] || die "UBI_QUOTA_VCPUS must be a whole number of vCPUs"
|
||||
|
||||
owner_id() {
|
||||
local raw=${UBI_OWNER:-} file="$STATE_ROOT/owner-id" tmp
|
||||
@@ -141,18 +177,134 @@ remote() {
|
||||
ssh $(ssh_opts) "ubi@$IP" "umask 022; $*"
|
||||
}
|
||||
|
||||
# vm_rows: "location name owner" for every VM in the project; owner is
|
||||
# "-" for untagged ubirun-<epoch>-* names and "." for VMs this runner did not name.
|
||||
vm_rows() {
|
||||
cli vm list -N -f location,name | while read -r loc name; do
|
||||
if [[ "$name" =~ ^${PREFIX}-([a-z][a-z0-9]{0,11})-[0-9]{10}- ]]; then
|
||||
echo "$loc $name ${BASH_REMATCH[1]}"
|
||||
elif [[ "$name" =~ ^${PREFIX}-[0-9]{10}- ]]; then
|
||||
echo "$loc $name -"
|
||||
else
|
||||
echo "$loc $name ."
|
||||
mtime() { stat -c %Y "$1" 2>/dev/null || stat -f %m "$1"; }
|
||||
|
||||
# sq STRING: STRING as one single-quoted shell word.
|
||||
sq() { printf "'%s'" "$(printf %s "$1" | sed "s/'/'\\\\''/g")"; }
|
||||
|
||||
# remote_cancel NAME SIDFILE: end the session a watched command started on the
|
||||
# VM (its leader pid is in SIDFILE there), bounded to 20 seconds. Killing the
|
||||
# local ssh client alone leaves a silent remote command running until it next
|
||||
# writes to the closed channel.
|
||||
remote_cancel() {
|
||||
local name=$1 sidf=$2 cpid i=0
|
||||
remote "$name" "sid=\$(cat $sidf 2>/dev/null); rm -f $sidf; [ -z \"\$sid\" ] || { pkill -TERM -s \"\$sid\"; sleep 2; pkill -KILL -s \"\$sid\"; }; true" >/dev/null 2>&1 </dev/null &
|
||||
cpid=$!
|
||||
while kill -0 "$cpid" 2>/dev/null && [ "$i" -lt 20 ]; do sleep 1; i=$(( i + 1 )); done
|
||||
kill "$cpid" 2>/dev/null || true
|
||||
wait "$cpid" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# watched NAME PHASE COMMAND: run COMMAND on the VM, streaming its output and
|
||||
# keeping a copy of both streams in the VM's state dir, with a heartbeat every HEARTBEAT
|
||||
# seconds and a watchdog that kills it after IDLE_TIMEOUT seconds of silence or
|
||||
# RUN_TIMEOUT seconds in total: exit 124, reason and the last 40 lines on
|
||||
# stderr. The ssh runs in its own session (python3's setsid) so the watchdog
|
||||
# kills it and anything it started, not just the shell that launched it; on
|
||||
# the VM the command runs in its own session too (setsid where the image has
|
||||
# it), whose leader pid is written to ~/.ubirun-PHASE-<runner pid>.sid so
|
||||
# remote_cancel can end it when the client is killed.
|
||||
watched() {
|
||||
local name=$1 phase=$2 cmd=$3 out pid rc=0 start now last_beat idle elapsed why="" old_traps inner sidf
|
||||
sidf="~/.ubirun-$phase-$$.sid"
|
||||
load "$name"
|
||||
out="$(state_dir "$name")/$phase.log"
|
||||
: >"$out"
|
||||
inner=$(sq "printf %s \$\$ >$sidf; $cmd")
|
||||
# The sid file outlives the command on purpose: a background child it left
|
||||
# behind keeps the ssh channel open after the wrapper exits, and remote_cancel
|
||||
# still needs the session id to end it. remote_cancel removes the file.
|
||||
cmd="if command -v setsid >/dev/null 2>&1; then setsid -w bash -c $inner; else bash -c $inner; fi"
|
||||
# The caller's stdin is handed to the background ssh explicitly (a background
|
||||
# job in a non-interactive shell would otherwise read /dev/null), so piped
|
||||
# input reaches the remote command as it did before the watchdog.
|
||||
exec 3<&0
|
||||
# shellcheck disable=SC2046
|
||||
( exec python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
|
||||
ssh $(ssh_opts) "ubi@$IP" "umask 022; $cmd" ) <&3 > >(tee -a "$out") 2> >(tee -a "$out" >&2) &
|
||||
pid=$!
|
||||
exec 3<&-
|
||||
# A signal to this process takes the ssh session down with it, so Ctrl-C on
|
||||
# `ssh NAME COMMAND` leaves no orphaned client behind.
|
||||
old_traps=$(trap -p INT TERM)
|
||||
trap 'kill -TERM -- "-$pid" 2>/dev/null; exit 130' INT TERM
|
||||
start=$(date +%s)
|
||||
last_beat=$start
|
||||
while :; do
|
||||
sleep 1
|
||||
kill -0 "$pid" 2>/dev/null || break
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - start ))
|
||||
idle=$(( now - $(mtime "$out") ))
|
||||
if [ "$IDLE_TIMEOUT" -gt 0 ] && [ "$idle" -ge "$IDLE_TIMEOUT" ]; then
|
||||
why="printed nothing for ${idle}s (UBI_IDLE_TIMEOUT=$IDLE_TIMEOUT)"
|
||||
elif [ "$RUN_TIMEOUT" -gt 0 ] && [ "$elapsed" -ge "$RUN_TIMEOUT" ]; then
|
||||
why="ran ${elapsed}s (UBI_RUN_TIMEOUT=$RUN_TIMEOUT)"
|
||||
fi
|
||||
if [ -n "$why" ]; then
|
||||
# The command may have finished while the clock was read; a finished
|
||||
# command is never reported as a timeout.
|
||||
kill -0 "$pid" 2>/dev/null || break
|
||||
log "watchdog: $phase $why; killing it. Last output:"
|
||||
tail -n 40 "$out" >&2
|
||||
kill -TERM -- "-$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true
|
||||
wait "$pid" 2>/dev/null || rc=$?
|
||||
trap - INT TERM; eval "$old_traps"
|
||||
# A command killed by the watchdog never exits 0; a 0 here means it
|
||||
# finished in the instant before the kill, and that result stands.
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
log "watchdog: $phase finished before it could be killed; its exit 0 stands"
|
||||
return 0
|
||||
fi
|
||||
remote_cancel "$name" "$sidf"
|
||||
return 124
|
||||
fi
|
||||
if [ "$HEARTBEAT" -gt 0 ] && [ $(( now - last_beat )) -ge "$HEARTBEAT" ]; then
|
||||
log "heartbeat: $phase ${elapsed}s elapsed, last output ${idle}s ago"
|
||||
last_beat=$now
|
||||
fi
|
||||
done
|
||||
wait "$pid" || rc=$?
|
||||
trap - INT TERM; eval "$old_traps"
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# vm_rows: "location name owner expires" for every VM in the project; owner is
|
||||
# "-" for untagged ubirun-<epoch>-* names and "." for VMs this runner did not
|
||||
# name; expires is the epoch the name's -t<hours> TTL ends at, or "-".
|
||||
vm_rows() {
|
||||
local loc name owner created expires
|
||||
cli vm list -N -f location,name | while read -r loc name; do
|
||||
owner=. created="" expires=-
|
||||
if [[ "$name" =~ ^${PREFIX}-([a-z][a-z0-9]{0,11})-([0-9]{10})- ]]; then
|
||||
owner=${BASH_REMATCH[1]} created=${BASH_REMATCH[2]}
|
||||
elif [[ "$name" =~ ^${PREFIX}-([0-9]{10})- ]]; then
|
||||
owner=- created=${BASH_REMATCH[1]}
|
||||
fi
|
||||
if [ -n "$created" ] && [[ "$name" =~ -t([0-9]{1,3})$ ]]; then
|
||||
expires=$(( 10#$created + 10#${BASH_REMATCH[1]} * 3600 ))
|
||||
fi
|
||||
echo "$loc $name $owner $expires"
|
||||
done
|
||||
}
|
||||
|
||||
# reap: destroy every VM whose name carries a TTL that ended more than
|
||||
# REAP_GRACE seconds ago, whichever owner created it (the TTL is that owner's
|
||||
# own declaration). Names without a TTL are never touched.
|
||||
cmd_reap() {
|
||||
local now rows loc name owner expires n=0 failed=0
|
||||
now=$(date +%s)
|
||||
rows=$(vm_rows) || { log "reap: could not list the project's VMs; nothing destroyed"; return 1; }
|
||||
while read -r loc name owner expires; do
|
||||
[ -n "$name" ] || continue
|
||||
[ "$expires" != - ] || continue
|
||||
[ "$now" -ge $(( expires + REAP_GRACE )) ] || continue
|
||||
n=$(( n + 1 ))
|
||||
log "reaping expired $loc/$name (owner $owner; its TTL ended $(( (now - expires) / 60 )) min ago)"
|
||||
cmd_down "$name" -l "$loc" || { failed=$(( failed + 1 )); log "failed to reap $loc/$name"; }
|
||||
done <<<"$rows"
|
||||
[ "$n" -gt 0 ] || log "reap: no expired VMs"
|
||||
[ "$failed" -eq 0 ] || { log "reap: $failed of $n expired VMs could not be destroyed"; return 1; }
|
||||
}
|
||||
|
||||
cmd_gc() {
|
||||
@@ -160,7 +312,7 @@ cmd_gc() {
|
||||
[[ "$hours" =~ ^[0-9]+$ ]] || die "gc: HOURS must be a whole number (or set UBI_GC_HOURS)"
|
||||
if [ "$hours" -eq 0 ]; then log "gc: no sweep (give HOURS > 0 or set UBI_GC_HOURS); nothing listed, nothing destroyed"; return 0; fi
|
||||
now=$(date +%s)
|
||||
vm_rows | while read -r loc name owner; do
|
||||
vm_rows | while read -r loc name owner _; do
|
||||
[ "$owner" = "$OWNER" ] || continue
|
||||
[[ "$name" =~ -([0-9]{10})- ]] || continue
|
||||
if [ $(( (now - BASH_REMATCH[1]) / 3600 )) -ge "$hours" ]; then
|
||||
@@ -182,7 +334,7 @@ cmd_list() {
|
||||
cmd_usage() {
|
||||
local tmp loc name owner i=0
|
||||
tmp=$(mktemp -d)
|
||||
while read -r loc name owner; do
|
||||
while read -r loc name owner _; do
|
||||
i=$(( i + 1 ))
|
||||
cli vm "$loc/$name" show 2>/dev/null | sed -n "s/^size: /$owner /p" >"$tmp/$i" &
|
||||
[ $(( i % 8 )) -ne 0 ] || wait
|
||||
@@ -226,7 +378,49 @@ cmd_records() {
|
||||
awk -F'\t' '{ printf "%-40s %-14s %-12s %-20s %s\n", $1, $2, $3, $4, $5 }' "$VM_RECORD"
|
||||
}
|
||||
|
||||
new_name() { echo "$PREFIX-$OWNER-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')"; }
|
||||
new_name() {
|
||||
local ttl=""
|
||||
[ "$TTL_HOURS" -eq 0 ] || ttl="-t$TTL_HOURS"
|
||||
echo "$PREFIX-$OWNER-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')$ttl"
|
||||
}
|
||||
|
||||
# vcpus SIZE: the vCPU count a Ubicloud size name carries (standard-16 -> 16).
|
||||
vcpus() { [[ "$1" =~ -([0-9]+)$ ]] && echo "${BASH_REMATCH[1]}"; }
|
||||
|
||||
# quota_preflight LOCATION NAME SIZE: refuse (or wait UBI_QUOTA_WAIT seconds)
|
||||
# before the create request when the project's running VMs plus this size
|
||||
# exceed the known vCPU ceiling. The ceiling comes from UBI_QUOTA_VCPUS or the
|
||||
# maximum the last refusal printed; without either the API's own refusal rules.
|
||||
QUOTA_CACHE_MAX_AGE=86400
|
||||
quota_preflight() {
|
||||
local location=$1 name=$2 size=$3 need max used wait_until remaining source="UBI_QUOTA_VCPUS" cache="$STATE_ROOT/quota-max"
|
||||
need=$(vcpus "$size") || return 0
|
||||
max=$QUOTA_VCPUS
|
||||
if [ -z "$max" ] && [ -f "$cache" ]; then
|
||||
# A refusal's maximum is remembered for a day: after that (a raised quota,
|
||||
# another project's key) the API's own answer decides again.
|
||||
if [ $(( $(date +%s) - $(mtime "$cache") )) -lt "$QUOTA_CACHE_MAX_AGE" ]; then
|
||||
max=$(cat "$cache" 2>/dev/null || true); source="the last refusal, cached in $cache"
|
||||
else
|
||||
rm -f "$cache"
|
||||
fi
|
||||
fi
|
||||
[[ "$max" =~ ^[0-9]+$ ]] || return 0
|
||||
wait_until=$(( $(date +%s) + QUOTA_WAIT ))
|
||||
while :; do
|
||||
used=$(cmd_usage 2>/dev/null | awk '$1=="total" {print $3}')
|
||||
[[ "$used" =~ ^[0-9]+$ ]] || return 0
|
||||
[ $(( used + need )) -gt "$max" ] || return 0
|
||||
remaining=$(( wait_until - $(date +%s) ))
|
||||
if [ "$remaining" -le 0 ]; then
|
||||
log "quota preflight refused $location/$name ($size): it needs $need vCPUs and the project's VMs already use $used of $max (ceiling from $source; other usage such as managed runners counts on top)"
|
||||
cmd_usage >&2 || true
|
||||
die "create refused before any request: vCPU quota exhausted (quota full); wait for running VMs to finish (UBI_QUOTA_WAIT=<seconds> waits here), run $0 reap, use a smaller size, or set UBI_QUOTA_VCPUS to the project's real ceiling"
|
||||
fi
|
||||
log "quota: $used of $max vCPUs in use and $need needed; waiting up to ${remaining}s more for room"
|
||||
sleep $(( remaining < 30 ? remaining : 30 ))
|
||||
done
|
||||
}
|
||||
|
||||
valid_name() { [[ "$1" =~ ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ ]]; }
|
||||
|
||||
@@ -254,9 +448,13 @@ cmd_up() {
|
||||
done
|
||||
[ -n "$name" ] || name=$(new_name)
|
||||
valid_name "$name" || die "invalid VM name '$name' (Ubicloud allows 1-63 of a-z, 0-9 and '-', starting and ending with a letter or digit)"
|
||||
if [ "$REAP" != 0 ]; then
|
||||
cmd_reap || log "reap failed; continuing"
|
||||
fi
|
||||
if [ "$GC_HOURS" -gt 0 ]; then
|
||||
cmd_gc "$GC_HOURS" || log "stale-VM cleanup failed; continuing"
|
||||
fi
|
||||
quota_preflight "$location" "$name" "$size"
|
||||
|
||||
local dir create_at
|
||||
dir=$(state_dir "$name")
|
||||
@@ -287,6 +485,7 @@ cmd_up() {
|
||||
die "create failed"
|
||||
fi
|
||||
read -r need used max <<<"$refusal"
|
||||
echo "$max" >"$STATE_ROOT/quota-max"
|
||||
log "quota refused $location/$name ($size): it needs $need vCPUs and the project already uses $used of $max"
|
||||
log "VMs in the project by owner (the rest of the used count is other usage, such as managed GitHub runners):"
|
||||
cmd_usage >&2 || true
|
||||
@@ -389,18 +588,34 @@ cmd_down() {
|
||||
|
||||
# pack SRC: write a gzipped checkout tarball to stdout (tracked + untracked
|
||||
# files that are not ignored, plus .git; a non-git directory is taken whole).
|
||||
# A linked worktree's .git is a one-line pointer into the main repository,
|
||||
# useless on the VM, so it is replaced by a self-contained clone of the
|
||||
# checked-out branch carrying the worktree's own index.
|
||||
cmd_pack() {
|
||||
local src
|
||||
local src tmp="" gitdir branch
|
||||
src=$(cd "${1:-.}" && pwd)
|
||||
if git -C "$src" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
if [ -f "$src/.git" ]; then
|
||||
branch=$(git -C "$src" rev-parse --abbrev-ref HEAD)
|
||||
[ "$branch" != HEAD ] || die "pack: $src is a linked worktree on a detached HEAD; check out a branch so the VM gets a self-contained .git"
|
||||
tmp=$(mktemp -d)
|
||||
gitdir=$(cd "$src" && cd "$(git rev-parse --git-common-dir)" && pwd)
|
||||
git clone -q --bare --no-local --single-branch --branch "$branch" "$gitdir" "$tmp/.git"
|
||||
git -C "$tmp/.git" config core.bare false
|
||||
# The worktree's index rides along so the VM's status matches, except a
|
||||
# split index, whose shared half lives only in the original git dir.
|
||||
[ "$(git -C "$src" config --bool core.splitIndex 2>/dev/null)" = true ] \
|
||||
|| cp "$(cd "$src" && cd "$(dirname "$(git rev-parse --git-path index)")" && pwd)/index" "$tmp/.git/index"
|
||||
fi
|
||||
(
|
||||
cd "$src"
|
||||
{ git ls-files -z -co --exclude-standard; printf '.git\0'; } \
|
||||
{ git ls-files -z -co --exclude-standard; [ -n "$tmp" ] || printf '.git\0'; } \
|
||||
| while IFS= read -r -d '' f; do
|
||||
if [ -e "$f" ] || [ -L "$f" ]; then printf '%s\0' "$f"; fi
|
||||
done \
|
||||
| tar --null -T - -czf -
|
||||
| if [ -n "$tmp" ]; then tar --null -T - -c -C "$tmp" .git -zf -; else tar --null -T - -czf -; fi
|
||||
)
|
||||
[ -z "$tmp" ] || rm -rf "$tmp"
|
||||
else
|
||||
tar -C "$src" -czf - .
|
||||
fi
|
||||
@@ -435,6 +650,34 @@ cmd_pull() {
|
||||
remote "$name" "cd $dir && tar -czf - $(basename "$from")" | tar -xzf - -C "$to"
|
||||
}
|
||||
|
||||
# pass_name NAME: a variable this runner may forward to a VM. Its own
|
||||
# credentials (UBICLOUD_*) never leave this machine, whoever asks.
|
||||
pass_name() {
|
||||
[[ "$1" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || die "--pass $1: not a variable name"
|
||||
case $1 in UBICLOUD_*) die "--pass $1: the runner never forwards its own Ubicloud credentials to a VM" ;; esac
|
||||
}
|
||||
|
||||
# inside_dir DIR ROOT: DIR resolves, symlinks followed, to ROOT or below it.
|
||||
# Its nearest existing ancestor is checked first, so a refused destination
|
||||
# creates nothing (not even outside the root); only then is DIR created and
|
||||
# checked again.
|
||||
inside_dir() {
|
||||
local a b near=$1
|
||||
b=$(cd "$2" 2>/dev/null && pwd -P) || return 1
|
||||
while [ ! -d "$near" ]; do
|
||||
[ "$near" != "$(dirname "$near")" ] || return 1
|
||||
near=$(dirname "$near")
|
||||
done
|
||||
a=$(cd "$near" 2>/dev/null && pwd -P) || return 1
|
||||
[ "$a" = "$b" ] || [[ "$a" = "$b"/* ]] || return 1
|
||||
mkdir -p "$1" 2>/dev/null || return 1
|
||||
a=$(cd "$1" 2>/dev/null && pwd -P) || return 1
|
||||
[ "$a" = "$b" ] || [[ "$a" = "$b"/* ]]
|
||||
}
|
||||
|
||||
# PULL_ROOT: set by verify; every pull destination must resolve inside it at
|
||||
# pull time (an earlier pull may have placed a symlink where a later one lands).
|
||||
PULL_ROOT=""
|
||||
cmd_run() {
|
||||
local up_args=() src=. dest="" setup="" keep=0 pulls=() envs=()
|
||||
while [ $# -gt 0 ]; do
|
||||
@@ -445,7 +688,7 @@ cmd_run() {
|
||||
--dest) dest=$2; shift 2 ;;
|
||||
--setup) setup=$2; shift 2 ;;
|
||||
--env) envs+=("$2"); shift 2 ;;
|
||||
--pass) [ -n "${!2+x}" ] || die "--pass $2: not set locally"; envs+=("$2=${!2}"); shift 2 ;;
|
||||
--pass) pass_name "$2"; [ -n "${!2+x}" ] || die "--pass $2: not set locally"; envs+=("$2=${!2}"); shift 2 ;;
|
||||
--pull) pulls+=("$2"); shift 2 ;;
|
||||
--keep) keep=1; shift ;;
|
||||
--) shift; break ;;
|
||||
@@ -468,6 +711,11 @@ cmd_run() {
|
||||
trap 'trap "" INT TERM HUP QUIT; cmd_down "$RUN_VM" || log "WARNING: failed to destroy $RUN_VM; run: $0 down $RUN_VM"' EXIT
|
||||
trap 'exit 130' INT TERM HUP QUIT
|
||||
cmd_up "${up_args[@]}" >/dev/null
|
||||
if [[ "$name" =~ -t([0-9]+)$ ]]; then
|
||||
log "$name carries a ${BASH_REMATCH[1]}h TTL (any later run of this script reaps it); if this process dies first: $0 down $name"
|
||||
else
|
||||
log "$name has no TTL; if this process dies before its teardown line, destroy it with: $0 down $name"
|
||||
fi
|
||||
if [ "$keep" = 1 ]; then
|
||||
trap - EXIT
|
||||
log "--keep: leaving $name running; destroy with: $0 down $name"
|
||||
@@ -478,27 +726,118 @@ cmd_run() {
|
||||
| remote "$name" 'umask 077 && cat > ~/.ubirun-env'
|
||||
|
||||
cmd_sync "$name" "$src" "$dest"
|
||||
local qdest
|
||||
local qdest rc=0
|
||||
qdest=$(printf %q "$dest")
|
||||
if [ -n "$setup" ]; then
|
||||
remote "$name" 'cat > ~/.ubirun-setup.sh' <"$setup"
|
||||
log "running setup $(basename "$setup")"
|
||||
remote "$name" "cd $qdest && . ~/.ubirun-env && bash -l ~/.ubirun-setup.sh" || die "setup failed"
|
||||
watched "$name" setup "cd $qdest && . ~/.ubirun-env && bash -l ~/.ubirun-setup.sh" || { rc=$?; log "setup failed (exit $rc)"; exit "$rc"; }
|
||||
fi
|
||||
|
||||
local start rc=0
|
||||
local start
|
||||
start=$(date +%s)
|
||||
log "running on $name: $*"
|
||||
remote "$name" "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "$*")" || rc=$?
|
||||
watched "$name" command "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "$*")" || rc=$?
|
||||
log "command exited $rc after $(( $(date +%s) - start ))s"
|
||||
|
||||
local p
|
||||
for p in ${pulls[@]+"${pulls[@]}"}; do
|
||||
if [ -n "$PULL_ROOT" ] && ! inside_dir "${p#*:}" "$PULL_ROOT"; then
|
||||
log "pull refused: ${p#*:} resolves outside $PULL_ROOT"
|
||||
[ "$rc" -ne 0 ] || rc=1
|
||||
continue
|
||||
fi
|
||||
cmd_pull "$name" "${p%%:*}" "${p#*:}" || log "pull failed: $p"
|
||||
done
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# verify [run opts] [--trust-config] [-- COMMAND]: run the repository's
|
||||
# committed remote verify. .gstack/ubicloud.json under --src (default .)
|
||||
# supplies setup, command, size, location, storage, image, env, pass and pull;
|
||||
# flags given here and a trailing -- COMMAND override it. Without the file it
|
||||
# refuses, so every lane runs the one committed definition instead of its own
|
||||
# wrapper.
|
||||
#
|
||||
# The file is repository content and gets no reach beyond the checkout on its
|
||||
# own: its setup script and pull destinations must resolve inside --src, and a
|
||||
# local variable it lists under pass is forwarded only when this command line
|
||||
# also names it (--pass NAME) or grants every listed name (--trust-config).
|
||||
cmd_verify() {
|
||||
local src=. i j g n cfg cfg_setup cfg_cmd parsed err rc=0 trust=0 cfg_args=() cfg_pass=() user_pre=() user_cmd=() granted=() missing=()
|
||||
for (( i = 1; i <= $#; i++ )); do
|
||||
[ "${!i}" = -- ] && break
|
||||
if [ "${!i}" = --src ]; then j=$(( i + 1 )); src=${!j}; fi
|
||||
done
|
||||
cfg="$src/.gstack/ubicloud.json"
|
||||
[ -f "$cfg" ] || die "verify: no $cfg; commit one ({\"setup\": \"scripts/ubicloud/bootstrap.sh\", \"command\": \"bash scripts/ubicloud/verify.sh\"}) so every lane runs the same verify instead of its own wrapper"
|
||||
err=$(mktemp)
|
||||
parsed=$(python3 -c '
|
||||
import json, os, shlex, sys
|
||||
cfg, src = sys.argv[1], sys.argv[2]
|
||||
try:
|
||||
c = json.load(open(cfg))
|
||||
except ValueError:
|
||||
sys.exit(3)
|
||||
if not isinstance(c, dict):
|
||||
sys.exit(3)
|
||||
base, root = os.path.abspath(src), os.path.realpath(src)
|
||||
def inside(rel, what):
|
||||
if not isinstance(rel, str) or not rel or os.path.isabs(rel) or rel.startswith("~") or ".." in rel.replace("\\", "/").split("/"):
|
||||
print("%s %r must be a relative path inside the checkout" % (what, rel), file=sys.stderr); sys.exit(2)
|
||||
full = os.path.realpath(os.path.join(root, rel))
|
||||
if full != root and not full.startswith(root + os.sep):
|
||||
print("%s %r resolves outside the checkout (%s)" % (what, rel, full), file=sys.stderr); sys.exit(2)
|
||||
return os.path.join(base, rel)
|
||||
q = lambda s: shlex.quote(str(s))
|
||||
args = []
|
||||
for key, flag in (("size", "-s"), ("location", "-l"), ("storage", "-S"), ("image", "-b")):
|
||||
if key in c: args += [flag, q(c[key])]
|
||||
for item in c.get("env", []): args += ["--env", q(item)]
|
||||
for item in c.get("pull", []):
|
||||
remote, sep, local = str(item).partition(":")
|
||||
if not sep or not remote:
|
||||
print("pull entry %r must be REMOTE_GLOB:LOCAL_DIR" % (item,), file=sys.stderr); sys.exit(2)
|
||||
args += ["--pull", q(remote + ":" + inside(local, "pull destination"))]
|
||||
setup = c.get("setup", "")
|
||||
print("cfg_args=(" + " ".join(args) + ")")
|
||||
print("cfg_pass=(" + " ".join(q(x) for x in c.get("pass", [])) + ")")
|
||||
print("cfg_setup=" + q(inside(setup, "setup") if setup else ""))
|
||||
print("cfg_cmd=" + q(c.get("command", "")))
|
||||
' "$cfg" "$src" 2>"$err") || rc=$?
|
||||
case $rc in
|
||||
0) ;;
|
||||
2) local why; why=$(cat "$err"); rm -f "$err"; die "verify: $cfg: $why" ;;
|
||||
*) rm -f "$err"; die "verify: $cfg is not valid JSON" ;;
|
||||
esac
|
||||
rm -f "$err"
|
||||
eval "$parsed"
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
--) shift; user_cmd=("$@"); break ;;
|
||||
--trust-config) trust=1; shift ;;
|
||||
--keep) user_pre+=("$1"); shift ;;
|
||||
--pass|-S|--storage|-s|--size|-l|--location|-n|--name|-b|--image|--src|--dest|--setup|--env|--pull)
|
||||
[ $# -ge 2 ] || die "verify: $1 needs a value"
|
||||
[ "$1" != --pass ] || granted+=("$2")
|
||||
user_pre+=("$1" "$2"); shift 2 ;;
|
||||
*) die "verify: unknown option $1" ;;
|
||||
esac
|
||||
done
|
||||
for n in ${cfg_pass[@]+"${cfg_pass[@]}"}; do
|
||||
pass_name "$n"
|
||||
if [ "$trust" = 1 ]; then cfg_args+=(--pass "$n"); continue; fi
|
||||
for g in ${granted[@]+"${granted[@]}"}; do [ "$g" = "$n" ] && continue 2; done
|
||||
missing+=("$n")
|
||||
done
|
||||
[ ${#missing[@]} -eq 0 ] || die "verify: $cfg asks to forward local ${missing[*]}; grant each on this command line with --pass NAME, or every name it lists with --trust-config"
|
||||
[ ${#user_cmd[@]} -gt 0 ] || [ -n "$cfg_cmd" ] || die "verify: $cfg names no command and none was given after --"
|
||||
[ ${#user_cmd[@]} -gt 0 ] || user_cmd=("$cfg_cmd")
|
||||
log "verify: $cfg"
|
||||
PULL_ROOT=$(cd "$src" && pwd)
|
||||
cmd_run ${cfg_args[@]+"${cfg_args[@]}"} ${cfg_setup:+--setup "$cfg_setup"} --src "$src" ${user_pre[@]+"${user_pre[@]}"} -- "${user_cmd[@]}"
|
||||
}
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
usage: ubi-runner.sh <command> [args]
|
||||
@@ -506,6 +845,11 @@ usage: ubi-runner.sh <command> [args]
|
||||
run [up opts] [--src DIR] [--dest PATH] [--setup FILE] [--env K=V]...
|
||||
[--pass NAME]... [--pull REMOTE_GLOB:LOCAL_DIR]... [--keep] -- COMMAND
|
||||
up + sync + setup + command + pull + destroy; exits with COMMAND's status
|
||||
verify [run opts] [--trust-config] [-- COMMAND]
|
||||
run the repository's committed verify (.gstack/ubicloud.json under --src);
|
||||
its setup script and every pull destination stay inside --src (checked
|
||||
again at pull time), and its pass names are forwarded only when granted
|
||||
here (--pass NAME, or all: --trust-config)
|
||||
up [-s SIZE] [-l LOCATION] [-n NAME] [-S STORAGE_GIB] [-b IMAGE]
|
||||
create a VM and wait for SSH; prints its name
|
||||
(STORAGE_GIB: one of $ALLOWED_STORAGE)
|
||||
@@ -521,12 +865,18 @@ usage: ubi-runner.sh <command> [args]
|
||||
owner print this caller's owner tag ($OWNER)
|
||||
records VMs this machine created and has not confirmed gone ($VM_RECORD)
|
||||
gc HOURS destroy this owner's VMs older than HOURS (never other owners')
|
||||
reap destroy every VM whose -t<hours> TTL has ended (any owner; TTL-less names never)
|
||||
cli ARGS... raw Ubicloud CLI passthrough (e.g. cli vm list)
|
||||
|
||||
VMs are named $PREFIX-<owner>-<epoch>-<suffix>; set UBI_OWNER (e.g. your thread
|
||||
code) to tag them, otherwise a per-machine id is used.
|
||||
VMs are named $PREFIX-<owner>-<epoch>-<suffix>-t<ttl hours>; set UBI_OWNER (e.g. your
|
||||
thread code) to tag them, otherwise a per-machine id is used.
|
||||
defaults: size=$DEFAULT_SIZE location=$DEFAULT_LOCATION (env UBI_SIZE, UBI_LOCATION)
|
||||
env: UBI_GC_HOURS (unset/0: no sweep on up), UBI_CREATE_GRACE (${CREATE_GRACE}s),
|
||||
env: UBI_TTL_HOURS (${TTL_HOURS}h in the name; 0 omits it), UBI_REAP (0: up skips the reap),
|
||||
UBI_IDLE_TIMEOUT (${IDLE_TIMEOUT}s of silence kills the remote command, exit 124; 0 disables),
|
||||
UBI_RUN_TIMEOUT (${RUN_TIMEOUT}s wall clock; 0 unlimited), UBI_HEARTBEAT (${HEARTBEAT}s between heartbeat lines),
|
||||
UBI_QUOTA_VCPUS (project vCPU ceiling for the preflight; else the last refusal's maximum),
|
||||
UBI_QUOTA_WAIT (${QUOTA_WAIT}s to wait for quota room instead of refusing),
|
||||
UBI_GC_HOURS (unset/0: no sweep on up), UBI_CREATE_GRACE (${CREATE_GRACE}s),
|
||||
UBI_DOWN_TIMEOUT (${DOWN_TIMEOUT}s), UBI_API_TIMEOUT (${API_TIMEOUT}s), UBI_VM_RECORD (scratch VM-id file),
|
||||
UBI_ALLOWED_STORAGE (the -S sizes Ubicloud accepts; default "320 640")
|
||||
EOF
|
||||
@@ -540,12 +890,13 @@ case $cmd in
|
||||
esac
|
||||
case $cmd in
|
||||
run) cmd_run "$@" ;;
|
||||
verify) cmd_verify "$@" ;;
|
||||
up) trap 'trap "" INT TERM HUP QUIT; [ -z "$UP_NAME" ] || [ -n "$UP_READY" ] || cmd_down "$UP_NAME" || log "WARNING: failed to destroy $UP_NAME; run: $0 down $UP_NAME"' EXIT
|
||||
trap 'exit 130' INT TERM HUP QUIT
|
||||
cmd_up "$@" ;;
|
||||
ssh) name=$1; shift; load "$name"
|
||||
# shellcheck disable=SC2046
|
||||
if [ $# -eq 0 ]; then exec ssh -t $(ssh_opts) "ubi@$IP"; else remote "$name" "bash -lc $(printf %q "$*")"; fi ;;
|
||||
if [ $# -eq 0 ]; then exec ssh -t $(ssh_opts) "ubi@$IP"; else watched "$name" ssh "bash -lc $(printf %q "$*")"; fi ;;
|
||||
sync) cmd_sync "$@" ;;
|
||||
pack) cmd_pack "$@" ;;
|
||||
unpack) cmd_unpack "$@" ;;
|
||||
@@ -556,6 +907,7 @@ case $cmd in
|
||||
owner) echo "$OWNER" ;;
|
||||
records) cmd_records ;;
|
||||
gc) cmd_gc "$@" ;;
|
||||
reap) cmd_reap ;;
|
||||
cli) cli "$@" ;;
|
||||
help|-h|--help) usage ;;
|
||||
*) usage >&2; exit 2 ;;
|
||||
|
||||
Reference in new issue
Block a user