mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-08 17:58:13 +01:00
Merge remote-tracking branch 'origin/main' into release-gate-and-lane-friction
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> # Conflicts: # CHANGELOG.md
This commit is contained in:
commit
fb39d7c9a5
117 files changed
+6413
-503
No files matched your search
@@ -12,7 +12,7 @@
|
||||
*
|
||||
* Derived in part from pbakaus/impeccable (Apache-2.0), modified. See NOTICE.md.
|
||||
*/
|
||||
import { DESIGN_SLOP_CATALOG, BANNED_FONTS, type DesignSlopEntry } from '../../lib/design-catalog';
|
||||
import { RENDERED, BANNED_FONTS, type DesignSlopEntry } from '../../lib/design-catalog';
|
||||
import { SENTINEL, DETECT_EXIT_ECHO } from '../../lib/design-detect-contract';
|
||||
|
||||
export const DESIGN_CHECKLIST_HEADER =
|
||||
@@ -24,17 +24,17 @@ export const DESIGN_CHECKLIST_SLOP_HEADING = 'AI Slop Detection';
|
||||
|
||||
const TIER_ORDER: Record<DesignSlopEntry['confidence'], number> = { HIGH: 0, MEDIUM: 1, LOW: 2 };
|
||||
|
||||
/** Category 1: slop entries a code reader can grep for, plus the legacy blacklist lines. */
|
||||
/** Category 1: rendered (non-checkerOnly) slop entries a code reader can grep for, plus the legacy blacklist lines. */
|
||||
export function checklistSlopEntries(): DesignSlopEntry[] {
|
||||
// Array.prototype.sort is stable, so catalog order survives within a tier.
|
||||
return DESIGN_SLOP_CATALOG
|
||||
return RENDERED
|
||||
.filter(e => e.kind === 'slop' && (e.detect.includes('grep') || e.legacyBlacklist))
|
||||
.sort((a, b) => TIER_ORDER[a.confidence] - TIER_ORDER[b.confidence]);
|
||||
}
|
||||
|
||||
/** Catalog rules a review may fix without asking: mechanical CSS changes with HIGH confidence. */
|
||||
export function autoFixEntries(): DesignSlopEntry[] {
|
||||
return DESIGN_SLOP_CATALOG.filter(e => e.tier === 'auto-fix');
|
||||
return RENDERED.filter(e => e.tier === 'auto-fix');
|
||||
}
|
||||
|
||||
function endsWithPunctuation(s: string): boolean {
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* Craft resolvers shared by /design-review's craft audit and /plan-design-review.
|
||||
*
|
||||
* {{DESIGN_CRAFT_RULE:<lead>}} renders one paragraph of design-html's craft
|
||||
* section verbatim, found by its bold lead (`**Ship floor.**`), with the
|
||||
* bullets under it, so the audit holds a page to the rules /design-html builds
|
||||
* it to and the two cannot drift. A renamed lead fails generation.
|
||||
*
|
||||
* {{DESIGN_PERSUADE_CHECKS}} is the five conversion checks, phrased to hold for
|
||||
* a built page and for a plan of one. Adapted from Elaya's landing-page-design
|
||||
* and redesign-skill (MIT); see NOTICE.md.
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import type { ResolverFn } from './types';
|
||||
|
||||
const CRAFT_SECTION = path.resolve(import.meta.dir, '../../design-html/sections/craft.md.tmpl');
|
||||
|
||||
export const DESIGN_CRAFT_RULE: ResolverFn = (_ctx, args) => {
|
||||
const lead = args?.join(':');
|
||||
if (!lead) throw new Error('{{DESIGN_CRAFT_RULE:<lead>}} requires the bold lead of a craft paragraph');
|
||||
const matches = fs.readFileSync(CRAFT_SECTION, 'utf-8').split(/\n{2,}/).filter(p => p.startsWith(`**${lead}.**`) || p.startsWith(`**${lead},`));
|
||||
if (matches.length !== 1) throw new Error(`{{DESIGN_CRAFT_RULE:${lead}}}: ${matches.length} paragraphs starting "**${lead}.**" or "**${lead}," in design-html/sections/craft.md.tmpl; need exactly one`);
|
||||
return matches[0].trimEnd();
|
||||
};
|
||||
|
||||
export const DESIGN_PERSUADE_CHECKS: ResolverFn = () => `1. **One primary action**, named; nothing competes with it above the fold, and the closing CTA repeats it.
|
||||
2. **Named objections**: the visitor's top objections, each answered where it arises (FAQ of real questions, comparison, guarantee).
|
||||
3. **Proof beside each claim**: a supplied customer, quote or figure next to the claim it supports, never a metric row; an unsupported claim is cut or marked \`[Needs proof: …]\`.
|
||||
4. **A risk reversal** the business really offers (free plan, trial, no card, cancel anytime, refund); never an invented one.
|
||||
5. **An index/noindex decision**: an ad-only or time-bound page is \`noindex\`; an indexed one has a real title and description, and its FAQ ships as FAQPage JSON-LD.`;
|
||||
@@ -39,6 +39,7 @@ import { generateDxFramework, generateDxErrorCompact, generateDxErrorItems } fro
|
||||
import { generateGBrainContextLoad, generateGBrainSaveResults, generateBrainPreflight, generateBrainCacheRefresh, generateBrainWriteBack } from './gbrain';
|
||||
import { generateTasksSectionEmit, generateTasksSectionAggregate } from './tasks-section';
|
||||
import { SECTION, SECTION_INDEX } from './sections';
|
||||
import { DESIGN_CRAFT_RULE, DESIGN_PERSUADE_CHECKS } from './design-craft';
|
||||
import { generateRedactInvocationBlock } from './redact-doc';
|
||||
import { generateFreeTextFile } from './free-text-file';
|
||||
import { FOREGROUND_DISPATCH_NOTE, codexSelect } from './constants';
|
||||
@@ -109,6 +110,8 @@ export const RESOLVERS: Record<string, ResolverFn> = {
|
||||
DESIGN_DETECTOR: generateDesignDetector,
|
||||
DESIGN_MD_CHECK: generateDesignMdCheck,
|
||||
DESIGN_SLOP_BULLETS: generateDesignSlopBullets,
|
||||
DESIGN_CRAFT_RULE,
|
||||
DESIGN_PERSUADE_CHECKS,
|
||||
UX_PRINCIPLES: generateUXPrinciples,
|
||||
DESIGN_OUTSIDE_VOICES: generateDesignOutsideVoices,
|
||||
DESIGN_REVIEW_LITE: generateDesignReviewLite,
|
||||
|
||||
@@ -131,7 +131,7 @@ Dispatch via the Agent tool with ${FOREGROUND_IF_AVAILABLE} (background is the d
|
||||
Subagent prompt:
|
||||
"${DEFENSIVE_REVIEW_FRAMING}
|
||||
|
||||
Read the diff for this branch. First list changed files: \`DIFF_BASE=$(git merge-base origin/<base> HEAD) && git diff --name-status "$DIFF_BASE"\`. ${FIXTURE_SUMMARY_MODE}
|
||||
Read the diff for this branch. First list changed files: \`DIFF_BASE=$(git merge-base origin/<base> HEAD) && git diff --name-status "$DIFF_BASE"\`. ${FIXTURE_SUMMARY_MODE}${isShip ? ' From Step 9\'s pass 2 on, \`DIFF_BASE=$RANGE_BASE\` applies here too; \`NO FINDINGS\` is the expected result for a clean range.' : ''}
|
||||
|
||||
Think like an attacker and a chaos engineer. Your job is to find ways this code will fail in production. Look for: edge cases, race conditions, security holes, resource leaks, failure modes, silent data corruption, logic errors that produce wrong results silently, error handling that swallows failures, and trust boundary violations. No compliments — just the problems. For each finding, classify as FIXABLE (you know how to fix it) or INVESTIGATE (needs human judgment). After listing findings, end your output with ONE line in the canonical format \`Recommendation: <action> because <one-line reason naming the most exploitable finding>\` — examples: \`Recommendation: Fix the unbounded retry at queue.ts:78 because it'll DoS the worker pool under sustained 429s\` or \`Recommendation: Ship as-is because the strongest finding is a theoretical race that requires conditions we can't trigger in production\`. The reason must point to a specific finding (or no-fix rationale). Generic reasons like 'because it's safer' do not qualify."
|
||||
|
||||
@@ -303,7 +303,9 @@ in order before leaving Step 11:
|
||||
new attempt separately, then reconsider these decisions. Without that correction,
|
||||
or if the recovery fails, ask for repair and remain blocked.
|
||||
2. **Fixes queued after native completion:** Keep the findings and their approvals.
|
||||
Insert Steps 9, 10 and 11 before the pending Step 11.5 in the work list.
|
||||
Under \`review_scope frozen\`, scope the queue first as Step 9.4 item 0 does; when
|
||||
nothing is \`scope: fix\`, run item 3.5 (defer and render) and take decision 3.
|
||||
Otherwise insert Steps 9, 10 and 11 before the pending Step 11.5 in the work list.
|
||||
Step 9 completes full review before fixes; any further repair inserts its checks
|
||||
ahead of the remaining items. These fresh reviews after code edits are not recovery retries.
|
||||
Returning here never resets Step 9's three-cycle fix limit.
|
||||
|
||||
@@ -57,7 +57,7 @@ ${ctx.paths.binDir}/gstack-specialist-stats 2>/dev/null || true
|
||||
|
||||
### Select specialists
|
||||
|
||||
Based on the scope signals above, select which specialists to dispatch.
|
||||
Select specialists from the scope signals above.
|
||||
|
||||
**Always-on (dispatch on every review with 50+ changed lines):**
|
||||
1. **Testing** — read \`${ctx.paths.skillRoot}/review/specialists/testing.md\`
|
||||
@@ -78,13 +78,13 @@ Based on the scope signals above, select which specialists to dispatch.
|
||||
|
||||
After scope-based selection, apply adaptive gating based on specialist hit rates:
|
||||
|
||||
For each conditional specialist that passed scope gating, check the \`gstack-specialist-stats\` output above:
|
||||
For each conditional specialist that passed scope gating, read the \`gstack-specialist-stats\` output:
|
||||
- If tagged \`[GATE_CANDIDATE]\` (0 findings in 10+ dispatches): skip it. Print: "[specialist] auto-gated (0 findings in N reviews)."
|
||||
- If tagged \`[NEVER_GATE]\`: always dispatch regardless of hit rate.
|
||||
|
||||
**Force flags:** \`--<specialist>\` (e.g. \`--simplification\`) or \`--all-specialists\` in the prompt force-includes it.
|
||||
|
||||
Note which specialists were selected, gated, and skipped. Print the selection:
|
||||
Print the selection:
|
||||
"Dispatching N specialists: [names]. Skipped: [names] (scope not detected). Gated: [names] (0 findings in N+ reviews)."`;
|
||||
}
|
||||
|
||||
@@ -97,7 +97,7 @@ so they run in parallel. Each subagent has fresh context — no prior review bia
|
||||
|
||||
**Each specialist subagent prompt:**
|
||||
|
||||
Construct the prompt for each specialist. The prompt includes:
|
||||
Each specialist prompt includes:
|
||||
|
||||
1. The specialist's checklist path from the selection above (the subagent reads it; never paste its content)
|
||||
2. Stack context: "This is a {STACK} project."
|
||||
@@ -120,20 +120,21 @@ Every other specialist, including Testing, reads the full diff with fixtures.
|
||||
"You are a specialist code reviewer. Read the checklist at {checklist path}, then run
|
||||
\`DIFF_BASE=$(git merge-base origin/<base> HEAD) && git diff "$DIFF_BASE"\` to get the full diff. Apply the checklist against the diff.
|
||||
|
||||
For each finding, output a JSON object on its own line:
|
||||
\`NO FINDINGS\` alone is the expected, complete result for a clean diff; never invent an item to show work. Otherwise output one JSON object per finding, on its own line:
|
||||
{\\"severity\\":\\"CRITICAL|INFORMATIONAL\\",\\"confidence\\":N,\\"path\\":\\"file\\",\\"line\\":N,\\"category\\":\\"category\\",\\"summary\\":\\"description\\",\\"fix\\":\\"recommended fix\\",\\"fingerprint\\":\\"path:line:category\\",\\"specialist\\":\\"name\\"}
|
||||
|
||||
Required fields: severity, confidence, path, category, summary, specialist.
|
||||
Optional: line, fix, fingerprint, evidence, test_stub, advisory, evidence_paths, helper_target.
|
||||
Optional: line, fix, fingerprint, evidence, test_stub, advisory, evidence_paths, helper_target, cause.
|
||||
CRITICAL means a test that would fail, a production break, or a security or data-loss defect; everything else is INFORMATIONAL.
|
||||
A defect the diff causes but that shows elsewhere (a caller broken by a changed signature, a consumer of a removed export) keeps the victim as \`path\` and names the changed line in \`cause\`: {\\"cause\\":{\\"path\\":\\"file\\",\\"line\\":N}}. Without it the finding reads as pre-existing code.
|
||||
|
||||
Optional extraction advice belongs to the core shared-code check; do not duplicate its proposals. Report real defects in duplicated code independently. Preserve advisory metadata when returning structural advice, and never label a demonstrated defect advisory merely because sharing a helper could fix it.
|
||||
Leave extraction advice to the core shared-code check; report real defects in duplicated code independently, keep advisory metadata on structural advice, and never call a demonstrated defect advisory because a shared helper could fix it.
|
||||
|
||||
If you can write a test that would catch this issue, include it in the \`test_stub\` field.
|
||||
Use the detected test framework ({TEST_FW}). Write a minimal skeleton — describe/it/test
|
||||
blocks with clear intent. Skip test_stub for architectural or design-only findings.
|
||||
|
||||
If no findings: output \`NO FINDINGS\` and nothing else.
|
||||
Do not output anything else — no preamble, no summary, no commentary.
|
||||
Output nothing else: no preamble, no summary or commentary.
|
||||
|
||||
Stack context: {STACK}
|
||||
Past learnings: {learnings or 'none'}"
|
||||
@@ -162,9 +163,9 @@ their source labels: specialist scoring is not the final review's defect count.
|
||||
#### 1. Parse outputs
|
||||
|
||||
After specialist attempts settle, collect their outputs, tagged by actual source.
|
||||
Successful \`NO FINDINGS\` is a completed empty result. Otherwise parse each JSON line and
|
||||
skip invalid lines. Missing or unusable output is incomplete coverage, not an
|
||||
empty success. Retain each specialist's returned findings for activity stats.
|
||||
\`NO FINDINGS\` is a completed, clean result, never a failed reviewer. Otherwise parse
|
||||
each JSON line and skip invalid lines. Missing or unusable output is incomplete
|
||||
coverage, not an empty success. Keep each specialist's returned findings for activity stats.
|
||||
|
||||
#### 2. Validate severity
|
||||
|
||||
@@ -190,12 +191,29 @@ Compute identities for both core and specialist findings:
|
||||
or \`{path}:{category}\` when no line exists.
|
||||
|
||||
Within the specialist list, merge matching identities in the same partition: keep
|
||||
the highest confidence and all source names. Confirmation by distinct specialists
|
||||
adds +1 (cap at 10) and \`MULTI-SPECIALIST CONFIRMED ({specialist1} + {specialist2})\`.
|
||||
the highest confidence and all source names. Confirmation by distinct specialists adds +1 (cap at 10) and \`MULTI-SPECIALIST CONFIRMED ({specialist1} + {specialist2})\`.
|
||||
Core findings never earn a specialist confidence boost. Preserve \`advisory\`,
|
||||
\`evidence_paths\` and \`helper_target\` through every merge.
|
||||
|
||||
#### 4. Apply specialist confidence gates
|
||||
#### 4. Lead judgment
|
||||
|
||||
Reviewers fill the space they are given. Before any merged finding counts, the
|
||||
parent reads each one and drops or downgrades it on these grounds, saying so:
|
||||
- **Nit gravity.** When a reviewer's whole return is style, naming and preference,
|
||||
the code is probably fine; write that down and count nothing from it as a defect.
|
||||
- **Hypothetical, not actual.** A defect names the call site or input that reaches
|
||||
it. One that does not is a note until someone traces it.
|
||||
- **Structure for a case that does not exist.** Advice to abstract, generalize or
|
||||
guard a path nothing takes is advisory, never a defect.
|
||||
- **A different design is not a defect.** "I would have done it differently" with
|
||||
no failing case behind it is dropped.
|
||||
- **Missing context.** A finding that assumes a rule the repository documents
|
||||
otherwise is dropped with the pointer.
|
||||
Agreement between reviewers that did not see each other's output is the strongest
|
||||
signal; the \`MULTI-SPECIALIST CONFIRMED\` boost above records it. A reviewer that
|
||||
returned \`NO FINDINGS\` was right until a traced defect says otherwise.
|
||||
|
||||
#### 5. Apply specialist confidence gates
|
||||
|
||||
- Confidence 7+: show normally in the findings output
|
||||
- Confidence 5-6: show with caveat "Medium confidence — verify this is actually an issue"
|
||||
@@ -204,14 +222,14 @@ Core findings never earn a specialist confidence boost. Preserve \`advisory\`,
|
||||
|
||||
Core findings keep the core Confidence Calibration gates.
|
||||
|
||||
#### 5. Score and present specialists
|
||||
#### 6. Score and present specialists
|
||||
|
||||
Only specialist findings enter this header and \`quality_score\`; core findings do not.
|
||||
Use the merged NON-advisory specialist findings for both counts and score;
|
||||
the header's N is X + Y, so advisory findings never add to it:
|
||||
\`quality_score = max(0, 10 - (critical_count * 2 + informational_count * 0.5))\`
|
||||
Cap at 10 and retain for ${persistRef}. These are not final unresolved-defect totals.
|
||||
Print only this block: the stage 6 activity object and \`test_stub\` bodies are log and Fix-First data.
|
||||
Print only this block: the stage 7 activity object and \`test_stub\` bodies are log and Fix-First data.
|
||||
Validated \`"advisory": true\` findings from any source are excluded from score,
|
||||
header, unresolved-defect totals and clean-status blockers. Show them separately;
|
||||
they remain ASK-only, never auto-applied. Real defects follow normal Fix-First.
|
||||
@@ -236,9 +254,9 @@ PR Quality Score: X/10
|
||||
\`Simplification: lean already — nothing to cut.\`
|
||||
- If it was not dispatched, print neither line.
|
||||
|
||||
Do not add core shared-code savings to this specialist footer. Explain any overlap once in the core proposal instead of presenting duplicate savings.
|
||||
Core shared-code savings never enter this footer; explain any overlap once in the core proposal.
|
||||
|
||||
#### 6. Save specialist activity
|
||||
#### 7. Save specialist activity
|
||||
|
||||
Compile a \`specialists\` object for ${persistRef}.
|
||||
${isShip ? 'For each specialist' : 'For DIFF_LINES < 50, keep `specialists: {}`; do not manufacture per-specialist scope records. Otherwise record each considered specialist'} (testing, maintainability, security, performance, data-migration, api-contract, design, simplification, state-machines, red-team):
|
||||
@@ -252,7 +270,7 @@ Advisory findings COUNT in the stats \`findings\` field, not its defect counts.
|
||||
Include Design despite its different checklist. Preserve dispatch/failure status:
|
||||
zero returned findings from a failed attempt is not a clean review.
|
||||
|
||||
#### 7. Hand off to Fix-First
|
||||
#### 8. Hand off to Fix-First
|
||||
|
||||
Send these findings to ${fixFirstRef} alongside ${critPassRef}.
|
||||
Consolidate equivalent shared-code advice under the core proposal, retaining all
|
||||
@@ -280,12 +298,12 @@ The Red Team subagent receives:
|
||||
Prompt, after the defensive framing and fixture handling from item 4 of the specialist dispatch: "You are a red team reviewer. The code has already been reviewed by N specialists
|
||||
who found the following issues: {merged findings summary}. Your job is to find what they
|
||||
MISSED. Read the checklist at {red-team checklist path}, list changed files with \`DIFF_BASE=$(git merge-base origin/<base> HEAD) && git diff --name-status "$DIFF_BASE"\`, read them as that fixture handling says, and look for gaps.
|
||||
Output findings as JSON objects (same schema as the specialists). Focus on cross-cutting
|
||||
Output findings as JSON objects (same schema as the specialists), or \`NO FINDINGS\` alone, the expected result when they missed nothing. Focus on cross-cutting
|
||||
concerns, integration boundary issues, and failure modes that specialist checklists
|
||||
don't cover."
|
||||
|
||||
If the Red Team finds additional issues, tag them \`"specialist":"red-team"\`.
|
||||
Add them to the original specialist outputs and rerun stages 1–7 of Step ${stepMerge}
|
||||
Tag Red Team findings \`"specialist":"red-team"\`.
|
||||
Add them to the original specialist outputs and rerun stages 1–8 of Step ${stepMerge}
|
||||
before ${fixFirstRef}; do not boost or count the earlier findings twice.
|
||||
|
||||
If the Red Team returns NO FINDINGS, note: "Red Team review: no additional issues found."
|
||||
|
||||
@@ -25,9 +25,10 @@ const ROOT = path.resolve(import.meta.dir, '..', '..');
|
||||
|
||||
export const QA_ASSET_BLOCKER = 'If missing or unreadable, report a QA setup blocker and its affected probes as blocked; continue other safe probes (independent functional/static checks). Missing/unreadable assets block required QA.';
|
||||
|
||||
/** Skills carved on every host: QA (portable assets) and those whose external
|
||||
* render would exceed SKILL_BYTE_CEILING inlined (C4, #2777). */
|
||||
const CARVED_ON_EVERY_HOST = ['qa', 'qa-only', 'ship', 'plan-ceo-review', 'office-hours'];
|
||||
/** Skills carved on every host: QA (portable assets), those whose external
|
||||
* render would exceed SKILL_BYTE_CEILING inlined (C4, #2777), and design-review,
|
||||
* whose on-demand craft audit stays out of every skeleton (GSTA-92). */
|
||||
const CARVED_ON_EVERY_HOST = ['qa', 'qa-only', 'ship', 'plan-ceo-review', 'office-hours', 'design-review'];
|
||||
|
||||
/** Hosts read at most this many UTF-8 bytes of one SKILL.md (~40K tokens). */
|
||||
export const SKILL_BYTE_CEILING = 160_000;
|
||||
|
||||
+378
-26
@@ -23,7 +23,7 @@
|
||||
# Needs only bash, curl, python3, ssh, ssh-keygen, and tar locally, so it works
|
||||
# from dev boxes, containers, and cloud sandboxes.
|
||||
#
|
||||
# Ownership: VMs are named ubirun-<owner>-<epoch>-<suffix>. <owner> is UBI_OWNER
|
||||
# Ownership: VMs are named ubirun-<owner>-<epoch>-<suffix>[-t<hours>]. <owner> is UBI_OWNER
|
||||
# (lowercased, letters and digits only, at most 12 characters, starting with a
|
||||
# letter) or a random per-machine id kept in $STATE_ROOT/owner-id. Ubicloud names
|
||||
# allow 63 characters of [a-z0-9-], so a full name stays under 40.
|
||||
@@ -38,6 +38,29 @@
|
||||
# Stale-VM sweeps are off by default. With UBI_GC_HOURS set to a positive
|
||||
# number, every `up` first destroys this owner's VMs older than that; `gc HOURS`
|
||||
# runs the same sweep on demand. Neither ever touches another owner's VMs.
|
||||
#
|
||||
# TTL and reaper: a name this script generates ends in -t<hours> (UBI_TTL_HOURS,
|
||||
# default 4; 0 omits it), the creator's own declaration of how long the VM may
|
||||
# live. `reap` destroys every VM in the project whose TTL expired more than ten
|
||||
# minutes ago, whichever owner made it, and every `up` runs it first (UBI_REAP=0
|
||||
# skips it), so a runner killed by a sleeping machine leaves nothing behind for
|
||||
# longer than its TTL. Names without a TTL are never reaped.
|
||||
#
|
||||
# Quota preflight: when the project's vCPU ceiling is known (UBI_QUOTA_VCPUS, or
|
||||
# the maximum a refusal printed, cached in $STATE_ROOT/quota-max), `up` adds the
|
||||
# size's vCPUs to the project's running VMs and refuses before any create when
|
||||
# that exceeds it; UBI_QUOTA_WAIT=<seconds> waits for room instead.
|
||||
#
|
||||
# Watchdog: `run`, `verify` and `ssh NAME COMMAND` stream the remote output
|
||||
# through a watchdog that logs a heartbeat every UBI_HEARTBEAT seconds (300)
|
||||
# and kills a command silent for UBI_IDLE_TIMEOUT seconds (1800) or older than
|
||||
# UBI_RUN_TIMEOUT seconds (0, unlimited), exiting 124 with the last 40 lines,
|
||||
# so a hung verify fails fast instead of holding the VM until its TTL.
|
||||
#
|
||||
# `verify` is the one entry point for a repository's full remote verify: it
|
||||
# reads .gstack/ubicloud.json (setup, command, size, location, env, pass, pull)
|
||||
# and runs it through `run`, so lanes share one committed definition instead of
|
||||
# per-lane wrappers.
|
||||
set -euo pipefail
|
||||
# Keep heredoc bodies on temp files, not the pipe window (test/heredoc-pipe-deadlock.test.ts).
|
||||
BASH_COMPAT=50
|
||||
@@ -52,6 +75,14 @@ CREATE_GRACE="${UBI_CREATE_GRACE:-180}"
|
||||
DOWN_TIMEOUT="${UBI_DOWN_TIMEOUT:-900}"
|
||||
POLL="${UBI_POLL_SECONDS:-5}"
|
||||
ALLOWED_STORAGE="${UBI_ALLOWED_STORAGE:-320 640}"
|
||||
TTL_HOURS="${UBI_TTL_HOURS:-4}"
|
||||
REAP="${UBI_REAP:-1}"
|
||||
REAP_GRACE=600
|
||||
IDLE_TIMEOUT="${UBI_IDLE_TIMEOUT:-1800}"
|
||||
RUN_TIMEOUT="${UBI_RUN_TIMEOUT:-0}"
|
||||
HEARTBEAT="${UBI_HEARTBEAT:-300}"
|
||||
QUOTA_VCPUS="${UBI_QUOTA_VCPUS:-}"
|
||||
QUOTA_WAIT="${UBI_QUOTA_WAIT:-0}"
|
||||
PREFIX="ubirun"
|
||||
if [ -n "${UBI_VM_RECORD:-}" ]; then VM_RECORD="$UBI_VM_RECORD"
|
||||
elif [ -d "$HOME/.capy/work" ]; then VM_RECORD="$HOME/.capy/work/ubi-runner/vms.tsv"
|
||||
@@ -65,6 +96,11 @@ for bin in curl python3 ssh ssh-keygen tar; do
|
||||
done
|
||||
UBICLOUD_API_KEY="${UBICLOUD_API_KEY:-${UBICLOUD_API_TOKEN:-}}"
|
||||
[[ "$GC_HOURS" =~ ^[0-9]+$ ]] || die "UBI_GC_HOURS must be a whole number of hours (unset or 0 disables the sweep)"
|
||||
[[ "$TTL_HOURS" =~ ^(0|[1-9][0-9]{0,2})$ ]] || die "UBI_TTL_HOURS must be a whole number of hours from 0 to 999 with no leading zero (0 names VMs without a TTL)"
|
||||
for v in IDLE_TIMEOUT RUN_TIMEOUT HEARTBEAT QUOTA_WAIT; do
|
||||
[[ "${!v}" =~ ^[0-9]+$ ]] || die "UBI_$v must be a whole number of seconds (0 disables it)"
|
||||
done
|
||||
[ -z "$QUOTA_VCPUS" ] || [[ "$QUOTA_VCPUS" =~ ^[0-9]+$ ]] || die "UBI_QUOTA_VCPUS must be a whole number of vCPUs"
|
||||
|
||||
owner_id() {
|
||||
local raw=${UBI_OWNER:-} file="$STATE_ROOT/owner-id" tmp
|
||||
@@ -141,18 +177,134 @@ remote() {
|
||||
ssh $(ssh_opts) "ubi@$IP" "umask 022; $*"
|
||||
}
|
||||
|
||||
# vm_rows: "location name owner" for every VM in the project; owner is
|
||||
# "-" for untagged ubirun-<epoch>-* names and "." for VMs this runner did not name.
|
||||
vm_rows() {
|
||||
cli vm list -N -f location,name | while read -r loc name; do
|
||||
if [[ "$name" =~ ^${PREFIX}-([a-z][a-z0-9]{0,11})-[0-9]{10}- ]]; then
|
||||
echo "$loc $name ${BASH_REMATCH[1]}"
|
||||
elif [[ "$name" =~ ^${PREFIX}-[0-9]{10}- ]]; then
|
||||
echo "$loc $name -"
|
||||
else
|
||||
echo "$loc $name ."
|
||||
mtime() { stat -c %Y "$1" 2>/dev/null || stat -f %m "$1"; }
|
||||
|
||||
# sq STRING: STRING as one single-quoted shell word.
|
||||
sq() { printf "'%s'" "$(printf %s "$1" | sed "s/'/'\\\\''/g")"; }
|
||||
|
||||
# remote_cancel NAME SIDFILE: end the session a watched command started on the
|
||||
# VM (its leader pid is in SIDFILE there), bounded to 20 seconds. Killing the
|
||||
# local ssh client alone leaves a silent remote command running until it next
|
||||
# writes to the closed channel.
|
||||
remote_cancel() {
|
||||
local name=$1 sidf=$2 cpid i=0
|
||||
remote "$name" "sid=\$(cat $sidf 2>/dev/null); rm -f $sidf; [ -z \"\$sid\" ] || { pkill -TERM -s \"\$sid\"; sleep 2; pkill -KILL -s \"\$sid\"; }; true" >/dev/null 2>&1 </dev/null &
|
||||
cpid=$!
|
||||
while kill -0 "$cpid" 2>/dev/null && [ "$i" -lt 20 ]; do sleep 1; i=$(( i + 1 )); done
|
||||
kill "$cpid" 2>/dev/null || true
|
||||
wait "$cpid" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# watched NAME PHASE COMMAND: run COMMAND on the VM, streaming its output and
|
||||
# keeping a copy of both streams in the VM's state dir, with a heartbeat every HEARTBEAT
|
||||
# seconds and a watchdog that kills it after IDLE_TIMEOUT seconds of silence or
|
||||
# RUN_TIMEOUT seconds in total: exit 124, reason and the last 40 lines on
|
||||
# stderr. The ssh runs in its own session (python3's setsid) so the watchdog
|
||||
# kills it and anything it started, not just the shell that launched it; on
|
||||
# the VM the command runs in its own session too (setsid where the image has
|
||||
# it), whose leader pid is written to ~/.ubirun-PHASE-<runner pid>.sid so
|
||||
# remote_cancel can end it when the client is killed.
|
||||
watched() {
|
||||
local name=$1 phase=$2 cmd=$3 out pid rc=0 start now last_beat idle elapsed why="" old_traps inner sidf
|
||||
sidf="~/.ubirun-$phase-$$.sid"
|
||||
load "$name"
|
||||
out="$(state_dir "$name")/$phase.log"
|
||||
: >"$out"
|
||||
inner=$(sq "printf %s \$\$ >$sidf; $cmd")
|
||||
# The sid file outlives the command on purpose: a background child it left
|
||||
# behind keeps the ssh channel open after the wrapper exits, and remote_cancel
|
||||
# still needs the session id to end it. remote_cancel removes the file.
|
||||
cmd="if command -v setsid >/dev/null 2>&1; then setsid -w bash -c $inner; else bash -c $inner; fi"
|
||||
# The caller's stdin is handed to the background ssh explicitly (a background
|
||||
# job in a non-interactive shell would otherwise read /dev/null), so piped
|
||||
# input reaches the remote command as it did before the watchdog.
|
||||
exec 3<&0
|
||||
# shellcheck disable=SC2046
|
||||
( exec python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
|
||||
ssh $(ssh_opts) "ubi@$IP" "umask 022; $cmd" ) <&3 > >(tee -a "$out") 2> >(tee -a "$out" >&2) &
|
||||
pid=$!
|
||||
exec 3<&-
|
||||
# A signal to this process takes the ssh session down with it, so Ctrl-C on
|
||||
# `ssh NAME COMMAND` leaves no orphaned client behind.
|
||||
old_traps=$(trap -p INT TERM)
|
||||
trap 'kill -TERM -- "-$pid" 2>/dev/null; exit 130' INT TERM
|
||||
start=$(date +%s)
|
||||
last_beat=$start
|
||||
while :; do
|
||||
sleep 1
|
||||
kill -0 "$pid" 2>/dev/null || break
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - start ))
|
||||
idle=$(( now - $(mtime "$out") ))
|
||||
if [ "$IDLE_TIMEOUT" -gt 0 ] && [ "$idle" -ge "$IDLE_TIMEOUT" ]; then
|
||||
why="printed nothing for ${idle}s (UBI_IDLE_TIMEOUT=$IDLE_TIMEOUT)"
|
||||
elif [ "$RUN_TIMEOUT" -gt 0 ] && [ "$elapsed" -ge "$RUN_TIMEOUT" ]; then
|
||||
why="ran ${elapsed}s (UBI_RUN_TIMEOUT=$RUN_TIMEOUT)"
|
||||
fi
|
||||
if [ -n "$why" ]; then
|
||||
# The command may have finished while the clock was read; a finished
|
||||
# command is never reported as a timeout.
|
||||
kill -0 "$pid" 2>/dev/null || break
|
||||
log "watchdog: $phase $why; killing it. Last output:"
|
||||
tail -n 40 "$out" >&2
|
||||
kill -TERM -- "-$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true
|
||||
wait "$pid" 2>/dev/null || rc=$?
|
||||
trap - INT TERM; eval "$old_traps"
|
||||
# A command killed by the watchdog never exits 0; a 0 here means it
|
||||
# finished in the instant before the kill, and that result stands.
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
log "watchdog: $phase finished before it could be killed; its exit 0 stands"
|
||||
return 0
|
||||
fi
|
||||
remote_cancel "$name" "$sidf"
|
||||
return 124
|
||||
fi
|
||||
if [ "$HEARTBEAT" -gt 0 ] && [ $(( now - last_beat )) -ge "$HEARTBEAT" ]; then
|
||||
log "heartbeat: $phase ${elapsed}s elapsed, last output ${idle}s ago"
|
||||
last_beat=$now
|
||||
fi
|
||||
done
|
||||
wait "$pid" || rc=$?
|
||||
trap - INT TERM; eval "$old_traps"
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# vm_rows: "location name owner expires" for every VM in the project; owner is
|
||||
# "-" for untagged ubirun-<epoch>-* names and "." for VMs this runner did not
|
||||
# name; expires is the epoch the name's -t<hours> TTL ends at, or "-".
|
||||
vm_rows() {
|
||||
local loc name owner created expires
|
||||
cli vm list -N -f location,name | while read -r loc name; do
|
||||
owner=. created="" expires=-
|
||||
if [[ "$name" =~ ^${PREFIX}-([a-z][a-z0-9]{0,11})-([0-9]{10})- ]]; then
|
||||
owner=${BASH_REMATCH[1]} created=${BASH_REMATCH[2]}
|
||||
elif [[ "$name" =~ ^${PREFIX}-([0-9]{10})- ]]; then
|
||||
owner=- created=${BASH_REMATCH[1]}
|
||||
fi
|
||||
if [ -n "$created" ] && [[ "$name" =~ -t([0-9]{1,3})$ ]]; then
|
||||
expires=$(( 10#$created + 10#${BASH_REMATCH[1]} * 3600 ))
|
||||
fi
|
||||
echo "$loc $name $owner $expires"
|
||||
done
|
||||
}
|
||||
|
||||
# reap: destroy every VM whose name carries a TTL that ended more than
|
||||
# REAP_GRACE seconds ago, whichever owner created it (the TTL is that owner's
|
||||
# own declaration). Names without a TTL are never touched.
|
||||
cmd_reap() {
|
||||
local now rows loc name owner expires n=0 failed=0
|
||||
now=$(date +%s)
|
||||
rows=$(vm_rows) || { log "reap: could not list the project's VMs; nothing destroyed"; return 1; }
|
||||
while read -r loc name owner expires; do
|
||||
[ -n "$name" ] || continue
|
||||
[ "$expires" != - ] || continue
|
||||
[ "$now" -ge $(( expires + REAP_GRACE )) ] || continue
|
||||
n=$(( n + 1 ))
|
||||
log "reaping expired $loc/$name (owner $owner; its TTL ended $(( (now - expires) / 60 )) min ago)"
|
||||
cmd_down "$name" -l "$loc" || { failed=$(( failed + 1 )); log "failed to reap $loc/$name"; }
|
||||
done <<<"$rows"
|
||||
[ "$n" -gt 0 ] || log "reap: no expired VMs"
|
||||
[ "$failed" -eq 0 ] || { log "reap: $failed of $n expired VMs could not be destroyed"; return 1; }
|
||||
}
|
||||
|
||||
cmd_gc() {
|
||||
@@ -160,7 +312,7 @@ cmd_gc() {
|
||||
[[ "$hours" =~ ^[0-9]+$ ]] || die "gc: HOURS must be a whole number (or set UBI_GC_HOURS)"
|
||||
if [ "$hours" -eq 0 ]; then log "gc: no sweep (give HOURS > 0 or set UBI_GC_HOURS); nothing listed, nothing destroyed"; return 0; fi
|
||||
now=$(date +%s)
|
||||
vm_rows | while read -r loc name owner; do
|
||||
vm_rows | while read -r loc name owner _; do
|
||||
[ "$owner" = "$OWNER" ] || continue
|
||||
[[ "$name" =~ -([0-9]{10})- ]] || continue
|
||||
if [ $(( (now - BASH_REMATCH[1]) / 3600 )) -ge "$hours" ]; then
|
||||
@@ -182,7 +334,7 @@ cmd_list() {
|
||||
cmd_usage() {
|
||||
local tmp loc name owner i=0
|
||||
tmp=$(mktemp -d)
|
||||
while read -r loc name owner; do
|
||||
while read -r loc name owner _; do
|
||||
i=$(( i + 1 ))
|
||||
cli vm "$loc/$name" show 2>/dev/null | sed -n "s/^size: /$owner /p" >"$tmp/$i" &
|
||||
[ $(( i % 8 )) -ne 0 ] || wait
|
||||
@@ -226,7 +378,49 @@ cmd_records() {
|
||||
awk -F'\t' '{ printf "%-40s %-14s %-12s %-20s %s\n", $1, $2, $3, $4, $5 }' "$VM_RECORD"
|
||||
}
|
||||
|
||||
new_name() { echo "$PREFIX-$OWNER-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')"; }
|
||||
new_name() {
|
||||
local ttl=""
|
||||
[ "$TTL_HOURS" -eq 0 ] || ttl="-t$TTL_HOURS"
|
||||
echo "$PREFIX-$OWNER-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')$ttl"
|
||||
}
|
||||
|
||||
# vcpus SIZE: the vCPU count a Ubicloud size name carries (standard-16 -> 16).
|
||||
vcpus() { [[ "$1" =~ -([0-9]+)$ ]] && echo "${BASH_REMATCH[1]}"; }
|
||||
|
||||
# quota_preflight LOCATION NAME SIZE: refuse (or wait UBI_QUOTA_WAIT seconds)
|
||||
# before the create request when the project's running VMs plus this size
|
||||
# exceed the known vCPU ceiling. The ceiling comes from UBI_QUOTA_VCPUS or the
|
||||
# maximum the last refusal printed; without either the API's own refusal rules.
|
||||
QUOTA_CACHE_MAX_AGE=86400
|
||||
quota_preflight() {
|
||||
local location=$1 name=$2 size=$3 need max used wait_until remaining source="UBI_QUOTA_VCPUS" cache="$STATE_ROOT/quota-max"
|
||||
need=$(vcpus "$size") || return 0
|
||||
max=$QUOTA_VCPUS
|
||||
if [ -z "$max" ] && [ -f "$cache" ]; then
|
||||
# A refusal's maximum is remembered for a day: after that (a raised quota,
|
||||
# another project's key) the API's own answer decides again.
|
||||
if [ $(( $(date +%s) - $(mtime "$cache") )) -lt "$QUOTA_CACHE_MAX_AGE" ]; then
|
||||
max=$(cat "$cache" 2>/dev/null || true); source="the last refusal, cached in $cache"
|
||||
else
|
||||
rm -f "$cache"
|
||||
fi
|
||||
fi
|
||||
[[ "$max" =~ ^[0-9]+$ ]] || return 0
|
||||
wait_until=$(( $(date +%s) + QUOTA_WAIT ))
|
||||
while :; do
|
||||
used=$(cmd_usage 2>/dev/null | awk '$1=="total" {print $3}')
|
||||
[[ "$used" =~ ^[0-9]+$ ]] || return 0
|
||||
[ $(( used + need )) -gt "$max" ] || return 0
|
||||
remaining=$(( wait_until - $(date +%s) ))
|
||||
if [ "$remaining" -le 0 ]; then
|
||||
log "quota preflight refused $location/$name ($size): it needs $need vCPUs and the project's VMs already use $used of $max (ceiling from $source; other usage such as managed runners counts on top)"
|
||||
cmd_usage >&2 || true
|
||||
die "create refused before any request: vCPU quota exhausted (quota full); wait for running VMs to finish (UBI_QUOTA_WAIT=<seconds> waits here), run $0 reap, use a smaller size, or set UBI_QUOTA_VCPUS to the project's real ceiling"
|
||||
fi
|
||||
log "quota: $used of $max vCPUs in use and $need needed; waiting up to ${remaining}s more for room"
|
||||
sleep $(( remaining < 30 ? remaining : 30 ))
|
||||
done
|
||||
}
|
||||
|
||||
valid_name() { [[ "$1" =~ ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ ]]; }
|
||||
|
||||
@@ -254,9 +448,13 @@ cmd_up() {
|
||||
done
|
||||
[ -n "$name" ] || name=$(new_name)
|
||||
valid_name "$name" || die "invalid VM name '$name' (Ubicloud allows 1-63 of a-z, 0-9 and '-', starting and ending with a letter or digit)"
|
||||
if [ "$REAP" != 0 ]; then
|
||||
cmd_reap || log "reap failed; continuing"
|
||||
fi
|
||||
if [ "$GC_HOURS" -gt 0 ]; then
|
||||
cmd_gc "$GC_HOURS" || log "stale-VM cleanup failed; continuing"
|
||||
fi
|
||||
quota_preflight "$location" "$name" "$size"
|
||||
|
||||
local dir create_at
|
||||
dir=$(state_dir "$name")
|
||||
@@ -287,6 +485,7 @@ cmd_up() {
|
||||
die "create failed"
|
||||
fi
|
||||
read -r need used max <<<"$refusal"
|
||||
echo "$max" >"$STATE_ROOT/quota-max"
|
||||
log "quota refused $location/$name ($size): it needs $need vCPUs and the project already uses $used of $max"
|
||||
log "VMs in the project by owner (the rest of the used count is other usage, such as managed GitHub runners):"
|
||||
cmd_usage >&2 || true
|
||||
@@ -389,18 +588,34 @@ cmd_down() {
|
||||
|
||||
# pack SRC: write a gzipped checkout tarball to stdout (tracked + untracked
|
||||
# files that are not ignored, plus .git; a non-git directory is taken whole).
|
||||
# A linked worktree's .git is a one-line pointer into the main repository,
|
||||
# useless on the VM, so it is replaced by a self-contained clone of the
|
||||
# checked-out branch carrying the worktree's own index.
|
||||
cmd_pack() {
|
||||
local src
|
||||
local src tmp="" gitdir branch
|
||||
src=$(cd "${1:-.}" && pwd)
|
||||
if git -C "$src" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
if [ -f "$src/.git" ]; then
|
||||
branch=$(git -C "$src" rev-parse --abbrev-ref HEAD)
|
||||
[ "$branch" != HEAD ] || die "pack: $src is a linked worktree on a detached HEAD; check out a branch so the VM gets a self-contained .git"
|
||||
tmp=$(mktemp -d)
|
||||
gitdir=$(cd "$src" && cd "$(git rev-parse --git-common-dir)" && pwd)
|
||||
git clone -q --bare --no-local --single-branch --branch "$branch" "$gitdir" "$tmp/.git"
|
||||
git -C "$tmp/.git" config core.bare false
|
||||
# The worktree's index rides along so the VM's status matches, except a
|
||||
# split index, whose shared half lives only in the original git dir.
|
||||
[ "$(git -C "$src" config --bool core.splitIndex 2>/dev/null)" = true ] \
|
||||
|| cp "$(cd "$src" && cd "$(dirname "$(git rev-parse --git-path index)")" && pwd)/index" "$tmp/.git/index"
|
||||
fi
|
||||
(
|
||||
cd "$src"
|
||||
{ git ls-files -z -co --exclude-standard; printf '.git\0'; } \
|
||||
{ git ls-files -z -co --exclude-standard; [ -n "$tmp" ] || printf '.git\0'; } \
|
||||
| while IFS= read -r -d '' f; do
|
||||
if [ -e "$f" ] || [ -L "$f" ]; then printf '%s\0' "$f"; fi
|
||||
done \
|
||||
| tar --null -T - -czf -
|
||||
| if [ -n "$tmp" ]; then tar --null -T - -c -C "$tmp" .git -zf -; else tar --null -T - -czf -; fi
|
||||
)
|
||||
[ -z "$tmp" ] || rm -rf "$tmp"
|
||||
else
|
||||
tar -C "$src" -czf - .
|
||||
fi
|
||||
@@ -435,6 +650,34 @@ cmd_pull() {
|
||||
remote "$name" "cd $dir && tar -czf - $(basename "$from")" | tar -xzf - -C "$to"
|
||||
}
|
||||
|
||||
# pass_name NAME: a variable this runner may forward to a VM. Its own
|
||||
# credentials (UBICLOUD_*) never leave this machine, whoever asks.
|
||||
pass_name() {
|
||||
[[ "$1" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || die "--pass $1: not a variable name"
|
||||
case $1 in UBICLOUD_*) die "--pass $1: the runner never forwards its own Ubicloud credentials to a VM" ;; esac
|
||||
}
|
||||
|
||||
# inside_dir DIR ROOT: DIR resolves, symlinks followed, to ROOT or below it.
|
||||
# Its nearest existing ancestor is checked first, so a refused destination
|
||||
# creates nothing (not even outside the root); only then is DIR created and
|
||||
# checked again.
|
||||
inside_dir() {
|
||||
local a b near=$1
|
||||
b=$(cd "$2" 2>/dev/null && pwd -P) || return 1
|
||||
while [ ! -d "$near" ]; do
|
||||
[ "$near" != "$(dirname "$near")" ] || return 1
|
||||
near=$(dirname "$near")
|
||||
done
|
||||
a=$(cd "$near" 2>/dev/null && pwd -P) || return 1
|
||||
[ "$a" = "$b" ] || [[ "$a" = "$b"/* ]] || return 1
|
||||
mkdir -p "$1" 2>/dev/null || return 1
|
||||
a=$(cd "$1" 2>/dev/null && pwd -P) || return 1
|
||||
[ "$a" = "$b" ] || [[ "$a" = "$b"/* ]]
|
||||
}
|
||||
|
||||
# PULL_ROOT: set by verify; every pull destination must resolve inside it at
|
||||
# pull time (an earlier pull may have placed a symlink where a later one lands).
|
||||
PULL_ROOT=""
|
||||
cmd_run() {
|
||||
local up_args=() src=. dest="" setup="" keep=0 pulls=() envs=()
|
||||
while [ $# -gt 0 ]; do
|
||||
@@ -445,7 +688,7 @@ cmd_run() {
|
||||
--dest) dest=$2; shift 2 ;;
|
||||
--setup) setup=$2; shift 2 ;;
|
||||
--env) envs+=("$2"); shift 2 ;;
|
||||
--pass) [ -n "${!2+x}" ] || die "--pass $2: not set locally"; envs+=("$2=${!2}"); shift 2 ;;
|
||||
--pass) pass_name "$2"; [ -n "${!2+x}" ] || die "--pass $2: not set locally"; envs+=("$2=${!2}"); shift 2 ;;
|
||||
--pull) pulls+=("$2"); shift 2 ;;
|
||||
--keep) keep=1; shift ;;
|
||||
--) shift; break ;;
|
||||
@@ -468,6 +711,11 @@ cmd_run() {
|
||||
trap 'trap "" INT TERM HUP QUIT; cmd_down "$RUN_VM" || log "WARNING: failed to destroy $RUN_VM; run: $0 down $RUN_VM"' EXIT
|
||||
trap 'exit 130' INT TERM HUP QUIT
|
||||
cmd_up "${up_args[@]}" >/dev/null
|
||||
if [[ "$name" =~ -t([0-9]+)$ ]]; then
|
||||
log "$name carries a ${BASH_REMATCH[1]}h TTL (any later run of this script reaps it); if this process dies first: $0 down $name"
|
||||
else
|
||||
log "$name has no TTL; if this process dies before its teardown line, destroy it with: $0 down $name"
|
||||
fi
|
||||
if [ "$keep" = 1 ]; then
|
||||
trap - EXIT
|
||||
log "--keep: leaving $name running; destroy with: $0 down $name"
|
||||
@@ -478,27 +726,118 @@ cmd_run() {
|
||||
| remote "$name" 'umask 077 && cat > ~/.ubirun-env'
|
||||
|
||||
cmd_sync "$name" "$src" "$dest"
|
||||
local qdest
|
||||
local qdest rc=0
|
||||
qdest=$(printf %q "$dest")
|
||||
if [ -n "$setup" ]; then
|
||||
remote "$name" 'cat > ~/.ubirun-setup.sh' <"$setup"
|
||||
log "running setup $(basename "$setup")"
|
||||
remote "$name" "cd $qdest && . ~/.ubirun-env && bash -l ~/.ubirun-setup.sh" || die "setup failed"
|
||||
watched "$name" setup "cd $qdest && . ~/.ubirun-env && bash -l ~/.ubirun-setup.sh" || { rc=$?; log "setup failed (exit $rc)"; exit "$rc"; }
|
||||
fi
|
||||
|
||||
local start rc=0
|
||||
local start
|
||||
start=$(date +%s)
|
||||
log "running on $name: $*"
|
||||
remote "$name" "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "$*")" || rc=$?
|
||||
watched "$name" command "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "$*")" || rc=$?
|
||||
log "command exited $rc after $(( $(date +%s) - start ))s"
|
||||
|
||||
local p
|
||||
for p in ${pulls[@]+"${pulls[@]}"}; do
|
||||
if [ -n "$PULL_ROOT" ] && ! inside_dir "${p#*:}" "$PULL_ROOT"; then
|
||||
log "pull refused: ${p#*:} resolves outside $PULL_ROOT"
|
||||
[ "$rc" -ne 0 ] || rc=1
|
||||
continue
|
||||
fi
|
||||
cmd_pull "$name" "${p%%:*}" "${p#*:}" || log "pull failed: $p"
|
||||
done
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# verify [run opts] [--trust-config] [-- COMMAND]: run the repository's
|
||||
# committed remote verify. .gstack/ubicloud.json under --src (default .)
|
||||
# supplies setup, command, size, location, storage, image, env, pass and pull;
|
||||
# flags given here and a trailing -- COMMAND override it. Without the file it
|
||||
# refuses, so every lane runs the one committed definition instead of its own
|
||||
# wrapper.
|
||||
#
|
||||
# The file is repository content and gets no reach beyond the checkout on its
|
||||
# own: its setup script and pull destinations must resolve inside --src, and a
|
||||
# local variable it lists under pass is forwarded only when this command line
|
||||
# also names it (--pass NAME) or grants every listed name (--trust-config).
|
||||
cmd_verify() {
|
||||
local src=. i j g n cfg cfg_setup cfg_cmd parsed err rc=0 trust=0 cfg_args=() cfg_pass=() user_pre=() user_cmd=() granted=() missing=()
|
||||
for (( i = 1; i <= $#; i++ )); do
|
||||
[ "${!i}" = -- ] && break
|
||||
if [ "${!i}" = --src ]; then j=$(( i + 1 )); src=${!j}; fi
|
||||
done
|
||||
cfg="$src/.gstack/ubicloud.json"
|
||||
[ -f "$cfg" ] || die "verify: no $cfg; commit one ({\"setup\": \"scripts/ubicloud/bootstrap.sh\", \"command\": \"bash scripts/ubicloud/verify.sh\"}) so every lane runs the same verify instead of its own wrapper"
|
||||
err=$(mktemp)
|
||||
parsed=$(python3 -c '
|
||||
import json, os, shlex, sys
|
||||
cfg, src = sys.argv[1], sys.argv[2]
|
||||
try:
|
||||
c = json.load(open(cfg))
|
||||
except ValueError:
|
||||
sys.exit(3)
|
||||
if not isinstance(c, dict):
|
||||
sys.exit(3)
|
||||
base, root = os.path.abspath(src), os.path.realpath(src)
|
||||
def inside(rel, what):
|
||||
if not isinstance(rel, str) or not rel or os.path.isabs(rel) or rel.startswith("~") or ".." in rel.replace("\\", "/").split("/"):
|
||||
print("%s %r must be a relative path inside the checkout" % (what, rel), file=sys.stderr); sys.exit(2)
|
||||
full = os.path.realpath(os.path.join(root, rel))
|
||||
if full != root and not full.startswith(root + os.sep):
|
||||
print("%s %r resolves outside the checkout (%s)" % (what, rel, full), file=sys.stderr); sys.exit(2)
|
||||
return os.path.join(base, rel)
|
||||
q = lambda s: shlex.quote(str(s))
|
||||
args = []
|
||||
for key, flag in (("size", "-s"), ("location", "-l"), ("storage", "-S"), ("image", "-b")):
|
||||
if key in c: args += [flag, q(c[key])]
|
||||
for item in c.get("env", []): args += ["--env", q(item)]
|
||||
for item in c.get("pull", []):
|
||||
remote, sep, local = str(item).partition(":")
|
||||
if not sep or not remote:
|
||||
print("pull entry %r must be REMOTE_GLOB:LOCAL_DIR" % (item,), file=sys.stderr); sys.exit(2)
|
||||
args += ["--pull", q(remote + ":" + inside(local, "pull destination"))]
|
||||
setup = c.get("setup", "")
|
||||
print("cfg_args=(" + " ".join(args) + ")")
|
||||
print("cfg_pass=(" + " ".join(q(x) for x in c.get("pass", [])) + ")")
|
||||
print("cfg_setup=" + q(inside(setup, "setup") if setup else ""))
|
||||
print("cfg_cmd=" + q(c.get("command", "")))
|
||||
' "$cfg" "$src" 2>"$err") || rc=$?
|
||||
case $rc in
|
||||
0) ;;
|
||||
2) local why; why=$(cat "$err"); rm -f "$err"; die "verify: $cfg: $why" ;;
|
||||
*) rm -f "$err"; die "verify: $cfg is not valid JSON" ;;
|
||||
esac
|
||||
rm -f "$err"
|
||||
eval "$parsed"
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
--) shift; user_cmd=("$@"); break ;;
|
||||
--trust-config) trust=1; shift ;;
|
||||
--keep) user_pre+=("$1"); shift ;;
|
||||
--pass|-S|--storage|-s|--size|-l|--location|-n|--name|-b|--image|--src|--dest|--setup|--env|--pull)
|
||||
[ $# -ge 2 ] || die "verify: $1 needs a value"
|
||||
[ "$1" != --pass ] || granted+=("$2")
|
||||
user_pre+=("$1" "$2"); shift 2 ;;
|
||||
*) die "verify: unknown option $1" ;;
|
||||
esac
|
||||
done
|
||||
for n in ${cfg_pass[@]+"${cfg_pass[@]}"}; do
|
||||
pass_name "$n"
|
||||
if [ "$trust" = 1 ]; then cfg_args+=(--pass "$n"); continue; fi
|
||||
for g in ${granted[@]+"${granted[@]}"}; do [ "$g" = "$n" ] && continue 2; done
|
||||
missing+=("$n")
|
||||
done
|
||||
[ ${#missing[@]} -eq 0 ] || die "verify: $cfg asks to forward local ${missing[*]}; grant each on this command line with --pass NAME, or every name it lists with --trust-config"
|
||||
[ ${#user_cmd[@]} -gt 0 ] || [ -n "$cfg_cmd" ] || die "verify: $cfg names no command and none was given after --"
|
||||
[ ${#user_cmd[@]} -gt 0 ] || user_cmd=("$cfg_cmd")
|
||||
log "verify: $cfg"
|
||||
PULL_ROOT=$(cd "$src" && pwd)
|
||||
cmd_run ${cfg_args[@]+"${cfg_args[@]}"} ${cfg_setup:+--setup "$cfg_setup"} --src "$src" ${user_pre[@]+"${user_pre[@]}"} -- "${user_cmd[@]}"
|
||||
}
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
usage: ubi-runner.sh <command> [args]
|
||||
@@ -506,6 +845,11 @@ usage: ubi-runner.sh <command> [args]
|
||||
run [up opts] [--src DIR] [--dest PATH] [--setup FILE] [--env K=V]...
|
||||
[--pass NAME]... [--pull REMOTE_GLOB:LOCAL_DIR]... [--keep] -- COMMAND
|
||||
up + sync + setup + command + pull + destroy; exits with COMMAND's status
|
||||
verify [run opts] [--trust-config] [-- COMMAND]
|
||||
run the repository's committed verify (.gstack/ubicloud.json under --src);
|
||||
its setup script and every pull destination stay inside --src (checked
|
||||
again at pull time), and its pass names are forwarded only when granted
|
||||
here (--pass NAME, or all: --trust-config)
|
||||
up [-s SIZE] [-l LOCATION] [-n NAME] [-S STORAGE_GIB] [-b IMAGE]
|
||||
create a VM and wait for SSH; prints its name
|
||||
(STORAGE_GIB: one of $ALLOWED_STORAGE)
|
||||
@@ -521,12 +865,18 @@ usage: ubi-runner.sh <command> [args]
|
||||
owner print this caller's owner tag ($OWNER)
|
||||
records VMs this machine created and has not confirmed gone ($VM_RECORD)
|
||||
gc HOURS destroy this owner's VMs older than HOURS (never other owners')
|
||||
reap destroy every VM whose -t<hours> TTL has ended (any owner; TTL-less names never)
|
||||
cli ARGS... raw Ubicloud CLI passthrough (e.g. cli vm list)
|
||||
|
||||
VMs are named $PREFIX-<owner>-<epoch>-<suffix>; set UBI_OWNER (e.g. your thread
|
||||
code) to tag them, otherwise a per-machine id is used.
|
||||
VMs are named $PREFIX-<owner>-<epoch>-<suffix>-t<ttl hours>; set UBI_OWNER (e.g. your
|
||||
thread code) to tag them, otherwise a per-machine id is used.
|
||||
defaults: size=$DEFAULT_SIZE location=$DEFAULT_LOCATION (env UBI_SIZE, UBI_LOCATION)
|
||||
env: UBI_GC_HOURS (unset/0: no sweep on up), UBI_CREATE_GRACE (${CREATE_GRACE}s),
|
||||
env: UBI_TTL_HOURS (${TTL_HOURS}h in the name; 0 omits it), UBI_REAP (0: up skips the reap),
|
||||
UBI_IDLE_TIMEOUT (${IDLE_TIMEOUT}s of silence kills the remote command, exit 124; 0 disables),
|
||||
UBI_RUN_TIMEOUT (${RUN_TIMEOUT}s wall clock; 0 unlimited), UBI_HEARTBEAT (${HEARTBEAT}s between heartbeat lines),
|
||||
UBI_QUOTA_VCPUS (project vCPU ceiling for the preflight; else the last refusal's maximum),
|
||||
UBI_QUOTA_WAIT (${QUOTA_WAIT}s to wait for quota room instead of refusing),
|
||||
UBI_GC_HOURS (unset/0: no sweep on up), UBI_CREATE_GRACE (${CREATE_GRACE}s),
|
||||
UBI_DOWN_TIMEOUT (${DOWN_TIMEOUT}s), UBI_API_TIMEOUT (${API_TIMEOUT}s), UBI_VM_RECORD (scratch VM-id file),
|
||||
UBI_ALLOWED_STORAGE (the -S sizes Ubicloud accepts; default "320 640")
|
||||
EOF
|
||||
@@ -540,12 +890,13 @@ case $cmd in
|
||||
esac
|
||||
case $cmd in
|
||||
run) cmd_run "$@" ;;
|
||||
verify) cmd_verify "$@" ;;
|
||||
up) trap 'trap "" INT TERM HUP QUIT; [ -z "$UP_NAME" ] || [ -n "$UP_READY" ] || cmd_down "$UP_NAME" || log "WARNING: failed to destroy $UP_NAME; run: $0 down $UP_NAME"' EXIT
|
||||
trap 'exit 130' INT TERM HUP QUIT
|
||||
cmd_up "$@" ;;
|
||||
ssh) name=$1; shift; load "$name"
|
||||
# shellcheck disable=SC2046
|
||||
if [ $# -eq 0 ]; then exec ssh -t $(ssh_opts) "ubi@$IP"; else remote "$name" "bash -lc $(printf %q "$*")"; fi ;;
|
||||
if [ $# -eq 0 ]; then exec ssh -t $(ssh_opts) "ubi@$IP"; else watched "$name" ssh "bash -lc $(printf %q "$*")"; fi ;;
|
||||
sync) cmd_sync "$@" ;;
|
||||
pack) cmd_pack "$@" ;;
|
||||
unpack) cmd_unpack "$@" ;;
|
||||
@@ -556,6 +907,7 @@ case $cmd in
|
||||
owner) echo "$OWNER" ;;
|
||||
records) cmd_records ;;
|
||||
gc) cmd_gc "$@" ;;
|
||||
reap) cmd_reap ;;
|
||||
cli) cli "$@" ;;
|
||||
help|-h|--help) usage ;;
|
||||
*) usage >&2; exit 2 ;;
|
||||
|
||||
Reference in new issue
Block a user