mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 00:03:15 +01:00
v1.92.33.0 stamp: train gate.train-6c99c3cbe73c-83ae96e086de-03db85 S3 (#3697)
This commit is contained in:
garrytan committed
commit
ee41fafef7
417 files changed
+30804
-7082
No files matched your search
@@ -10,6 +10,8 @@ export type CsoRuntimeBuildStack = typeof CSO_RUNTIME_STACKS[number];
|
||||
|
||||
const IMAGE = /^[a-z0-9.-]+(?::[0-9]+)?\/[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}$/;
|
||||
const TAGGED_IMAGE = /^[a-z0-9.-]+(?::[0-9]+)?\/[a-z0-9][a-z0-9._/-]*:[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
||||
const RELEASE_TAG = /:v?\d+\.\d+\.\d+(?:[._-][A-Za-z0-9._-]*)?$/;
|
||||
const TWO_PART_RELEASE_TAG = /:v?\d+\.\d+(?:\.\d+)?(?:[._-][A-Za-z0-9._-]*)?$/;
|
||||
const REVISION = /^[a-z0-9][a-z0-9._-]{0,100}$/;
|
||||
const REQUIRED_VERSIONS: Record<CsoRuntimeBuildStack, string[]> = {
|
||||
node: ['node', 'npm', 'cso-preparation'],
|
||||
@@ -56,8 +58,9 @@ function repositoryOfTag(image: string): string {
|
||||
return image.slice(0, colon);
|
||||
}
|
||||
|
||||
function assertPinnedFamily(source: unknown, indexImage: unknown, images: unknown, label: string): Record<CsoRuntimeBuildPlatform, string> {
|
||||
function assertPinnedFamily(source: unknown, indexImage: unknown, images: unknown, label: string, releaseTag = RELEASE_TAG): Record<CsoRuntimeBuildPlatform, string> {
|
||||
if (typeof source !== 'string' || !TAGGED_IMAGE.test(source)) throw new Error(`INVALID_${label}_SOURCE`);
|
||||
if (!releaseTag.test(source)) throw new Error(`FLOATING_${label}_SOURCE: ${source} must name an exact release version, not a moving major, minor or alias tag`);
|
||||
if (typeof indexImage !== 'string' || !IMAGE.test(indexImage)) throw new Error(`UNPINNED_${label}_INDEX`);
|
||||
if (!record(images)) throw new Error(`MISSING_${label}_IMAGES`);
|
||||
const repository = repositoryOfTag(source);
|
||||
@@ -101,7 +104,7 @@ export function imageBuildMatrix(input: unknown): { include: ImageBuildRow[] } {
|
||||
if (typeof stack !== 'string' || !CSO_RUNTIME_STACKS.includes(stack as CsoRuntimeBuildStack) || seenStacks.has(stack)) throw new Error('INVALID_STACK');
|
||||
seenStacks.add(stack);
|
||||
const typedStack = stack as CsoRuntimeBuildStack;
|
||||
const bases = assertPinnedFamily(source, indexImage, baseImages, 'BASE');
|
||||
const bases = assertPinnedFamily(source, indexImage, baseImages, 'BASE', typedStack === 'postgresql' ? TWO_PART_RELEASE_TAG : RELEASE_TAG);
|
||||
if (!record(versions)) throw new Error('MISSING_TOOL_VERSIONS');
|
||||
const exactVersion = (value: unknown) => typeof value === 'string' &&
|
||||
(typedStack === 'postgresql'
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
+1446
-1260
File diff suppressed because it is too large.
Load diff
@@ -41,9 +41,9 @@
|
||||
"test/helpers/pty/launch.ts": {"blob":"e4413de380b799659170e7da248d51afda3b0e3f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6},
|
||||
"test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6},
|
||||
"test/helpers/pty/runners/floor.ts": {"blob":"54e2145832bb98913644d10ff303792d67f9362e","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."},
|
||||
"test/helpers/pty/runners/floor.ts": {"blob":"a9938cb98f285474ebc4860cd61012504a36466e","since":6,"nonBehavioral":"The PTY floor and observation runners end a plan-mode turn at an idle prompt (the plan-mode footer without its running 'esc to interrupt' segment, after a 10 s settle) instead of waiting out the deadline. The floor runner exits as api_error only after the idle screen with its error panel held still for 10 s and every tick's question and assessment steps found nothing to credit (the transcript is re-read each tick), so a native or prose question already on screen is credited as before, even when its transcript line lands after the screen draws done (test/helpers/claude-pty-runner.runners.unit.test.ts, the prose-question-before-panel case). With no qualifying question an idle turn stays idle, so a trial that failed at the deadline as observer_timeout now fails earlier as api_error. An observation returns what the deadline would have returned. A running footer, extra footer text and a cut footer still wait for the deadline. No pass becomes a fail or a fail a pass. The session ledger may now record failure_cause api_error or refusal instead of observer_timeout or session_timeout for a plan-mode session whose last idle turn shows a panel; failure_class, the gate red class and pass rates are unchanged, so no pass-rate series resets"},
|
||||
"test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6},
|
||||
"test/helpers/pty/screen.ts": {"blob":"f3743152fc9bb7a8e8c70002f75bf45914f260d5","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
"test/helpers/pty/screen.ts": {"blob":"1384f765e48e511d5f843df2660b2ec70bbcb6b2","since":6,"nonBehavioral":"The PTY floor and observation runners end a plan-mode turn at an idle prompt (the plan-mode footer without its running 'esc to interrupt' segment, after a 10 s settle) instead of waiting out the deadline. The floor runner exits as api_error only after the idle screen with its error panel held still for 10 s and every tick's question and assessment steps found nothing to credit (the transcript is re-read each tick), so a native or prose question already on screen is credited as before, even when its transcript line lands after the screen draws done (test/helpers/claude-pty-runner.runners.unit.test.ts, the prose-question-before-panel case). With no qualifying question an idle turn stays idle, so a trial that failed at the deadline as observer_timeout now fails earlier as api_error. An observation returns what the deadline would have returned. A running footer, extra footer text and a cut footer still wait for the deadline. No pass becomes a fail or a fail a pass. The session ledger may now record failure_cause api_error or refusal instead of observer_timeout or session_timeout for a plan-mode session whose last idle turn shows a panel; failure_class, the gate red class and pass rates are unchanged, so no pass-rate series resets"},
|
||||
"test/helpers/pty/session.ts": {"blob":"54fe04b61d953e4e55e2fc251532c04a49fab66f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6},
|
||||
"test/helpers/session-ledger.ts": {"blob":"a10bf28bf100f135b93c73818670dd147253cdd3","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
|
||||
@@ -322,9 +322,16 @@ function flag(args: string[], name: string): string | undefined {
|
||||
return index >= 0 ? args[index + 1] : undefined;
|
||||
}
|
||||
|
||||
async function main(args: string[]): Promise<void> {
|
||||
/** The W2 per-judge table plus the arm benchmark's paired judge (run by --config only; the manifest prices the W2 table). */
|
||||
export async function loadCalibrationConfigs(): Promise<{ w2: CalibrationConfig[]; all: CalibrationConfig[] }> {
|
||||
// The per-judge table (builders, schemas, verdict rules) is committed separately from this harness.
|
||||
const { CALIBRATION_CONFIGS } = await import(path.join(import.meta.dir, 'judge-calibration-configs.ts')) as { CALIBRATION_CONFIGS: CalibrationConfig[] };
|
||||
const { ARM_PAIR_CALIBRATION } = await import(path.join(import.meta.dir, '..', 'test', 'helpers', 'arm-pair-judge.ts')) as { ARM_PAIR_CALIBRATION: CalibrationConfig };
|
||||
return { w2: CALIBRATION_CONFIGS, all: [...CALIBRATION_CONFIGS, ARM_PAIR_CALIBRATION] };
|
||||
}
|
||||
|
||||
async function main(args: string[]): Promise<void> {
|
||||
const { w2: CALIBRATION_CONFIGS, all: configs } = await loadCalibrationConfigs();
|
||||
const command = args[0];
|
||||
if (command === 'manifest') {
|
||||
const client = new Anthropic();
|
||||
@@ -338,8 +345,8 @@ async function main(args: string[]): Promise<void> {
|
||||
console.table(manifest.entries);
|
||||
return;
|
||||
}
|
||||
const config = CALIBRATION_CONFIGS.find(candidate => candidate.id === flag(args, 'config'));
|
||||
if (!config) throw new Error(`unknown --config; expected one of ${CALIBRATION_CONFIGS.map(candidate => candidate.id).join(', ')}`);
|
||||
const config = configs.find(candidate => candidate.id === flag(args, 'config'));
|
||||
if (!config) throw new Error(`unknown --config; expected one of ${configs.map(candidate => candidate.id).join(', ')}`);
|
||||
const items = loadCorpus(config.id);
|
||||
const runsDir = path.join(CORPUS_ROOT, config.id, 'runs');
|
||||
const samplesPath = path.join(runsDir, 'comparison-1.samples.jsonl');
|
||||
|
||||
@@ -258,6 +258,26 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
|
||||
file: 'test/claude-overlay-setup-installs.test.ts',
|
||||
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/claude-overlay-fixture.ts, so the source scan misses them',
|
||||
},
|
||||
{
|
||||
file: 'test/claude-overlay-setup-lifecycle.test.ts',
|
||||
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/claude-overlay-fixture.ts, so the source scan misses them',
|
||||
},
|
||||
{
|
||||
file: 'test/claude-overlay-setup-project.test.ts',
|
||||
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/claude-overlay-fixture.ts, so the source scan misses them',
|
||||
},
|
||||
{
|
||||
file: 'test/host-renders-prune-shared-root.test.ts',
|
||||
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/install-fixture.ts, so the source scan misses them',
|
||||
},
|
||||
{
|
||||
file: 'test/host-renders-prune-migration.test.ts',
|
||||
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/install-fixture.ts, so the source scan misses them',
|
||||
},
|
||||
{
|
||||
file: 'test/setup-codex-scope-runtime-links.test.ts',
|
||||
reason: 'every case runs the real ./setup under describe.skipIf(win32); its bin/ spawns live in test/helpers/install-fixture.ts and test/helpers/setup-codex-scope-cyclic.ts, so the source scan misses them',
|
||||
},
|
||||
{
|
||||
file: 'test/cso-scanner-cli.test.ts',
|
||||
reason: 'drives the prebuilt POSIX CSO launcher with /usr/bin/git and a POSIX-only PATH; native Windows launcher behavior is covered by the dedicated cso-windows-launcher gate',
|
||||
@@ -299,14 +319,55 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
|
||||
file: 'test/ship-queue-train.test.ts',
|
||||
reason: 'drives bin/gstack-ship-queue against the stateful #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead of the shim (windows-free-shard run 39152702056)',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-queue-train-bisect.test.ts',
|
||||
reason: 'split from test/ship-queue-train.test.ts (GSTA-151): drives bin/gstack-ship-queue against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-queue-train-verify.test.ts',
|
||||
reason: 'split from test/ship-queue-train.test.ts (GSTA-151): drives bin/gstack-ship-queue against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts',
|
||||
},
|
||||
{
|
||||
file: 'test/ubicloud-runner-tiers.test.ts',
|
||||
reason: 'split from test/ubicloud-runner.test.ts, excluded as a whole: runs scripts/ubicloud/ubi-runner.sh through bash with the #!/usr/bin/env bash curl, ssh and make shims test/helpers/ubi-runner-fake-vm.ts puts on a colon-joined PATH, like test/ubicloud-runner-run.test.ts',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-queue-train-freeze.test.ts',
|
||||
reason: 'split from test/ship-queue-train.test.ts (GSTA-151): drives bin/gstack-ship-queue against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-queue-train-cost.test.ts',
|
||||
reason: 'drives bin/gstack-ship-queue train verdict against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead of the shim (windows-free-shard 2 on #3478 at d5e6a7d)',
|
||||
},
|
||||
// GSTA-151 splits: parts of files the curation excluded as a whole keep that platform scope.
|
||||
{
|
||||
file: 'test/docsync-fault-interface-legacy.test.ts',
|
||||
reason: 'split from test/docsync-fault-interface.test.ts, excluded as a whole by its POSIX mode-bit checks; replays the same docsync callbacks through the generated bun-test child',
|
||||
},
|
||||
{
|
||||
file: 'test/docsync-fault-interface-faults.test.ts',
|
||||
reason: 'split from test/docsync-fault-interface.test.ts, excluded as a whole by its POSIX mode-bit checks; replays the same docsync callbacks through the generated bun-test child',
|
||||
},
|
||||
{
|
||||
file: 'test/shared-libs-checker-interface-evidence.test.ts',
|
||||
reason: 'runs bin/gstack-review-log (shebang) through test/helpers/shared-libs-checker-evidence.ts captures (Windows CreateProcess does not parse shebangs)',
|
||||
},
|
||||
{
|
||||
file: 'test/shared-libs-checker-interface-evidence-declared.test.ts',
|
||||
reason: 'split from test/shared-libs-checker-interface-evidence.test.ts: runs bin/gstack-review-log (shebang) through test/helpers/shared-libs-checker-evidence.ts captures (Windows CreateProcess does not parse shebangs)',
|
||||
},
|
||||
{
|
||||
file: 'test/shared-libs-checker-interface-evidence-replays.test.ts',
|
||||
reason: 'split from test/shared-libs-checker-interface-evidence.test.ts, which spawned bin/gstack-review-log (shebang); its prerequisite checks run the same review-log commands through bash',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-queue-train-ops.test.ts',
|
||||
reason: 'drives bin/gstack-ship-queue (screen verdicts, scratch-merge screens, prune --stale, watch, forget, verify --post-merge, ready --dispatch-ci, --stack-tests) against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-queue-train-generic.test.ts',
|
||||
reason: 'drives bin/gstack-ship-queue train build, verdict, land, watch, init and doctor against the same #!/bin/sh gh shim plus an extensionless node gh wrapper on a colon-joined PATH; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead (windows-free-shard 1, run 41218228855)',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-policy-compat.test.ts',
|
||||
reason: 'spawns bin/gstack-ship-policy, bin/gstack-restamp and bin/gstack-tree-receipt (shebangs) through test/helpers/restamp-fixture.ts',
|
||||
@@ -323,6 +384,10 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
|
||||
file: 'test/ship-release.test.ts',
|
||||
reason: 'spawns bin/gstack-ship-release (shebang) directly and writes #!/bin/sh gh stubs through test/helpers/ship-bin-fixture.ts; windows-latest exits before the bin runs (windows-free-shard 3, run 41073592483)',
|
||||
},
|
||||
{
|
||||
file: 'test/ship-rereview-triggers.test.ts',
|
||||
reason: 'drives bin/gstack-ship-release, gstack-docs-candidate and gstack-wtree (shebangs) through test/helpers/ship-bin-fixture.ts like test/ship-release.test.ts; windows-latest fails every bin case at the first spawn (windows-free-shard 6, run 41193336209)',
|
||||
},
|
||||
];
|
||||
|
||||
// Force-include overrides: files a WINDOWS_FRAGILE_PATTERNS regex excludes for
|
||||
@@ -347,6 +412,8 @@ export const KNOWN_WINDOWS_SAFE: Array<{ file: string; reason: string }> = [
|
||||
{ file: 'test/gstack-config-gbrain-refresh.test.ts', reason: 'E6: runs bin/gstack-config through explicit bash; its gbrain/python3 shims are found by bash PATH lookup, never launched by CreateProcess' },
|
||||
{ file: 'test/hook-scripts.test.ts', reason: '#3067 PowerShell and NotebookEdit hook coverage: every hook runs through spawnSync("bash", [script]) with the payload on stdin, scratch state lives under os.tmpdir(); its /tmp/ literals are payload strings, never opened; symlink, newline-path and POSIX-PATH cases are skipIf win32' },
|
||||
{ file: 'test/gstack-doctor.test.ts', reason: 'runs bin/gstack-doctor and setup --status through explicit bash argv in an os.tmpdir fixture; its bun/codex/claude/probe stubs are found by bash PATH lookup, and expected paths use bash spellings' },
|
||||
{ file: 'test/gstack-doctor-check.test.ts', reason: 'split from test/gstack-doctor.test.ts: runs bin/gstack-doctor --check through explicit bash argv in the test/helpers/doctor-fixture.ts os.tmpdir fixture; its stubs are found by bash PATH lookup, and expected paths use bash spellings' },
|
||||
{ file: 'test/gstack-doctor-launch.test.ts', reason: 'split from test/gstack-doctor.test.ts: runs bin/gstack-doctor through explicit bash argv in the test/helpers/doctor-fixture.ts os.tmpdir fixture; Windows bash and platform rows are injected through GSTACK_DOCTOR_PLATFORM, and POSIX-path cases skip on win32' },
|
||||
// Fix wave PR 5 (item 10b, #3197): the fixture runBin launches bins through lib/script-invocation.ts
|
||||
// (bun bins as Bun argv, bash bins through Git Bash); passed on windows-latest in the fix wave PR 5 windows-free-tests run.
|
||||
{ file: 'test/gstack-regen.test.ts', reason: 'runs bin/gstack-regen through the fixture runBin (lib/script-invocation.ts) and its regen commands through the resolved Git Bash' },
|
||||
|
||||
@@ -216,7 +216,11 @@
|
||||
"test/skill-e2e-triage.test.ts": 1000,
|
||||
"test/skill-e2e-workflow.test.ts": 121594,
|
||||
"test/skill-llm-eval.test.ts": 339704,
|
||||
"test/skill-routing-e2e.test.ts": 81360
|
||||
"test/skill-routing-e2e.test.ts": 81360,
|
||||
"test/skill-e2e-planning-wave.test.ts#planning-wave-planted-facts": 74000,
|
||||
"test/skill-e2e-planning-wave.test.ts#planning-wave-schema": 52000,
|
||||
"test/skill-e2e-planning-wave.test.ts#planning-wave-tasks-rerun": 37000,
|
||||
"test/skill-e2e-planning-wave.test.ts#planning-wave-closing-signals": 27000
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -89,7 +89,7 @@ export const SELECTION_DATA_MODULES: readonly string[] = [
|
||||
// The build entry list: every compiled binary, which skills reach through their `--outfile` path instead.
|
||||
'scripts/build.sh',
|
||||
// The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes.
|
||||
'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts',
|
||||
'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', 'lib/result-codes-plan.ts',
|
||||
// The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case).
|
||||
'test/helpers/exercised-commands.json',
|
||||
];
|
||||
|
||||
@@ -593,11 +593,10 @@ Show the screenshot to the user. Ask: "Does this feel right? Want to iterate on
|
||||
If they want changes, regenerate the HTML with their feedback and re-render.
|
||||
If they approve or say "good enough," proceed.
|
||||
|
||||
**Step 5: Include in design doc**
|
||||
**Step 5: Save the sketch**
|
||||
|
||||
Reference the wireframe screenshot in the design doc's "Recommended Approach" section.
|
||||
The screenshot file at \`<sketch-dir>/sketch.png\` (name the full path in the doc) can be referenced by downstream skills
|
||||
(\`/plan-design-review\`, \`/design-review\`) to see what was originally envisioned.
|
||||
Copy the temporary \`sketch.png\` and \`sketch.html\` to \`<PROJECT_DIR>/{user}-{branch}-sketch-{datetime}.*\`.
|
||||
In "Recommended Approach" add \`Wireframe: projects/<slug>/<png name>\` (state-root relative), or \`Wireframe: none (<reason>)\`.
|
||||
|
||||
**Step 6: Outside design voices** (optional)
|
||||
|
||||
|
||||
@@ -32,6 +32,10 @@ empty JSONL files array. Each task needs a concrete verification step.` : `Befor
|
||||
build-actionable tasks. Each task derives from a specific finding — no padding.`}
|
||||
${conditionalWrites ? `Always emit the markdown section. Write its JSONL artifact for \`/autoplan\` only when the ${storagePolicy} permits it; otherwise label the complete task output not persisted and do not claim an aggregation artifact exists.` : 'Emit the markdown section AND write a JSONL artifact that `/autoplan` can\naggregate across phases.'}
|
||||
|
||||
### Earlier lists on this branch
|
||||
|
||||
Before listing tasks, run \`~/.claude/skills/gstack/bin/gstack-tasks prior --phase ${phase}\` and reconcile what it prints, so a rerun never leaves two lists.
|
||||
|
||||
### Markdown section (always emit)
|
||||
|
||||
\`\`\`markdown
|
||||
@@ -54,49 +58,11 @@ Rules:
|
||||
|
||||
### JSONL artifact (${conditionalWrites ? 'write when permitted, including zero tasks' : 'always write, even if zero tasks'})
|
||||
|
||||
\`/autoplan\` reads this file to aggregate across phases. Build each line with
|
||||
\`jq -nc\` so titles and source findings containing quotes, newlines, or
|
||||
backslashes serialize cleanly — never use hand-rolled \`echo\` / \`printf\`.
|
||||
\`/autoplan\` aggregates this list. With your file-write tool, write one JSON object per line to \`.gstack/tmp/tasks-${phase}-rows.jsonl\` at the repo root (no free text in a command; no rows, no file): a task is \`{"id":"T1","priority":"P1","component":"","title":"","source_finding":"","effort_human":"~2h","effort_cc":"~15min","files":[]}\`, plus \`"carried_from":"<run_id>/<id>"\` when carried; a closed earlier task is \`{"prior":"<run_id>/<id>","disposition":"done|dropped","reason":""}\`. Then write the list (empty means "ran, no findings", distinct from "didn't run"):
|
||||
|
||||
\`\`\`bash
|
||||
GSTACK_STATE_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get GSTACK_STATE_ROOT); : "\${GSTACK_STATE_ROOT:?gstack-paths failed; reinstall with ./setup or /gstack-upgrade}"
|
||||
SLUG=$(~/.claude/skills/gstack/bin/gstack-slug --get SLUG 2>/dev/null)
|
||||
TASKS_DIR="$GSTACK_STATE_ROOT/projects/\${SLUG:-unknown}"
|
||||
mkdir -p "$TASKS_DIR"
|
||||
TASKS_FILE="$TASKS_DIR/tasks-${phase}-$(date +%Y%m%d-%H%M%S).jsonl"
|
||||
COMMIT=$(git rev-parse HEAD 2>/dev/null || echo unknown)
|
||||
# The task record keeps the raw git branch; artifact filenames keep gstack-slug's sanitized BRANCH.
|
||||
GIT_BRANCH=$(git branch --show-current 2>/dev/null || echo unknown)
|
||||
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
||||
|
||||
# Repeat ONE jq invocation per task identified during this review.
|
||||
# Substitute the placeholders inline with shell variables you set per task:
|
||||
# TASK_ID (T1, T2, ...), PRIORITY (P1/P2/P3), COMPONENT, TITLE,
|
||||
# SOURCE_FINDING, EFFORT_HUMAN, EFFORT_CC, FILES_JSON (a JSON array literal
|
||||
# like '["browse/src/sanitize.ts","browse/src/server.ts"]').
|
||||
jq -nc \\
|
||||
--arg phase '${phase}' \\
|
||||
--arg run_id "$RUN_ID" \\
|
||||
--arg branch "$GIT_BRANCH" \\
|
||||
--arg commit "$COMMIT" \\
|
||||
--arg id "$TASK_ID" \\
|
||||
--arg priority "$PRIORITY" \\
|
||||
--arg component "$COMPONENT" \\
|
||||
--arg effort_human "$EFFORT_HUMAN" \\
|
||||
--arg effort_cc "$EFFORT_CC" \\
|
||||
--arg title "$TITLE" \\
|
||||
--arg source_finding "$SOURCE_FINDING" \\
|
||||
--argjson files "$FILES_JSON" \\
|
||||
'{phase:$phase, run_id:$run_id, branch:$branch, commit:$commit, id:$id, priority:$priority, component:$component, files:$files, effort_human:$effort_human, effort_cc:$effort_cc, title:$title, source_finding:$source_finding}' \\
|
||||
>> "$TASKS_FILE"
|
||||
~/.claude/skills/gstack/bin/gstack-tasks add --phase ${phase}
|
||||
\`\`\`
|
||||
|
||||
If \`jq\` is not installed, fall back to skipping the JSONL write and warn
|
||||
the user to install jq for autoplan aggregation. Never hand-roll JSONL.
|
||||
|
||||
${conditionalWrites ? 'When writes are permitted and zero tasks were identified, touch the JSONL file' : 'If zero tasks were identified in this review, still touch the JSONL file'}
|
||||
(\`: > "$TASKS_FILE"\`) so the aggregator sees that the phase produced output
|
||||
this run (an empty file means "ran, no findings" — distinct from "didn't run").
|
||||
`;
|
||||
};
|
||||
|
||||
|
||||
@@ -277,7 +277,7 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
|
||||
return { value: `${(total / 60_000 * 7 / sinceDays).toFixed(0)} min/week (${cancelled.length} cancelled run(s))` };
|
||||
}));
|
||||
|
||||
metrics.push(metric('push-to-verdict', 'PR lane push-to-verdict', 'created_at to updated_at of evals.yml pull_request runs that reached success or failure (includes queue and the readiness wait)', () => {
|
||||
metrics.push(metric('push-to-verdict', 'PR lane push-to-verdict', 'created_at to updated_at of evals.yml pull_request runs that reached success or failure (includes queue and the readiness wait, which pr-summary runs skip since GSTA-149, so their verdicts land about 10 min sooner)', () => {
|
||||
const walls = need(prRuns).filter(run => run.conclusion === 'success' || run.conclusion === 'failure').map(run => span(run.createdAt, run.updatedAt)!);
|
||||
if (walls.length === 0) throw new Error('no PR run reached a verdict in the window');
|
||||
return { value: `p50 ${minutes(percentile(walls, 0.5))}, p95 ${minutes(percentile(walls, 0.95))} (${walls.length} runs)` };
|
||||
|
||||
@@ -241,6 +241,7 @@
|
||||
"test/helpers/qa-functional-evidence.ts\tTS7006\tParameter 'row' implicitly has an 'any' type.": 2,
|
||||
"test/helpers/session-runner.ts\tTS2352\tConversion of type 'ReadableStream<any>' to type 'ReadableStream<Uint8Array<ArrayBufferLike>>' may be a mistake because neither type sufficiently overlaps with the other. If this was intentional, convert the expression to 'unknown' first. Types of property 'getReader' are incompatible. Type '{ (): ReadableStreamDefaultReader<any>; (options: { mode: \"byob\"; }): ReadableStreamBYOBReader; (options?: ReadableStreamGetReaderOptions | undefined): ReadableStreamReader<...>; }' is not comparable to type '{ (): ReadableStreamDefaultReader<Uint8Array<ArrayBufferLike>>; (options: { mode: \"byob\"; }): ReadableStreamBYOBReader; (options?: ReadableStreamGetReaderOptions | undefined): ReadableStreamReader<...>; }'. Target signature provides too few arguments. Expected 1 or more, but got 0.": 1,
|
||||
"test/helpers/setup-gbrain-sandbox.ts\tTS2345\tArgument of type '(event: any) => { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }[] | { message: { content: any; id: any; role: any; }; parent_tool_use_id: any; session_id: any; type: any; }[]' is not assignable to parameter of type '(this: undefined, value: unknown, index: number, array: unknown[]) => readonly { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }[] | { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }'. Type '{ claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }[] | { message: { content: any; id: any; role: any; }; parent_tool_use_id: any; session_id: any; type: any; }[]' is not assignable to type 'readonly { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }[] | { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }'. Type '{ message: { content: any; id: any; role: any; }; parent_tool_use_id: any; session_id: any; type: any; }[]' is not assignable to type 'readonly { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }[] | { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }'. Type '{ message: { content: any; id: any; role: any; }; parent_tool_use_id: any; session_id: any; type: any; }[]' is not assignable to type 'readonly { claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }[]'. Type '{ message: { content: any; id: any; role: any; }; parent_tool_use_id: any; session_id: any; type: any; }' is missing the following properties from type '{ claude_code_version: any; cwd: any; model: any; session_id: any; subtype: any; tools: any; type: any; }': subtype, cwd, model, tools, claude_code_version": 1,
|
||||
"test/helpers/shared-libs-checker-evidence.ts\tTS18048\t'current' is possibly 'undefined'.": 1,
|
||||
"test/helpers/shared-libs-eval-fixture.ts\tTS7006\tParameter 'candidate' implicitly has an 'any' type.": 2,
|
||||
"test/helpers/shared-libs-path-fixture.ts\tTS2352\tConversion of type '{ env: { GSTACK_HOME: string; }; repo: string; root: string; state: string; }' to type 'SharedLibsFixture' may be a mistake because neither type sufficiently overlaps with the other. If this was intentional, convert the expression to 'unknown' first. Type '{ env: { GSTACK_HOME: string; }; repo: string; root: string; state: string; }' is missing the following properties from type 'SharedLibsFixture': bin, trace, hookTrace, tip": 1,
|
||||
"test/helpers/shared-libs-plan-actor.ts\tTS18046\t'questions' is of type 'unknown'.": 1,
|
||||
@@ -431,7 +432,6 @@
|
||||
"test/setup-gbrain-path4-caller.test.ts\tTS2339\tProperty 'unregistered' does not exist on type '{ 'no-init': number; 'no-install': number; 'no-registration': number; 'no-verifier': number; }'.": 1,
|
||||
"test/setup-gbrain-path4-caller.test.ts\tTS2339\tProperty 'wrong-engine' does not exist on type '{ 'no-init': number; 'no-install': number; 'no-registration': number; 'no-verifier': number; }'.": 1,
|
||||
"test/setup-gbrain-remote-caller.test.ts\tTS18048\t'lateDecision' is possibly 'undefined'.": 1,
|
||||
"test/shared-libs-checker-interface-evidence.test.ts\tTS18048\t'current' is possibly 'undefined'.": 1,
|
||||
"test/shared-libs-fixture.test.ts\tTS18046\t'packet' is of type 'unknown'.": 4,
|
||||
"test/shared-libs-fixture.test.ts\tTS2345\tArgument of type '(options: { label: string; } | { description: string; label: string; } | { description: string; label: string; }) => Promise<void>' is not assignable to parameter of type '(...args: [{ label: string; } | { description: string; label: string; } | { description: string; label: string; }, done: (err?: unknown) => void] | [{ label: string; } | { description: string; label: string; } | { ...; }, { ...; } | undefined, done: (err?: unknown) => void]) => void | Promise<...>'. Types of parameters 'options' and 'args' are incompatible. Type '[{ label: string; } | { description: string; label: string; } | { description: string; label: string; }, done: (err?: unknown) => void] | [{ label: string; } | { description: string; label: string; } | { description: string; label: string; }, { ...; } | undefined, done: (err?: unknown) => void]' is not assignable to type '[options: { label: string; } | { description: string; label: string; } | { description: string; label: string; }]'. Type '[{ label: string; } | { description: string; label: string; } | { description: string; label: string; }, done: (err?: unknown) => void]' is not assignable to type '[options: { label: string; } | { description: string; label: string; } | { description: string; label: string; }]'. Source has 2 element(s) but target allows only 1.": 1,
|
||||
"test/shared-libs-fixture.test.ts\tTS2345\tArgument of type '(options: { label: string; } | { description: string; label: string; } | { label: string; preview: string; } | { description: string; label: string; preview: string; } | { label: string; } | { description: string; label: string; } | { ...; }) => Promise<...>' is not assignable to parameter of type '(...args: [{ label: string; } | { description: string; label: string; } | { label: string; preview: string; } | { description: string; label: string; preview: string; } | { label: string; } | { description: string; label: string; } | { ...; }, done: (err?: unknown) => void] | [...]) => void | Promise<...>'. Types of parameters 'options' and 'args' are incompatible. Type '[{ label: string; } | { description: string; label: string; } | { label: string; preview: string; } | { description: string; label: string; preview: string; } | { label: string; } | { description: string; label: string; } | { ...; }, done: (err?: unknown) => void] | [...]' is not assignable to type '[options: { label: string; } | { description: string; label: string; } | { label: string; preview: string; } | { description: string; label: string; preview: string; } | { label: string; } | { description: string; label: string; } | { ...; }]'. Type '[{ label: string; } | { description: string; label: string; } | { label: string; preview: string; } | { description: string; label: string; preview: string; } | { label: string; } | { description: string; label: string; } | { ...; }, done: (err?: unknown) => void]' is not assignable to type '[options: { label: string; } | { description: string; label: string; } | { label: string; preview: string; } | { description: string; label: string; preview: string; } | { label: string; } | { description: string; label: string; } | { ...; }]'. Source has 2 element(s) but target allows only 1.": 1,
|
||||
|
||||
@@ -27,10 +27,18 @@
|
||||
# teardown) times the size's price. Without a key it runs the same files
|
||||
# locally in parallel shards and reports spent=0.
|
||||
#
|
||||
# Reattach: the tree and the planned set's digest go into ubi-runner's run key
|
||||
# (--key-input), so a rerun after the local runner died (a machine stop, a
|
||||
# SIGKILL) reattaches to the VM still running this gate instead of paying for a
|
||||
# second one. Its minutes since created_at count against the cap: the wall
|
||||
# budget shrinks by them (PREGATE_REMOTE_CAP_SPENT when none is left), and
|
||||
# spent= bills from the original create.
|
||||
#
|
||||
# Exit codes: 0 pass (or a paid-only selection, tests=0); 1 test failures (or
|
||||
# a rejected file list); 2 missing input; 3 tree or plan mismatch (or the tree
|
||||
# moved during the run); 4 infrastructure failure (create, setup, SSH, stop
|
||||
# deadline); 5 the spend cap or pregate's deadline refuses the run. Every refusal line starts with a
|
||||
# deadline, another gate on this machine owning the VM to reattach to); 5 the
|
||||
# spend cap or pregate's deadline refuses the run. Every refusal line starts with a
|
||||
# PREGATE_REMOTE_* reason code (docs/pregate.md#remote-gate). Nothing is
|
||||
# uploaded or spent before the local tree check.
|
||||
set -euo pipefail
|
||||
@@ -147,8 +155,6 @@ if [ -n "$outer_s" ]; then
|
||||
fi
|
||||
if [ "$outer_stop" -lt "$stop_after" ]; then stop_after=$outer_stop stop_why="$outer_from ${outer_s}s minus 120 s teardown reserve and 15 s margin"; fi
|
||||
fi
|
||||
say "$count files on one $size VM; worst case \$$(usd "$wall_min") under the cap \$$cap; the VM is told to stop after ${stop_after}s ($stop_why)"
|
||||
|
||||
digest=$(tr -d '\r' <"$files" | grep . | LC_ALL=C sort | sha256sum | cut -d' ' -f1)
|
||||
prelude="$logs/vm-setup.sh"
|
||||
{
|
||||
@@ -183,6 +189,28 @@ args=(run -s "$size" --src "$root" --setup "$prelude"
|
||||
--pull "/tmp/gstack-free-test-*:$logs"
|
||||
--pull "work/$(basename "$root")/.context/free-test-logs:$logs")
|
||||
[ -z "${GSTACK_FREE_JOBS:-}" ] || args+=(--env "GSTACK_FREE_JOBS=$GSTACK_FREE_JOBS")
|
||||
args+=(--key-input "pregate-remote tree=$tree plan=$digest")
|
||||
|
||||
# A rerun of this gate for the same tree and planned set has the same
|
||||
# ubi-runner run key, so it reattaches to the VM a killed run left behind
|
||||
# (`ubi-runner: reattached to <vm> created_at=<epoch>`). The minutes that VM
|
||||
# has billed since created_at come off the cap's wall budget first.
|
||||
prior_vm=""
|
||||
attach=$(bash "$here/ubi-runner.sh" "${args[@]}" --probe -- "bash ~/.pregate-run.sh" 2>/dev/null \
|
||||
| sed -n 's/^UBI_ATTACH_INFO: vm=\([^ ]*\) created_at=\([0-9]*\) key=[0-9a-f]* busy=\([01]\)$/\1 \2 \3/p' | tail -1) || attach=""
|
||||
if [ -n "$attach" ]; then
|
||||
read -r prior_vm prior_created prior_busy <<<"$attach"
|
||||
[ "$prior_busy" = 0 ] || stop 4 PREGATE_REMOTE_ATTACH_BUSY "another gate on this machine still runs VM $prior_vm for this tree and planned set" "wait for it; this run created nothing and spent nothing"
|
||||
prior_min=$(( ($(date +%s) - prior_created + 59) / 60 ))
|
||||
left_s=$(( (wall_min - 3 - prior_min) * 60 ))
|
||||
if [ "$left_s" -le 0 ]; then
|
||||
stop 5 PREGATE_REMOTE_CAP_SPENT "VM $prior_vm, created ${prior_min} min ago for this tree and planned set, has already used the cap's $wall_min min wall (\$$(usd "$prior_min") of \$$cap)" \
|
||||
"nothing reattached; destroy it: bash scripts/ubicloud/ubi-runner.sh down $prior_vm" "or rerun with a higher --spend-cap to let it finish: $rerun"
|
||||
fi
|
||||
if [ "$left_s" -lt "$stop_after" ]; then stop_after=$left_s stop_why="the cap's $wall_min min wall minus 3 min for teardown and $prior_min min already billed since $prior_vm was created"; fi
|
||||
say "reattaching to VM $prior_vm, created ${prior_min} min ago by an earlier run of this gate for the same tree and planned set"
|
||||
fi
|
||||
say "$count files on one $size VM; worst case \$$(usd "$wall_min") under the cap \$$cap; the VM is told to stop after ${stop_after}s ($stop_why)"
|
||||
|
||||
# The VM run gets its own session (python3, which ubi-runner already needs, for
|
||||
# setsid on Linux and macOS): pregate's --timeout kill signals this
|
||||
@@ -220,7 +248,7 @@ while :; do
|
||||
rm -f "$log.pid" "$log.timedout" "$log.left"
|
||||
t0=$(date +%s)
|
||||
vm_name="ubirun-$(bash "$here/ubi-runner.sh" owner)-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')-t$(( (stop_after + 900 + 3599) / 3600 ))"
|
||||
say "VM $vm_name; if this run is killed before its teardown line, destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name"
|
||||
[ -n "$prior_vm" ] || say "VM $vm_name; if this run is killed before its teardown line, destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name"
|
||||
UBI_LEFT_FILE="$log.left" python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
|
||||
bash -c "$vm_session" pregate-remote-vm "$stop_after" "$log" "$stream" "$here/ubi-runner.sh" "${args[@]}" -n "$vm_name" &
|
||||
vmrun=$!
|
||||
@@ -236,6 +264,9 @@ while :; do
|
||||
tailer=""
|
||||
t_end=$(date +%s)
|
||||
t_create=$(at 'ubi-runner: creating ')
|
||||
# A reattached VM is billed from its original create.
|
||||
reattached=$(sed -n 's/^[0-9]* ubi-runner: reattached to [^ ]* created_at=\([0-9][0-9]*\) .*/\1/p' "$log" | tail -1)
|
||||
[ -z "$reattached" ] || t_create=$reattached
|
||||
if [ -n "$t_create" ] && ! grep -q 'ubi-runner: create failed' "$log"; then minutes=$(( (t_end - t_create + 59) / 60 )); billed=$(( billed + (minutes > 0 ? minutes : 1) )); fi
|
||||
if [ "$rc" -ne 0 ] && [ "$attempt" -eq 1 ] && grep -q 'vCPU quota exhausted' "$log"; then
|
||||
say "vCPU quota refused the VM; one retry in ${PREGATE_REMOTE_QUOTA_WAIT_S:-60} s"
|
||||
@@ -279,7 +310,7 @@ if [ -n "$result" ] && [ "$rc" -ne 124 ]; then
|
||||
echo "PREGATE_REMOTE: tree=$vm_tree tests=$(field tests "$result") failed=$(field failed "$result") spent=$spent"
|
||||
exit $(( rc == 0 ? 0 : 1 ))
|
||||
fi
|
||||
vm=$(sed -n 's/^[0-9]* ubi-runner: creating \([^ ]*\) .*/\1/p' "$log" | tail -1)
|
||||
vm=$(sed -n -e 's/^[0-9]* ubi-runner: creating \([^ ]*\) .*/\1/p' -e 's/^[0-9]* ubi-runner: reattached to \([^ ]*\) .*/\1/p' "$log" | tail -1)
|
||||
down=$(grep -E 'ubi-runner: (destroyed|gone:|WARNING|FAILED to destroy)' "$log" | tail -1 | cut -d' ' -f2- || true)
|
||||
cause=$(grep -E 'ubi-runner: ' "$log" | grep -vE 'ubi-runner: (destroyed|gone:|destroy requested|waiting for an in-flight)' | tail -1 | cut -d' ' -f2- || true)
|
||||
[ "$rc" -ne 124 ] || cause="the VM was told to stop after ${stop_after}s ($stop_why) and torn down"
|
||||
|
||||
+628
-40
@@ -33,8 +33,9 @@
|
||||
# Teardown: `up` records each VM's state (location, create status) before it
|
||||
# sends the create request, and `down` destroys and polls until the VM is
|
||||
# confirmed gone or provably never existed, waiting out a create that may still
|
||||
# be in flight. An interrupted `up` or `run` (EXIT, INT, TERM, HUP, QUIT) runs
|
||||
# that `down`; further signals are ignored until it finishes. A create refused
|
||||
# be in flight. An interrupted `up` or `run` (EXIT, INT, TERM, QUIT, and HUP
|
||||
# until a remote phase is live) runs that `down`; further signals are ignored
|
||||
# until it finishes. A create refused
|
||||
# for vCPU quota prints the project's usage per owner.
|
||||
#
|
||||
# The owner-age sweep is off by default (the TTL reap below always runs). With UBI_GC_HOURS set to a positive
|
||||
@@ -69,6 +70,33 @@
|
||||
# UBI_RUN_TIMEOUT seconds (0, unlimited), exiting 124 with the last 40 lines,
|
||||
# so a hung verify fails fast instead of holding the VM until its TTL.
|
||||
#
|
||||
# Detached phases and reattach: `run` and `verify` start their setup, command
|
||||
# and tier phases on the VM detached from the ssh connection (setsid; stdout to
|
||||
# ~/.ubirun-<phase>.out, stderr to .err, the session leader's pid in .pid, and
|
||||
# the exit status in .exit, written by rename once the streams are closed) and
|
||||
# stream both files back. Before creating a VM they compute a run key locally
|
||||
# (the checkout's content tree, the command, the setup script's content, every
|
||||
# --env entry, each --pass name with a sha256 of its value, size, location,
|
||||
# image, storage, origin/HEAD, tiers and this script's own sha256) and look for
|
||||
# a run record (<state dir>/<vm>/run) with the same key whose VM still exists:
|
||||
# its runner alive (lease pid and start time) refuses with UBI_ATTACH_BUSY,
|
||||
# otherwise the run reattaches, printing `reattached to <vm> created_at=<epoch>
|
||||
# key=<k12>`, and continues from the first phase without an exit file. After
|
||||
# unpack the VM recomputes the tree and refuses a mismatch (UBI_VM_TREE_MISMATCH).
|
||||
# A leader gone with no exit file exits 125 (UBI_ATTACH_LOST). Signals: INT
|
||||
# and TERM tear the VM down as before; HUP once a remote phase is live detaches
|
||||
# (UBI_DETACHED, exit 129) and leaves the VM under its TTL; SIGKILL or a machine
|
||||
# stop leaves the record and the VM, and the next run of the same command
|
||||
# reattaches. `attach NAME` resumes a recorded run explicitly. Reattach never
|
||||
# extends a TTL, and the reaper still applies.
|
||||
#
|
||||
# Tier checkpoints: a .gstack/ubicloud.json with `tiers` runs each tier as its
|
||||
# own detached phase after setup, and records start/pass/fail per tier under
|
||||
# UBI_CHECKPOINT_DIR/<run key>/ (write-once JSON plus the tier's log, published
|
||||
# under a lock with a monotonic generation). A rerun reuses a pass younger than
|
||||
# UBI_CHECKPOINT_MAX_AGE whose log re-hashes, so a resume on a fresh VM reruns
|
||||
# only setup and the tiers without a pass.
|
||||
#
|
||||
# `verify` is the one entry point for a repository's full remote verify: it
|
||||
# reads .gstack/ubicloud.json (setup, command, size, location, env, pass, pull)
|
||||
# and runs it through `run`, so lanes share one committed definition instead of
|
||||
@@ -100,6 +128,11 @@ PREFIX="ubirun"
|
||||
if [ -n "${UBI_VM_RECORD:-}" ]; then VM_RECORD="$UBI_VM_RECORD"
|
||||
elif [ -d "$HOME/.capy/work" ]; then VM_RECORD="$HOME/.capy/work/ubi-runner/vms.tsv"
|
||||
else VM_RECORD="$STATE_ROOT/vms.tsv"; fi
|
||||
if [ -n "${UBI_CHECKPOINT_DIR:-}" ]; then CHECKPOINT_DIR="$UBI_CHECKPOINT_DIR"
|
||||
elif [ -d "$HOME/.capy/work" ]; then CHECKPOINT_DIR="$HOME/.capy/work/ubi-runner/checkpoints"
|
||||
else CHECKPOINT_DIR="$STATE_ROOT/checkpoints"; fi
|
||||
CHECKPOINT_MAX_AGE="${UBI_CHECKPOINT_MAX_AGE:-86400}"
|
||||
REATTACH_MARGIN="${UBI_REATTACH_MARGIN:-900}"
|
||||
|
||||
die() { echo "ubi-runner: $*" >&2; exit 1; }
|
||||
log() { echo "ubi-runner: $*" >&2; }
|
||||
@@ -109,7 +142,7 @@ for bin in curl python3 ssh ssh-keygen tar; do
|
||||
done
|
||||
UBICLOUD_API_KEY="${UBICLOUD_API_KEY:-${UBICLOUD_API_TOKEN:-}}"
|
||||
[[ "$GC_HOURS" =~ ^[0-9]+$ ]] || die "UBI_GC_HOURS must be a whole number of hours (unset or 0 disables the sweep)"
|
||||
for v in IDLE_TIMEOUT RUN_TIMEOUT HEARTBEAT QUOTA_WAIT; do
|
||||
for v in IDLE_TIMEOUT RUN_TIMEOUT HEARTBEAT QUOTA_WAIT CHECKPOINT_MAX_AGE REATTACH_MARGIN; do
|
||||
[[ "${!v}" =~ ^[0-9]+$ ]] || die "UBI_$v must be a whole number of seconds (0 disables it)"
|
||||
done
|
||||
[ -z "$QUOTA_VCPUS" ] || [[ "$QUOTA_VCPUS" =~ ^[0-9]+$ ]] || die "UBI_QUOTA_VCPUS must be a whole number of vCPUs"
|
||||
@@ -296,6 +329,410 @@ watched() {
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
proc_start() { ps -o lstart= -p "$1" 2>/dev/null | tr -s ' ' || true; }
|
||||
fsize() { if [ -f "$1" ]; then wc -c <"$1" | tr -d ' '; else echo 0; fi; }
|
||||
sha256() { if command -v sha256sum >/dev/null 2>&1; then sha256sum | cut -d' ' -f1; else shasum -a 256 | cut -d' ' -f1; fi; }
|
||||
|
||||
# TREE_SH: print the content tree of the current directory, the same way here
|
||||
# and on the VM: at a checkout's root, `git add -A` into a copy of its index
|
||||
# (tracked plus untracked files that are not ignored, the files pack uploads);
|
||||
# elsewhere, the same into a scratch repository over the directory.
|
||||
TREE_SH='idx=$(mktemp -d "${TMPDIR:-/tmp}/ubirun-tree.XXXXXX") || exit 1
|
||||
if [ "$(git rev-parse --show-toplevel 2>/dev/null)" = "$(pwd -P)" ]; then
|
||||
gi=$(git rev-parse --git-path index)
|
||||
if [ "$(git config --bool core.splitIndex)" != true ] && [ -f "$gi" ]; then cp "$gi" "$idx/index"; fi
|
||||
else
|
||||
git init -q --bare "$idx/git" && export GIT_DIR="$idx/git" GIT_WORK_TREE=.
|
||||
fi
|
||||
GIT_INDEX_FILE="$idx/index" git add -A >/dev/null 2>&1 && GIT_INDEX_FILE="$idx/index" git write-tree
|
||||
rc=$?; rm -rf "$idx"; exit $rc'
|
||||
|
||||
# vm_tree NAME QDEST: the content tree of QDEST on the VM.
|
||||
vm_tree() {
|
||||
remote "$1" "command -v git >/dev/null || { sudo apt-get update -qq && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git; } >/dev/null 2>&1; cd $2 && bash -c $(sq "$TREE_SH")" </dev/null
|
||||
}
|
||||
|
||||
base_sha() { git -C "$1" rev-parse -q --verify 'refs/remotes/origin/HEAD^{commit}' 2>/dev/null || echo none; }
|
||||
|
||||
# phase_status NAME PHASE: "exit <rc>" once the phase wrote its exit file,
|
||||
# "alive" while its session leader runs, "lost" when the leader is gone with no
|
||||
# exit file, "none" when it never started.
|
||||
phase_status() {
|
||||
local p="~/.ubirun-$2"
|
||||
remote "$1" "if [ -f $p.exit ]; then echo exit \$(cat $p.exit); elif [ -s $p.pid ] && kill -0 \$(cat $p.pid) 2>/dev/null; then echo alive; elif [ -s $p.pid ]; then echo lost; else echo none; fi" </dev/null
|
||||
}
|
||||
|
||||
# phase_start NAME PHASE INNER: start INNER on the VM in its own session,
|
||||
# detached from this ssh connection. The leader records its pid, runs INNER with
|
||||
# stdout and stderr in files, and writes the exit status by rename only after
|
||||
# INNER's streams are closed.
|
||||
phase_start() {
|
||||
local p="~/.ubirun-$2" inner
|
||||
inner=$(sq "echo \$\$ >$p.pid.tmp && mv $p.pid.tmp $p.pid; ( $3 ) >$p.out 2>$p.err </dev/null; echo \$? >$p.exit.tmp && mv $p.exit.tmp $p.exit")
|
||||
remote "$1" "rm -f $p.exit $p.pid; : >$p.out; : >$p.err; if command -v setsid >/dev/null 2>&1; then setsid -f bash -c $inner </dev/null >/dev/null 2>&1; else nohup bash -c $inner </dev/null >/dev/null 2>&1 & fi" </dev/null >/dev/null
|
||||
}
|
||||
|
||||
# stream_cmd PHASE OUT_OFFSET ERR_OFFSET: follow both output files from the
|
||||
# given byte offsets until the session leader exits.
|
||||
stream_cmd() {
|
||||
local p="~/.ubirun-$1"
|
||||
printf '%s' "i=0; while [ ! -s $p.pid ] && [ ! -f $p.exit ] && [ \$i -lt 600 ]; do sleep 0.1; i=\$((i + 1)); done; pid=\$(cat $p.pid 2>/dev/null); f=; [ -z \"\$pid\" ] || f=\"-f --pid=\$pid\"; tail -c +$(( $3 + 1 )) \$f $p.err >&2 & tail -c +$(( $2 + 1 )) \$f $p.out; wait"
|
||||
}
|
||||
|
||||
PHASE_LIVE="" STREAM_PID="" RUN_KEY="" TREE=""
|
||||
|
||||
# on_hup: once a remote phase is live, HUP detaches: the VM keeps running under
|
||||
# its TTL and the next run of the same command reattaches. Before that, HUP
|
||||
# tears down like INT and TERM.
|
||||
on_hup() {
|
||||
[ -n "$PHASE_LIVE" ] || exit 130
|
||||
[ -z "$STREAM_PID" ] || kill -TERM -- "-$STREAM_PID" 2>/dev/null || true
|
||||
trap - EXIT
|
||||
rm -f "$(state_dir "$RUN_VM")/lease"
|
||||
leftover "left=1 owner=$OWNER vm=$RUN_VM detached=1"
|
||||
echo "UBI_DETACHED: vm=$RUN_VM key=${RUN_KEY:0:12} reattach: rerun the same command" >&2
|
||||
exit 129
|
||||
}
|
||||
|
||||
# detached NAME PHASE INNER: run INNER as a detached phase (or reattach to it),
|
||||
# streaming its output to this process and to the VM's state dir
|
||||
# (<phase>.out, <phase>.err and the combined <phase>.log) under the same
|
||||
# heartbeat and watchdog as watched. RUN_TIMEOUT counts from the phase's
|
||||
# recorded start, not from a reattach. Returns the phase's exit status, 124
|
||||
# after a watchdog kill, 125 when the session is gone without an exit status.
|
||||
detached() {
|
||||
local name=$1 phase=$2 inner=$3 dir st rc=0 tries=0 start now idle elapsed last_beat why old_traps spid sst oo oe
|
||||
load "$name"
|
||||
dir=$(state_dir "$name")
|
||||
local lout="$dir/$phase.out" lerr="$dir/$phase.err" out="$dir/$phase.log" p="~/.ubirun-$phase"
|
||||
st=none
|
||||
if [ -f "$dir/phase.$phase.start" ]; then
|
||||
st=$(phase_status "$name" "$phase") || st=""
|
||||
fi
|
||||
case $st in
|
||||
none)
|
||||
: >"$lout"; : >"$lerr"; : >"$out"
|
||||
date +%s >"$dir/phase.$phase.start"
|
||||
phase_start "$name" "$phase" "$inner" || { log "could not start $phase on $name"; return 255; } ;;
|
||||
alive) log "$phase is still running on $name; following it" ;;
|
||||
"exit "*) ;;
|
||||
lost) log "UBI_ATTACH_LOST: vm=$name phase=$phase: its session is gone and left no exit status ($p.exit)"; return 125 ;;
|
||||
*) log "could not read the state of $phase on $name"; return 255 ;;
|
||||
esac
|
||||
PHASE_LIVE=1
|
||||
start=$(cat "$dir/phase.$phase.start")
|
||||
# A runner killed on this machine leaves its stream client behind; it would
|
||||
# append to the same local copies.
|
||||
if [ -f "$dir/stream" ] && read -r spid sst <"$dir/stream" && [ -n "$spid" ] && [ "$(proc_start "$spid")" = "$sst" ]; then
|
||||
kill -TERM -- "-$spid" 2>/dev/null || true
|
||||
fi
|
||||
oo=$(fsize "$lout") oe=$(fsize "$lerr")
|
||||
if [ "$st" != none ]; then head -c "$oo" "$lout"; head -c "$oe" "$lerr" >&2; fi
|
||||
old_traps=$(trap -p INT TERM)
|
||||
trap '[ -z "$STREAM_PID" ] || kill -TERM -- "-$STREAM_PID" 2>/dev/null; exit 130' INT TERM
|
||||
last_beat=$(date +%s)
|
||||
while :; do
|
||||
touch "$out"
|
||||
# shellcheck disable=SC2046
|
||||
( exec python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
|
||||
ssh $(ssh_opts) "ubi@$IP" "umask 022; $(stream_cmd "$phase" "$oo" "$oe")" ) \
|
||||
</dev/null > >(tee -a "$lout" | tee -a "$out") 2> >(tee -a "$lerr" | tee -a "$out" >&2) &
|
||||
STREAM_PID=$!
|
||||
echo "$STREAM_PID $(proc_start "$STREAM_PID")" >"$dir/stream"
|
||||
why=""
|
||||
while :; do
|
||||
sleep 1
|
||||
kill -0 "$STREAM_PID" 2>/dev/null || break
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - start ))
|
||||
idle=$(( now - $(mtime "$out") ))
|
||||
if [ "$IDLE_TIMEOUT" -gt 0 ] && [ "$idle" -ge "$IDLE_TIMEOUT" ]; then
|
||||
why="printed nothing for ${idle}s (UBI_IDLE_TIMEOUT=$IDLE_TIMEOUT)"
|
||||
elif [ "$RUN_TIMEOUT" -gt 0 ] && [ "$elapsed" -ge "$RUN_TIMEOUT" ]; then
|
||||
why="ran ${elapsed}s (UBI_RUN_TIMEOUT=$RUN_TIMEOUT)"
|
||||
fi
|
||||
[ -z "$why" ] || break
|
||||
if [ "$HEARTBEAT" -gt 0 ] && [ $(( now - last_beat )) -ge "$HEARTBEAT" ]; then
|
||||
log "heartbeat: $phase ${elapsed}s elapsed, last output ${idle}s ago"
|
||||
last_beat=$now
|
||||
fi
|
||||
done
|
||||
if [ -n "$why" ]; then
|
||||
log "watchdog: $phase $why; killing it. Last output:"
|
||||
tail -n 40 "$out" >&2
|
||||
kill -TERM -- "-$STREAM_PID" 2>/dev/null || kill "$STREAM_PID" 2>/dev/null || true
|
||||
wait "$STREAM_PID" 2>/dev/null || true
|
||||
STREAM_PID=""
|
||||
# A phase that finished in the instant before the kill keeps its exit 0.
|
||||
if [ "$(phase_status "$name" "$phase" 2>/dev/null)" = "exit 0" ]; then
|
||||
log "watchdog: $phase finished before it could be killed; its exit 0 stands"
|
||||
rc=0
|
||||
else
|
||||
remote_cancel "$name" "$p.pid"
|
||||
rc=124
|
||||
fi
|
||||
break
|
||||
fi
|
||||
wait "$STREAM_PID" 2>/dev/null || true
|
||||
STREAM_PID=""
|
||||
st=$(phase_status "$name" "$phase") || st=""
|
||||
case $st in
|
||||
"exit "*) rc=${st#exit }; break ;;
|
||||
lost|none) log "UBI_ATTACH_LOST: vm=$name phase=$phase: its session is gone and left no exit status ($p.exit)"; rc=125; break ;;
|
||||
esac
|
||||
tries=$(( tries + 1 ))
|
||||
if [ "$tries" -ge 5 ]; then log "lost the connection to $phase on $name 5 times; giving up (the VM keeps it until teardown)"; rc=255; break; fi
|
||||
log "the stream of $phase on $name dropped; following it again"
|
||||
sleep "$POLL"
|
||||
oo=$(fsize "$lout") oe=$(fsize "$lerr")
|
||||
done
|
||||
rm -f "$dir/stream"
|
||||
trap - INT TERM; eval "$old_traps"
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# write_record DIR: the run record a later run of the same command reattaches
|
||||
# by (key, tree, and this invocation for `attach NAME`). Remote files of each
|
||||
# phase: ~/.ubirun-<phase>.{out,err,exit,pid} (setup, command, tier-<name>).
|
||||
qwords() { local x; for x in "$@"; do printf '%q ' "$x"; done; }
|
||||
write_record() {
|
||||
{
|
||||
printf 'RUN_KEY=%q\nTREE=%q\nPULL_ROOT=%q\nRESUME=%q\nRERUN_FAILED=%q\n' "$RUN_KEY" "$TREE" "$PULL_ROOT" "$RESUME" "$RERUN_FAILED"
|
||||
printf 'RUN_ARGS=(%s)\n' "$(qwords "${RUN_ARGS[@]}")"
|
||||
printf 'TIER_NAMES=(%s)\n' "$(qwords ${TIER_NAMES[@]+"${TIER_NAMES[@]}"})"
|
||||
printf 'TIER_CMDS=(%s)\n' "$(qwords ${TIER_CMDS[@]+"${TIER_CMDS[@]}"})"
|
||||
echo "# remote files: ~/.ubirun-<phase>.{out,err,exit,pid}; phase starts: phase.<phase>.start; lease: lease (pid start time)"
|
||||
} >"$1/run.$$" && mv "$1/run.$$" "$1/run"
|
||||
}
|
||||
|
||||
lease_take() { echo "$$ $(proc_start $$)" >"$1/lease.$$" && mv "$1/lease.$$" "$1/lease"; }
|
||||
lease_busy() {
|
||||
local pid st
|
||||
[ -f "$1/lease" ] || return 1
|
||||
read -r pid st <"$1/lease" || true
|
||||
[ -n "$pid" ] && [ "$pid" != "$$" ] && kill -0 "$pid" 2>/dev/null && [ "$(proc_start "$pid")" = "$st" ]
|
||||
}
|
||||
|
||||
# vm_alive NAME: the recorded VM still answers show. One confirmed gone is
|
||||
# cleaned up (state dir and scratch record), so its record is never tried again.
|
||||
vm_alive() {
|
||||
local n=$1 out code
|
||||
( load "$n"; [ "$CREATED" = yes ] && [ -n "$IP" ] ) 2>/dev/null || return 1
|
||||
out=$(mktemp)
|
||||
code=$(load "$n"; api "$out" vm "$LOCATION/$n" show)
|
||||
rm -f "$out"
|
||||
case $code in
|
||||
200) return 0 ;;
|
||||
404) log "run record for $n: the VM is gone; starting a new one"; cmd_down "$n" >/dev/null 2>&1 || true; return 1 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ttl_short NAME SETUP: true when NAME's TTL (its created_at plus the name's
|
||||
# -t<hours>, minus now) does not cover the run still to do plus
|
||||
# REATTACH_MARGIN, with the numbers in TTL_WHY. The run still to do is the
|
||||
# UBI_RUN_TIMEOUT bound of each phase that has not finished (setup when SETUP is
|
||||
# set, then the command or every tier the plan does not reuse), counted from a
|
||||
# started phase's recorded start; it is 0 when UBI_RUN_TIMEOUT is 0.
|
||||
TTL_WHY=""
|
||||
ttl_short() {
|
||||
local name=$1 dir created left now need=0 p i phases=()
|
||||
[[ "$name" =~ -t([0-9]{1,3})$ ]] || return 1
|
||||
dir=$(state_dir "$name")
|
||||
created=$(load "$name"; echo "$CREATE_AT")
|
||||
now=$(date +%s)
|
||||
left=$(( created + 10#${BASH_REMATCH[1]} * 3600 - now ))
|
||||
[ -z "$2" ] || phases+=(setup)
|
||||
if [ ${#TIER_NAMES[@]} -gt 0 ]; then
|
||||
for i in "${!TIER_NAMES[@]}"; do [ "${TIER_ACT[i]}" = reuse ] || phases+=("tier-${TIER_NAMES[i]}"); done
|
||||
else
|
||||
phases+=(command)
|
||||
fi
|
||||
if [ "$RUN_TIMEOUT" -gt 0 ]; then
|
||||
for p in "${phases[@]}"; do
|
||||
if [ -f "$dir/phase.$p.start" ]; then i=$(( RUN_TIMEOUT - (now - $(cat "$dir/phase.$p.start")) )); [ "$i" -gt 0 ] || i=0
|
||||
else i=$RUN_TIMEOUT; fi
|
||||
need=$(( need + i ))
|
||||
done
|
||||
fi
|
||||
TTL_WHY="ttl_left=${left}s needs=$(( need + REATTACH_MARGIN ))s (run still to do ${need}s + UBI_REATTACH_MARGIN=${REATTACH_MARGIN}s)"
|
||||
[ "$left" -lt $(( need + REATTACH_MARGIN )) ]
|
||||
}
|
||||
|
||||
# find_record: the VM of a run record with this RUN_KEY whose VM still exists.
|
||||
find_record() {
|
||||
local f n
|
||||
for f in "$STATE_ROOT"/*/run; do
|
||||
[ -f "$f" ] || continue
|
||||
grep -qx "RUN_KEY=$RUN_KEY" "$f" || continue
|
||||
n=$(basename "$(dirname "$f")")
|
||||
if vm_alive "$n"; then echo "$n"; return 0; fi
|
||||
done
|
||||
}
|
||||
|
||||
# Tier checkpoints: write-once records <tier>.<epoch>.<pid>.json beside a copy
|
||||
# of the tier's log, under CHECKPOINT_DIR/<run key>/, published under a lock
|
||||
# with a monotonic generation; the newest generation per tier decides.
|
||||
CKPT_PY='
|
||||
import fcntl, hashlib, json, os, shutil, sys, time
|
||||
op, d = sys.argv[1], sys.argv[2]
|
||||
def records():
|
||||
out = []
|
||||
for f in (os.listdir(d) if os.path.isdir(d) else []):
|
||||
if not f.endswith(".json") or f.startswith("."):
|
||||
continue
|
||||
try:
|
||||
r = json.load(open(os.path.join(d, f)))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if isinstance(r, dict) and isinstance(r.get("generation"), int):
|
||||
r["_file"] = f
|
||||
out.append(r)
|
||||
return out
|
||||
def newest(rs, tier):
|
||||
rs = [r for r in rs if r.get("tier") == tier]
|
||||
return max(rs, key=lambda r: r["generation"]) if rs else None
|
||||
def sha(p):
|
||||
h = hashlib.sha256()
|
||||
with open(p, "rb") as fh:
|
||||
for b in iter(lambda: fh.read(65536), b""):
|
||||
h.update(b)
|
||||
return h.hexdigest()
|
||||
if op == "decide":
|
||||
now, maxage, resume, rerun = int(sys.argv[3]), int(sys.argv[4]), sys.argv[5] == "1", sys.argv[6] == "1"
|
||||
rs = records()
|
||||
for tier in sys.argv[7:]:
|
||||
r = newest(rs, tier)
|
||||
f = r["_file"] if r else "-"
|
||||
age = now - int(r.get("finished_at") or r.get("started_at") or now) if r else 0
|
||||
dur = int(r.get("duration_s") or 0) if r else 0
|
||||
code = str(r.get("exit", "-")) if r else "-"
|
||||
log = os.path.join(d, str(r.get("log"))) if r and r.get("log") else ""
|
||||
if not resume:
|
||||
act, why = "run", "--no-resume"
|
||||
elif r is None:
|
||||
act, why = "run", "no record"
|
||||
elif r.get("status") == "start":
|
||||
act, why = "run", "interrupted (only a start record, %s)" % f
|
||||
elif r.get("status") == "pass":
|
||||
if age >= maxage:
|
||||
act, why = "run", "pass %s is %ds old (UBI_CHECKPOINT_MAX_AGE=%d)" % (f, age, maxage)
|
||||
elif not log or not os.path.isfile(log) or sha(log) != r.get("log_sha256"):
|
||||
act, why = "run", "pass %s: its log does not re-hash" % f
|
||||
else:
|
||||
act, why = "reuse", "-"
|
||||
elif r.get("status") == "fail":
|
||||
act, why = ("rerun", "prior fail %s (exit %s; --rerun-failed)" % (f, code)) if rerun else ("blocked", "-")
|
||||
else:
|
||||
act, why = "run", "unreadable record %s" % f
|
||||
print("\t".join([tier, act, f, str(age), str(dur), code, why]))
|
||||
elif op == "publish":
|
||||
tier, status, vm, started, finished, code, log, startgen, key, tree, pid = sys.argv[3:14]
|
||||
os.makedirs(d, exist_ok=True)
|
||||
with open(os.path.join(d, ".lock"), "a") as lk:
|
||||
fcntl.flock(lk, fcntl.LOCK_EX)
|
||||
rs = records()
|
||||
gen = max([r["generation"] for r in rs] + [0]) + 1
|
||||
if status != "start":
|
||||
r = newest(rs, tier)
|
||||
if r is None or str(r["generation"]) != startgen:
|
||||
print("stale writer: its start record is generation %s, the newest for %s is %s" % (startgen, tier, r["generation"] if r else "none"), file=sys.stderr)
|
||||
sys.exit(2)
|
||||
rec = {"tier": tier, "status": status, "generation": gen, "vm": vm, "key": key, "tree": tree, "started_at": int(started)}
|
||||
if status != "start":
|
||||
rec.update(finished_at=int(finished), duration_s=int(finished) - int(started), exit=int(code))
|
||||
epoch = int(time.time())
|
||||
while os.path.exists(os.path.join(d, "%s.%d.%s.json" % (tier, epoch, pid))) or os.path.exists(os.path.join(d, "%s.%d.%s.log" % (tier, epoch, pid))):
|
||||
epoch += 1
|
||||
stem = "%s.%d.%s" % (tier, epoch, pid)
|
||||
if status != "start":
|
||||
shutil.copyfile(log, os.path.join(d, stem + ".log"))
|
||||
rec["log"], rec["log_sha256"] = stem + ".log", sha(os.path.join(d, stem + ".log"))
|
||||
tmp = os.path.join(d, ".%s.tmp" % stem)
|
||||
with open(tmp, "w") as fh:
|
||||
json.dump(rec, fh, sort_keys=True)
|
||||
fh.write("\n")
|
||||
os.link(tmp, os.path.join(d, stem + ".json"))
|
||||
os.unlink(tmp)
|
||||
print(gen)
|
||||
'
|
||||
|
||||
# ckpt_publish TIER STATUS VM STARTED FINISHED EXIT LOG STARTGEN: print the new
|
||||
# record's generation; fails (nothing published) for a stale writer.
|
||||
ckpt_publish() { python3 -c "$CKPT_PY" publish "$CHECKPOINT_DIR/$RUN_KEY" "$@" "$RUN_KEY" "$TREE" "$$"; }
|
||||
|
||||
TIER_ACT=() TIER_TO_RUN=0 TIER_PASSED=0 TIER_REUSED="" TIER_FAILED="" TIER_PRIOR="" TIER_SAVED=0 TIER_BLOCKED="" TIER_BLOCKED_RC=1
|
||||
|
||||
# tier_plan: decide each tier from its newest checkpoint and print one
|
||||
# UBI_VERIFY_RESUME line per tier before anything runs.
|
||||
tier_plan() {
|
||||
local plan t act f age dur code why
|
||||
plan=$(python3 -c "$CKPT_PY" decide "$CHECKPOINT_DIR/$RUN_KEY" "$(date +%s)" "$CHECKPOINT_MAX_AGE" "$RESUME" "$RERUN_FAILED" "${TIER_NAMES[@]}") \
|
||||
|| die "verify: could not read the tier checkpoints in $CHECKPOINT_DIR/$RUN_KEY"
|
||||
while IFS=$'\t' read -r t act f age dur code why; do
|
||||
case $act in
|
||||
reuse)
|
||||
echo "UBI_VERIFY_RESUME: tier=$t reuse $f (${age}s)" >&2
|
||||
TIER_REUSED=${TIER_REUSED:+$TIER_REUSED,}$t TIER_SAVED=$(( TIER_SAVED + dur )) TIER_PASSED=$(( TIER_PASSED + 1 )) ;;
|
||||
blocked)
|
||||
echo "UBI_VERIFY_RESUME: tier=$t fail $f (${age}s, exit $code): not rerun without --rerun-failed" >&2
|
||||
if [ -z "$TIER_BLOCKED" ]; then TIER_BLOCKED=$t; if [[ "$code" =~ ^[0-9]+$ ]]; then TIER_BLOCKED_RC=$code; fi; fi ;;
|
||||
rerun)
|
||||
echo "UBI_VERIFY_RESUME: tier=$t run: $why" >&2
|
||||
TIER_PRIOR=${TIER_PRIOR:+$TIER_PRIOR,}$t:$code TIER_TO_RUN=$(( TIER_TO_RUN + 1 )) ;;
|
||||
*)
|
||||
echo "UBI_VERIFY_RESUME: tier=$t run: $why" >&2
|
||||
TIER_TO_RUN=$(( TIER_TO_RUN + 1 )) ;;
|
||||
esac
|
||||
TIER_ACT+=("$act")
|
||||
done <<<"$plan"
|
||||
}
|
||||
|
||||
verify_line() {
|
||||
echo "UBI_VERIFY: tree=${TREE:0:12} key=${RUN_KEY:0:12} tiers=${#TIER_NAMES[@]} passed=$TIER_PASSED reused=${TIER_REUSED:-none} failed=${TIER_FAILED:-none} prior_failures=${TIER_PRIOR:-none} saved_s=$TIER_SAVED"
|
||||
}
|
||||
|
||||
# run_tiers NAME QDEST: run every tier the plan did not reuse, in order, each
|
||||
# as its own detached phase; stop at the first failure and return its status.
|
||||
# A pass is recorded only when the VM's tree after the tier equals the tree
|
||||
# before it.
|
||||
run_tiers() {
|
||||
local name=$1 qdest=$2 dir i t phase before after trc started finished gen
|
||||
dir=$(state_dir "$name")
|
||||
for i in "${!TIER_NAMES[@]}"; do
|
||||
[ "${TIER_ACT[i]}" != reuse ] || continue
|
||||
t=${TIER_NAMES[i]} phase="tier-${TIER_NAMES[i]}"
|
||||
if [ ! -f "$dir/phase.$phase.start" ]; then
|
||||
before=$(vm_tree "$name" "$qdest") || before=unavailable
|
||||
echo "$before" >"$dir/phase.$phase.tree"
|
||||
if gen=$(ckpt_publish "$t" start "$name" "$(date +%s)" 0 0 "" 0); then echo "$gen" >"$dir/phase.$phase.gen"
|
||||
else echo "UBI_CHECKPOINT_WRITE_FAILED: tier=$t start record (the tier still runs)" >&2; fi
|
||||
fi
|
||||
log "running tier $t on $name: ${TIER_CMDS[i]}"
|
||||
trc=0
|
||||
detached "$name" "$phase" "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "${TIER_CMDS[i]}")" || trc=$?
|
||||
started=$(cat "$dir/phase.$phase.start" 2>/dev/null || date +%s) finished=$(date +%s)
|
||||
log "tier $t exited $trc after $(( finished - started ))s"
|
||||
case $trc in 125|255) TIER_FAILED=$t; return "$trc" ;; esac
|
||||
gen=$(cat "$dir/phase.$phase.gen" 2>/dev/null || echo 0)
|
||||
if [ "$trc" -ne 0 ]; then
|
||||
ckpt_publish "$t" fail "$name" "$started" "$finished" "$trc" "$dir/$phase.log" "$gen" >/dev/null \
|
||||
|| echo "UBI_CHECKPOINT_WRITE_FAILED: tier=$t fail record (the result stands)" >&2
|
||||
TIER_FAILED=$t
|
||||
return "$trc"
|
||||
fi
|
||||
TIER_PASSED=$(( TIER_PASSED + 1 ))
|
||||
after=$(vm_tree "$name" "$qdest") || after=unavailable
|
||||
before=$(cat "$dir/phase.$phase.tree" 2>/dev/null || echo unavailable)
|
||||
if [ "$after" != "$before" ] || [ "$after" = unavailable ]; then
|
||||
echo "UBI_VERIFY_TREE_MOVED: tier=$t before=${before:0:12} after=${after:0:12}: a tier must leave the source tree as it found it; no pass record" >&2
|
||||
continue
|
||||
fi
|
||||
ckpt_publish "$t" pass "$name" "$started" "$finished" 0 "$dir/$phase.log" "$gen" >/dev/null \
|
||||
|| echo "UBI_CHECKPOINT_WRITE_FAILED: tier=$t pass record (the result stands)" >&2
|
||||
done
|
||||
}
|
||||
|
||||
# vm_rows: "location name owner expires created" for every VM in the project;
|
||||
# owner is "-" for untagged ubirun-<epoch>-* names and "." for VMs this runner
|
||||
# did not name; expires is the epoch the name's -t<hours> TTL ends at, or "-";
|
||||
@@ -771,9 +1208,11 @@ run_teardown() {
|
||||
[ "$rc" -ne 0 ] || exit 1
|
||||
}
|
||||
|
||||
TIER_NAMES=() TIER_CMDS=() RESUME=1 RERUN_FAILED=0 FORCE_ATTACH="" RUN_ARGS=()
|
||||
cmd_run() {
|
||||
require_owner
|
||||
local up_args=() src=. dest="" setup="" keep=0 pulls=() envs=()
|
||||
local up_args=() src=. dest="" setup="" keep=0 pulls=() envs=() passes=() key_inputs=() probe=0
|
||||
RUN_ARGS=("$@")
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
-S|--storage) check_storage "$2"; up_args+=("$1" "$2"); shift 2 ;;
|
||||
@@ -782,60 +1221,161 @@ cmd_run() {
|
||||
--dest) dest=$2; shift 2 ;;
|
||||
--setup) setup=$2; shift 2 ;;
|
||||
--env) envs+=("$2"); shift 2 ;;
|
||||
--pass) pass_name "$2"; [ -n "${!2+x}" ] || die "--pass $2: not set locally"; envs+=("$2=${!2}"); shift 2 ;;
|
||||
--pass) pass_name "$2"; [ -n "${!2+x}" ] || die "--pass $2: not set locally"; passes+=("$2"); shift 2 ;;
|
||||
--pull) pulls+=("$2"); shift 2 ;;
|
||||
--keep) keep=1; shift ;;
|
||||
--key-input) key_inputs+=("$2"); shift 2 ;;
|
||||
--probe) probe=1; shift ;;
|
||||
--resume) RESUME=1; shift ;;
|
||||
--no-resume) RESUME=0; shift ;;
|
||||
--rerun-failed) RERUN_FAILED=1; shift ;;
|
||||
--) shift; break ;;
|
||||
*) die "run: unknown option $1" ;;
|
||||
esac
|
||||
done
|
||||
[ $# -gt 0 ] || die "run: missing command after --"
|
||||
[ $# -gt 0 ] || [ ${#TIER_NAMES[@]} -gt 0 ] || die "run: missing command after --"
|
||||
[ -z "$setup" ] || [ -f "$setup" ] || die "setup script not found: $setup"
|
||||
command -v git >/dev/null || die "git is required: the run key fingerprints the checkout with it"
|
||||
src=$(cd "$src" && pwd)
|
||||
[ -n "$dest" ] || dest="work/$(basename "$src")"
|
||||
|
||||
local i name=""
|
||||
local i name="" size=$DEFAULT_SIZE location=$DEFAULT_LOCATION image=ubuntu-noble storage="" e n
|
||||
for (( i = 0; i < ${#up_args[@]}; i += 2 )); do
|
||||
case ${up_args[i]} in -n|--name) name=${up_args[i+1]} ;; esac
|
||||
case ${up_args[i]} in
|
||||
-n|--name) name=${up_args[i+1]} ;;
|
||||
-s|--size) size=${up_args[i+1]} ;;
|
||||
-l|--location) location=${up_args[i+1]} ;;
|
||||
-b|--image) image=${up_args[i+1]} ;;
|
||||
-S|--storage) storage=${up_args[i+1]} ;;
|
||||
esac
|
||||
done
|
||||
# The run key: everything that decides what the VM runs, computed before any
|
||||
# VM exists. Pass values enter only as their sha256.
|
||||
if [ -z "$FORCE_ATTACH" ]; then
|
||||
TREE=$(cd "$src" && bash -c "$TREE_SH") || die "could not fingerprint $src (git add -A into a temporary index failed)"
|
||||
RUN_KEY=$({
|
||||
echo "ubi-runner run key v1"
|
||||
echo "tree $TREE"
|
||||
printf 'command %q\n' "$*"
|
||||
if [ -n "$setup" ]; then echo "setup $(sha256 <"$setup")"; else echo "setup none"; fi
|
||||
for e in ${envs[@]+"${envs[@]}"}; do printf 'env %q\n' "$e"; done
|
||||
for n in ${passes[@]+"${passes[@]}"}; do printf 'pass %s %s\n' "$n" "$(printf %s "${!n}" | sha256)"; done
|
||||
printf 'vm %q %q %q %q\n' "$size" "$location" "$image" "$storage"
|
||||
echo "base $(base_sha "$src")"
|
||||
echo "runner $(sha256 <"${BASH_SOURCE[0]}")"
|
||||
for e in ${key_inputs[@]+"${key_inputs[@]}"}; do printf 'input %q\n' "$e"; done
|
||||
for i in "${!TIER_NAMES[@]}"; do printf 'tier %q %q\n' "${TIER_NAMES[i]}" "${TIER_CMDS[i]}"; done
|
||||
} | sha256)
|
||||
fi
|
||||
[ ${#TIER_NAMES[@]} -eq 0 ] || tier_plan
|
||||
|
||||
local attach=""
|
||||
if [ -n "$FORCE_ATTACH" ]; then
|
||||
vm_alive "$FORCE_ATTACH" || die "attach: $FORCE_ATTACH no longer exists; rerun the original command to start fresh"
|
||||
attach=$FORCE_ATTACH
|
||||
else
|
||||
attach=$(find_record)
|
||||
fi
|
||||
if [ "$probe" = 1 ]; then
|
||||
if [ -z "$attach" ]; then echo "UBI_ATTACH_INFO: none key=${RUN_KEY:0:12}"
|
||||
else
|
||||
local busy=0
|
||||
! lease_busy "$(state_dir "$attach")" || busy=1
|
||||
if [ "$busy" = 0 ] && [ -z "$FORCE_ATTACH" ] && ttl_short "$attach" "$setup"; then
|
||||
echo "UBI_ATTACH_INFO: none key=${RUN_KEY:0:12} ttl_short vm=$attach $TTL_WHY"
|
||||
else
|
||||
echo "UBI_ATTACH_INFO: vm=$attach created_at=$(load "$attach"; echo "$CREATE_AT") key=${RUN_KEY:0:12} busy=$busy"
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
if [ ${#TIER_NAMES[@]} -gt 0 ] && { [ -n "$TIER_BLOCKED" ] || [ "$TIER_TO_RUN" -eq 0 ]; } && [ -n "$attach" ] && ! lease_busy "$(state_dir "$attach")"; then
|
||||
log "verify: no tier needs $attach, which a dead runner left behind; destroying it"
|
||||
cmd_down "$attach" >/dev/null || log "WARNING: failed to destroy $attach; run: $0 down $attach"
|
||||
fi
|
||||
if [ ${#TIER_NAMES[@]} -gt 0 ] && [ -n "$TIER_BLOCKED" ]; then
|
||||
TIER_FAILED=$TIER_BLOCKED
|
||||
verify_line
|
||||
return "$TIER_BLOCKED_RC"
|
||||
fi
|
||||
if [ ${#TIER_NAMES[@]} -gt 0 ] && [ "$TIER_TO_RUN" -eq 0 ]; then
|
||||
log "verify: every tier passed under this key; no VM needed"
|
||||
verify_line
|
||||
return 0
|
||||
fi
|
||||
if [ -n "$attach" ] && lease_busy "$(state_dir "$attach")"; then
|
||||
die "UBI_ATTACH_BUSY: vm=$attach key=${RUN_KEY:0:12}: runner pid $(cut -d' ' -f1 <"$(state_dir "$attach")/lease") on this machine still owns it; wait for it or stop it, then rerun"
|
||||
fi
|
||||
if [ -n "$attach" ] && [ -z "$FORCE_ATTACH" ] && ttl_short "$attach" "$setup"; then
|
||||
log "UBI_REATTACH_TTL_SHORT: vm=$attach key=${RUN_KEY:0:12} $TTL_WHY: starting a fresh VM (tier checkpoints carry over) and destroying $attach"
|
||||
cmd_down "$attach" >/dev/null || log "WARNING: failed to destroy $attach; run: $0 down $attach"
|
||||
attach=""
|
||||
fi
|
||||
[ -z "$attach" ] || name=$attach
|
||||
[ -n "$name" ] || { name=$(new_name); up_args+=(-n "$name"); }
|
||||
RUN_VM=$name
|
||||
# Armed before the create request: an interrupted `up` still destroys its VM.
|
||||
# A second signal must not cut teardown short, so the EXIT handler ignores them.
|
||||
trap 'run_teardown "$?"' EXIT
|
||||
trap 'exit 130' INT TERM HUP QUIT
|
||||
cmd_up "${up_args[@]}" >/dev/null
|
||||
if [[ "$name" =~ -t([0-9]+)$ ]]; then
|
||||
log "$name carries a ${BASH_REMATCH[1]}h TTL (any later run of this script reaps it); if this process dies first: $0 down $name"
|
||||
elif [[ "$name" =~ ^$PREFIX-$OWNER-[0-9]{10}- ]]; then
|
||||
log "$name has no TTL (reap destroys it once it is ${LEGACY_HOURS}h old); if this process dies before its teardown line, destroy it with: $0 down $name"
|
||||
trap 'exit 130' INT TERM QUIT
|
||||
trap on_hup HUP
|
||||
if [ -n "$attach" ]; then
|
||||
load "$name"
|
||||
log "reattached to $name created_at=$CREATE_AT key=${RUN_KEY:0:12}"
|
||||
else
|
||||
log "$name has no TTL and no reapable name, so nothing reaps it; if this process dies before its teardown line, destroy it with: $0 down $name"
|
||||
cmd_up "${up_args[@]}" >/dev/null
|
||||
trap 'exit 130' INT TERM QUIT
|
||||
trap on_hup HUP
|
||||
if [[ "$name" =~ -t([0-9]+)$ ]]; then
|
||||
log "$name carries a ${BASH_REMATCH[1]}h TTL (any later run of this script reaps it); if this process dies first: $0 down $name"
|
||||
elif [[ "$name" =~ ^$PREFIX-$OWNER-[0-9]{10}- ]]; then
|
||||
log "$name has no TTL (reap destroys it once it is ${LEGACY_HOURS}h old); if this process dies before its teardown line, destroy it with: $0 down $name"
|
||||
else
|
||||
log "$name has no TTL and no reapable name, so nothing reaps it; if this process dies before its teardown line, destroy it with: $0 down $name"
|
||||
fi
|
||||
fi
|
||||
local dir
|
||||
dir=$(state_dir "$name")
|
||||
write_record "$dir"
|
||||
lease_take "$dir"
|
||||
if [ "$keep" = 1 ]; then
|
||||
trap - EXIT
|
||||
# A kept VM's finished run is not reattached by the next run; a detach (HUP) keeps the record.
|
||||
trap 'rm -f "$(state_dir "$RUN_VM")/run"' EXIT
|
||||
leftover "left=1 owner=$OWNER vm=$name kept=1"
|
||||
log "--keep: leaving $name running; destroy with: $0 down $name"
|
||||
fi
|
||||
|
||||
local e
|
||||
for e in ${envs[@]+"${envs[@]}"}; do printf 'export %s=%q\n' "${e%%=*}" "${e#*=}"; done \
|
||||
| remote "$name" 'umask 077 && cat > ~/.ubirun-env'
|
||||
|
||||
cmd_sync "$name" "$src" "$dest"
|
||||
local qdest rc=0
|
||||
local qdest rc=0 vmt
|
||||
qdest=$(printf %q "$dest")
|
||||
if [ ! -f "$dir/synced" ]; then
|
||||
{ for e in ${envs[@]+"${envs[@]}"}; do printf 'export %s=%q\n' "${e%%=*}" "${e#*=}"; done
|
||||
for n in ${passes[@]+"${passes[@]}"}; do printf 'export %s=%q\n' "$n" "${!n}"; done; } \
|
||||
| remote "$name" 'umask 077 && cat > ~/.ubirun-env'
|
||||
cmd_sync "$name" "$src" "$dest"
|
||||
# A subdirectory of a checkout uploads the whole .git with only its own
|
||||
# files, so its tree is compared VM to VM (tiers) but not to this side.
|
||||
if [ -z "$(git -C "$src" rev-parse --show-prefix 2>/dev/null)" ]; then
|
||||
vmt=$(vm_tree "$name" "$qdest") || vmt=unavailable
|
||||
if [ "$vmt" != "$TREE" ]; then
|
||||
log "UBI_VM_TREE_MISMATCH: vm=$name tree=${vmt:0:12} expected=${TREE:0:12}: the unpacked checkout is not the one this run was keyed on"
|
||||
exit 3
|
||||
fi
|
||||
fi
|
||||
[ -z "$setup" ] || remote "$name" 'cat > ~/.ubirun-setup.sh' <"$setup"
|
||||
: >"$dir/synced"
|
||||
fi
|
||||
if [ -n "$setup" ]; then
|
||||
remote "$name" 'cat > ~/.ubirun-setup.sh' <"$setup"
|
||||
log "running setup $(basename "$setup")"
|
||||
watched "$name" setup "cd $qdest && . ~/.ubirun-env && bash -l ~/.ubirun-setup.sh" || { rc=$?; log "setup failed (exit $rc)"; exit "$rc"; }
|
||||
detached "$name" setup "cd $qdest && . ~/.ubirun-env && bash -l ~/.ubirun-setup.sh" || { rc=$?; log "setup failed (exit $rc)"; exit "$rc"; }
|
||||
fi
|
||||
|
||||
local start
|
||||
start=$(date +%s)
|
||||
log "running on $name: $*"
|
||||
watched "$name" command "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "$*")" || rc=$?
|
||||
log "command exited $rc after $(( $(date +%s) - start ))s"
|
||||
if [ ${#TIER_NAMES[@]} -gt 0 ]; then
|
||||
run_tiers "$name" "$qdest" || rc=$?
|
||||
else
|
||||
log "running on $name: $*"
|
||||
detached "$name" command "cd $qdest && . ~/.ubirun-env && bash -lc $(printf %q "$*")" || rc=$?
|
||||
log "command exited $rc after $(( $(date +%s) - $(cat "$dir/phase.command.start" 2>/dev/null || date +%s) ))s"
|
||||
fi
|
||||
|
||||
local p
|
||||
for p in ${pulls[@]+"${pulls[@]}"}; do
|
||||
@@ -846,9 +1386,23 @@ cmd_run() {
|
||||
fi
|
||||
cmd_pull "$name" "${p%%:*}" "${p#*:}" || log "pull failed: $p"
|
||||
done
|
||||
[ ${#TIER_NAMES[@]} -eq 0 ] || verify_line
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# attach NAME: resume the run recorded for NAME on this machine (the same
|
||||
# invocation, key and tiers), as a rerun of the same command would.
|
||||
cmd_attach() {
|
||||
local name=${1:-}
|
||||
[ -n "$name" ] || die "attach: missing NAME"
|
||||
valid_name "$name" || die "invalid VM name '$name'"
|
||||
[ -f "$(state_dir "$name")/run" ] || die "attach: no run record for $name on this machine ($(state_dir "$name")/run); a machine replacement loses it"
|
||||
# shellcheck disable=SC1091
|
||||
. "$(state_dir "$name")/run"
|
||||
FORCE_ATTACH=$name
|
||||
cmd_run "${RUN_ARGS[@]}"
|
||||
}
|
||||
|
||||
# verify [run opts] [--trust-config] [-- COMMAND]: run the repository's
|
||||
# committed remote verify. .gstack/ubicloud.json under --src (default .)
|
||||
# supplies setup, command, size, location, storage, image, env, pass and pull;
|
||||
@@ -861,7 +1415,7 @@ cmd_run() {
|
||||
# local variable it lists under pass is forwarded only when this command line
|
||||
# also names it (--pass NAME) or grants every listed name (--trust-config).
|
||||
cmd_verify() {
|
||||
local src=. i j g n cfg cfg_setup cfg_cmd parsed err rc=0 trust=0 cfg_args=() cfg_pass=() user_pre=() user_cmd=() granted=() missing=()
|
||||
local src=. i j g n cfg cfg_setup cfg_cmd parsed err rc=0 trust=0 cfg_args=() cfg_pass=() user_pre=() user_cmd=() granted=() missing=() cfg_tier_names=() cfg_tier_cmds=()
|
||||
for (( i = 1; i <= $#; i++ )); do
|
||||
[ "${!i}" = -- ] && break
|
||||
if [ "${!i}" = --src ]; then j=$(( i + 1 )); src=${!j}; fi
|
||||
@@ -870,7 +1424,7 @@ cmd_verify() {
|
||||
[ -f "$cfg" ] || die "verify: no $cfg; commit one ({\"setup\": \"scripts/ubicloud/bootstrap.sh\", \"command\": \"bash scripts/ubicloud/verify.sh\"}) so every lane runs the same verify instead of its own wrapper"
|
||||
err=$(mktemp)
|
||||
parsed=$(python3 -c '
|
||||
import json, os, shlex, sys
|
||||
import json, os, re, shlex, sys
|
||||
cfg, src = sys.argv[1], sys.argv[2]
|
||||
try:
|
||||
c = json.load(open(cfg))
|
||||
@@ -896,7 +1450,25 @@ for item in c.get("pull", []):
|
||||
if not sep or not remote:
|
||||
print("pull entry %r must be REMOTE_GLOB:LOCAL_DIR" % (item,), file=sys.stderr); sys.exit(2)
|
||||
args += ["--pull", q(remote + ":" + inside(local, "pull destination"))]
|
||||
tiers = c.get("tiers")
|
||||
names = []
|
||||
if tiers is not None:
|
||||
if "command" in c:
|
||||
print("UBI_CONFIG_TIERS_AND_COMMAND: give either command or tiers, not both", file=sys.stderr); sys.exit(2)
|
||||
if not isinstance(tiers, list) or not tiers:
|
||||
print("UBI_CONFIG_TIER_NAME: tiers must be a non-empty list of {\"name\", \"command\"}", file=sys.stderr); sys.exit(2)
|
||||
for t in tiers:
|
||||
name = t.get("name") if isinstance(t, dict) else None
|
||||
if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,31}", name):
|
||||
print("UBI_CONFIG_TIER_NAME: tier name %r must match ^[a-z0-9][a-z0-9_-]{0,31}$" % (name,), file=sys.stderr); sys.exit(2)
|
||||
if name in names:
|
||||
print("UBI_CONFIG_TIER_NAME: tier name %r appears twice" % name, file=sys.stderr); sys.exit(2)
|
||||
if not isinstance(t.get("command"), str) or not t["command"].strip():
|
||||
print("UBI_CONFIG_TIER_NAME: tier %r needs a command" % name, file=sys.stderr); sys.exit(2)
|
||||
names.append(name)
|
||||
setup = c.get("setup", "")
|
||||
print("cfg_tier_names=(" + " ".join(q(n) for n in names) + ")")
|
||||
print("cfg_tier_cmds=(" + " ".join(q(t["command"]) for t in (tiers or [])) + ")")
|
||||
print("cfg_args=(" + " ".join(args) + ")")
|
||||
print("cfg_pass=(" + " ".join(q(x) for x in c.get("pass", [])) + ")")
|
||||
print("cfg_setup=" + q(inside(setup, "setup") if setup else ""))
|
||||
@@ -913,7 +1485,7 @@ print("cfg_cmd=" + q(c.get("command", "")))
|
||||
case $1 in
|
||||
--) shift; user_cmd=("$@"); break ;;
|
||||
--trust-config) trust=1; shift ;;
|
||||
--keep) user_pre+=("$1"); shift ;;
|
||||
--keep|--resume|--no-resume|--rerun-failed|--probe) user_pre+=("$1"); shift ;;
|
||||
--pass|-S|--storage|-s|--size|-l|--location|-n|--name|-b|--image|--src|--dest|--setup|--env|--pull)
|
||||
[ $# -ge 2 ] || die "verify: $1 needs a value"
|
||||
[ "$1" != --pass ] || granted+=("$2")
|
||||
@@ -928,11 +1500,16 @@ print("cfg_cmd=" + q(c.get("command", "")))
|
||||
missing+=("$n")
|
||||
done
|
||||
[ ${#missing[@]} -eq 0 ] || die "verify: $cfg asks to forward local ${missing[*]}; grant each on this command line with --pass NAME, or every name it lists with --trust-config"
|
||||
[ ${#user_cmd[@]} -gt 0 ] || [ -n "$cfg_cmd" ] || die "verify: $cfg names no command and none was given after --"
|
||||
[ ${#user_cmd[@]} -gt 0 ] || user_cmd=("$cfg_cmd")
|
||||
# A trailing -- COMMAND replaces the configured tiers as it replaces command.
|
||||
if [ ${#user_cmd[@]} -eq 0 ] && [ ${#cfg_tier_names[@]} -gt 0 ]; then
|
||||
TIER_NAMES=("${cfg_tier_names[@]}") TIER_CMDS=("${cfg_tier_cmds[@]}")
|
||||
else
|
||||
[ ${#user_cmd[@]} -gt 0 ] || [ -n "$cfg_cmd" ] || die "verify: $cfg names no command and none was given after --"
|
||||
[ ${#user_cmd[@]} -gt 0 ] || user_cmd=("$cfg_cmd")
|
||||
fi
|
||||
log "verify: $cfg"
|
||||
PULL_ROOT=$(cd "$src" && pwd)
|
||||
cmd_run ${cfg_args[@]+"${cfg_args[@]}"} ${cfg_setup:+--setup "$cfg_setup"} --src "$src" ${user_pre[@]+"${user_pre[@]}"} -- "${user_cmd[@]}"
|
||||
cmd_run ${cfg_args[@]+"${cfg_args[@]}"} ${cfg_setup:+--setup "$cfg_setup"} --src "$src" ${user_pre[@]+"${user_pre[@]}"} -- ${user_cmd[@]+"${user_cmd[@]}"}
|
||||
}
|
||||
|
||||
usage() {
|
||||
@@ -940,13 +1517,17 @@ usage() {
|
||||
usage: ubi-runner.sh <command> [args]
|
||||
|
||||
run [up opts] [--src DIR] [--dest PATH] [--setup FILE] [--env K=V]...
|
||||
[--pass NAME]... [--pull REMOTE_GLOB:LOCAL_DIR]... [--keep] -- COMMAND
|
||||
up + sync + setup + command + pull + destroy; exits with COMMAND's status
|
||||
verify [run opts] [--trust-config] [-- COMMAND]
|
||||
[--pass NAME]... [--pull REMOTE_GLOB:LOCAL_DIR]... [--keep] [--key-input S]... [--probe] -- COMMAND
|
||||
up + sync + setup + command + pull + destroy; exits with COMMAND's status.
|
||||
A rerun of the same command (same run key) reattaches to its live VM;
|
||||
--probe prints UBI_ATTACH_INFO (and the tier plan) and exits
|
||||
verify [run opts] [--trust-config] [--resume|--no-resume] [--rerun-failed] [-- COMMAND]
|
||||
run the repository's committed verify (.gstack/ubicloud.json under --src);
|
||||
its setup script and every pull destination stay inside --src (checked
|
||||
again at pull time), and its pass names are forwarded only when granted
|
||||
here (--pass NAME, or all: --trust-config)
|
||||
here (--pass NAME, or all: --trust-config). With tiers, a pass checkpoint
|
||||
under the same run key is reused (resume is the default)
|
||||
attach NAME resume the run recorded for NAME on this machine, as a rerun would
|
||||
up [-s SIZE] [-l LOCATION] [-n NAME] [-S STORAGE_GIB] [-b IMAGE]
|
||||
create a VM and wait for SSH; prints its name
|
||||
(STORAGE_GIB: one of $ALLOWED_STORAGE)
|
||||
@@ -966,6 +1547,11 @@ usage: ubi-runner.sh <command> [args]
|
||||
owner's TTL-less ubirun-* VMs older than UBI_LEGACY_HOURS
|
||||
cli ARGS... raw Ubicloud CLI passthrough (e.g. cli vm list)
|
||||
|
||||
signals: INT and TERM tear the VM down. HUP, once a remote phase is live, detaches
|
||||
(UBI_DETACHED, exit 129; the VM runs on under its TTL). SIGKILL or a machine stop leaves
|
||||
the run record and the VM; rerun the same command to reattach (UBI_ATTACH_BUSY while
|
||||
another runner owns it; exit 125 UBI_ATTACH_LOST when the remote session left no status).
|
||||
|
||||
VMs are named $PREFIX-<owner>-<epoch>-<suffix>-t<ttl hours>; up, run and verify need
|
||||
UBI_OWNER (e.g. your thread code); run and verify end with a UBICLOUD: left=<n> line.
|
||||
defaults: size=$DEFAULT_SIZE location=$DEFAULT_LOCATION (env UBI_SIZE, UBI_LOCATION)
|
||||
@@ -977,7 +1563,8 @@ env: UBI_TTL_HOURS (${TTL_HOURS}h in the name, 1-999), UBI_REAP (0: up skips the
|
||||
UBI_QUOTA_WAIT (${QUOTA_WAIT}s to wait for quota room instead of refusing),
|
||||
UBI_GC_HOURS (unset/0: no sweep on up), UBI_CREATE_GRACE (${CREATE_GRACE}s),
|
||||
UBI_DOWN_TIMEOUT (${DOWN_TIMEOUT}s), UBI_API_TIMEOUT (${API_TIMEOUT}s), UBI_VM_RECORD (scratch VM-id file),
|
||||
UBI_ALLOWED_STORAGE (the -S sizes Ubicloud accepts; default "320 640")
|
||||
UBI_ALLOWED_STORAGE (the -S sizes Ubicloud accepts; default "320 640"),
|
||||
UBI_CHECKPOINT_DIR (tier checkpoints; $CHECKPOINT_DIR), UBI_CHECKPOINT_MAX_AGE (${CHECKPOINT_MAX_AGE}s a pass is reused)
|
||||
EOF
|
||||
}
|
||||
|
||||
@@ -990,6 +1577,7 @@ esac
|
||||
case $cmd in
|
||||
run) cmd_run "$@" ;;
|
||||
verify) cmd_verify "$@" ;;
|
||||
attach) cmd_attach "$@" ;;
|
||||
up) trap 'trap "" INT TERM HUP QUIT; [ -z "$UP_NAME" ] || [ -n "$UP_READY" ] || cmd_down "$UP_NAME" || log "WARNING: failed to destroy $UP_NAME; run: $0 down $UP_NAME"' EXIT
|
||||
trap 'exit 130' INT TERM HUP QUIT
|
||||
cmd_up "$@" ;;
|
||||
|
||||
Reference in new issue
Block a user