From 0f26859bd1bcca32ab9dce71468e6835031c8010 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 31 Aug 2026 04:08:10 +0000 Subject: [PATCH] refactor: extract composite actions for eval-lane setup; surviving lanes gain the fail-fast registry verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'Fix bun temp' x3, 'Restore deps' x5, 'Seed claude interactive config' x3, and 'Register gstack skills' x3 were byte-near-identical copies across the legacy matrix, the sliced lane, and the periodic lane — and only the MATRIX copy of register-skills carried the 19-line dangling-symlink + frontmatter fail-fast loop written after a silent 'Unknown command' + 35-min-timeout incident. Extract all four into .github/actions/ composites; the register composite carries the verification loop (generalized over the skill list), so the sliced and periodic lanes — the lanes that SURVIVE the matrix deletion — now inherit the check they had silently dropped. Matrix-job inline copies are left untouched: that job is deleted next. Co-Authored-By: Claude Fable 5 --- .github/actions/fix-bun-temp/action.yml | 18 ++++ .../actions/register-gstack-skills/action.yml | 98 +++++++++++++++++++ .github/actions/restore-deps/action.yml | 19 ++++ .github/actions/seed-claude-config/action.yml | 30 ++++++ .github/workflows/evals-periodic.yml | 88 +++-------------- .github/workflows/evals.yml | 73 +++----------- 6 files changed, 189 insertions(+), 137 deletions(-) create mode 100644 .github/actions/fix-bun-temp/action.yml create mode 100644 .github/actions/register-gstack-skills/action.yml create mode 100644 .github/actions/restore-deps/action.yml create mode 100644 .github/actions/seed-claude-config/action.yml diff --git a/.github/actions/fix-bun-temp/action.yml b/.github/actions/fix-bun-temp/action.yml new file mode 100644 index 000000000..02a4b0444 --- /dev/null +++ b/.github/actions/fix-bun-temp/action.yml @@ -0,0 +1,18 @@ +name: Fix bun temp +description: > + Redirect bun's cache/temp to a runner-writable dir. Bun creates root-owned + temp dirs during the Docker image build; GH Actions runs container jobs as + the `runner` user with HOME=/github/home, so without this redirect bun hits + EACCES on its default temp paths. Shared by every eval lane — extracted from + three byte-identical copies (legacy matrix, sliced lane, periodic). +runs: + using: composite + steps: + - shell: bash + run: | + mkdir -p /home/runner/.cache/bun + { + echo "BUN_INSTALL_CACHE_DIR=/home/runner/.cache/bun" + echo "BUN_TMPDIR=/home/runner/.cache/bun" + echo "TMPDIR=/home/runner/.cache" + } >> "$GITHUB_ENV" diff --git a/.github/actions/register-gstack-skills/action.yml b/.github/actions/register-gstack-skills/action.yml new file mode 100644 index 000000000..6782a11dd --- /dev/null +++ b/.github/actions/register-gstack-skills/action.yml @@ -0,0 +1,98 @@ +name: Register gstack skills for PTY tests +description: > + Register the skills PTY smokes invoke, in every place claude looks: + user-scoped ($HOME/.claude/skills — a gstack root symlink for the + preamble's absolute runtime paths, plus per-skill REAL-FILE copies because + claude's interactive-TUI skill scanner does not follow the + /github/home -> /__w cross-mount symlink), and project-scoped + (/.claude/skills, which the TUI reads for /slash commands and which + is gitignored so absent on a fresh CI checkout). Also pre-seeds every + one-time preamble marker so no PTY child takes a first-run branch mid-test + (feature discovery under ~/.claude/skills/gstack trips Claude Code's + sensitive-file permission prompt — the documented intermittent + scope-gate-question-NOT-observed failure). + + Ends with the fail-fast verification loop the legacy matrix copy grew + after a silent "Unknown command" + 35-min-timeout incident: a dangling + symlink or renamed committed target fails HERE, in seconds, with a named + path — never as a wedged PTY session at the shard wall. Every consuming + lane inherits the loop by construction (it previously existed only in the + matrix copy; the sliced + periodic copies had silently dropped it). +inputs: + skills: + description: Space-separated skill dirs (each must have SKILL.md + sections/). + required: false + default: office-hours plan-ceo-review plan-eng-review plan-design-review +runs: + using: composite + steps: + - shell: bash + env: + SKILLS: ${{ inputs.skills }} + run: | + set -eu + SKILLS_DIR="$HOME/.claude/skills" + REPO="$GITHUB_WORKSPACE" + mkdir -p "$SKILLS_DIR" + # The gstack root stays a symlink — the preamble's runtime bash resolves + # ~/.claude/skills/gstack/bin/* and ~/.claude/skills/gstack//sections/* + # through it, and bash follows cross-mount symlinks fine. + ln -snf "$REPO" "$SKILLS_DIR/gstack" + for s in $SKILLS; do + rm -rf "${SKILLS_DIR:?}/$s" + mkdir -p "$SKILLS_DIR/$s" + cp "$REPO/$s/SKILL.md" "$SKILLS_DIR/$s/SKILL.md" + cp -R "$REPO/$s/sections" "$SKILLS_DIR/$s/sections" + done + PROJ_SKILLS="$REPO/.claude/skills" + mkdir -p "$PROJ_SKILLS" + for s in $SKILLS; do + rm -rf "${PROJ_SKILLS:?}/$s" + mkdir -p "$PROJ_SKILLS/$s" + cp "$REPO/$s/SKILL.md" "$PROJ_SKILLS/$s/SKILL.md" + cp -R "$REPO/$s/sections" "$PROJ_SKILLS/$s/sections" + done + # Pre-seed every ONE-TIME preamble marker so no PTY child ever takes a + # first-run branch mid-test. + mkdir -p "$HOME/.gstack" + touch "$HOME/.gstack/.activated" \ + "$HOME/.gstack/.first-loop-tip-shown" \ + "$HOME/.gstack/.telemetry-prompted" \ + "$HOME/.gstack/.proactive-prompted" \ + "$HOME/.gstack/.completeness-intro-seen" \ + "$HOME/.gstack/.plan-tune-nudge-shown" + # These two resolve through the gstack root symlink into $REPO — + # untracked scratch in the CI checkout, exactly where the preamble looks. + touch "$SKILLS_DIR/gstack/.feature-prompted-continuous-checkpoint" \ + "$SKILLS_DIR/gstack/.feature-prompted-model-overlay" + echo "--- registry under $SKILLS_DIR ---" + ls -la "$SKILLS_DIR/gstack" + # ── Fail-fast verification ────────────────────────────────────────── + # A dangling symlink or moved/renamed committed target must fail here + # with a named path, not resurface as a silent "Unknown command" and a + # PTY session wedged to its wall timeout. + for f in \ + "$SKILLS_DIR/gstack/bin/gstack-update-check" \ + "$SKILLS_DIR/gstack/scripts/gen-skill-docs.ts"; do + if [ ! -e "$f" ]; then + echo "ERROR: gstack root symlink dangles or target moved: $f" >&2 + exit 1 + fi + done + for s in $SKILLS; do + if [ ! -e "$SKILLS_DIR/$s/SKILL.md" ]; then + echo "ERROR: skill-registry target missing: $SKILLS_DIR/$s/SKILL.md" >&2 + exit 1 + fi + grep -m1 "^name: $s\$" "$SKILLS_DIR/$s/SKILL.md" >/dev/null \ + || { echo "ERROR: $s SKILL.md missing 'name: $s' frontmatter" >&2; exit 1; } + # Sections must exist BOTH as the copied real files (TUI reads) and + # through the gstack root symlink (the preamble's runtime paths). + for d in "$SKILLS_DIR/$s/sections" "$SKILLS_DIR/gstack/$s/sections"; do + if [ ! -d "$d" ] || [ -z "$(ls -A "$d")" ]; then + echo "ERROR: skill sections missing or empty: $d" >&2 + exit 1 + fi + done + done + echo "skill registry OK" diff --git a/.github/actions/restore-deps/action.yml b/.github/actions/restore-deps/action.yml new file mode 100644 index 000000000..187956a4e --- /dev/null +++ b/.github/actions/restore-deps/action.yml @@ -0,0 +1,19 @@ +name: Restore deps +description: > + Restore the CI image's pre-installed node_modules via recursive copy, or + fall back to bun install when the lockfile changed. Symlinking breaks bun's + realpath-based module resolution (realpath escapes the workspace and + sibling deps stop resolving); hardlink copy fails across overlay-fs layers + ("Invalid cross-device link"). Recursive copy costs ~5s for ~200 packages — + still far cheaper than a network install. Extracted from five byte-similar + copies across the eval lanes. +runs: + using: composite + steps: + - shell: bash + run: | + if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then + cp -r /opt/node_modules_cache node_modules + else + bun install + fi diff --git a/.github/actions/seed-claude-config/action.yml b/.github/actions/seed-claude-config/action.yml new file mode 100644 index 000000000..0ac4d32b9 --- /dev/null +++ b/.github/actions/seed-claude-config/action.yml @@ -0,0 +1,30 @@ +name: Seed claude interactive config +description: > + Seed ~/.claude.json with onboarding-complete plus the API-key approval so + PTY sessions never wedge on the fresh-container onboarding / "use detected + ANTHROPIC_API_KEY?" dialog. Mirrors what the hermetic E2E child env seeds. + Idempotent — safe to run unconditionally in sliced lanes where any slice + can host a PTY test. Only the key's last 20 chars are persisted (the + approval-hash form claude itself writes), never the whole key. +inputs: + anthropic-api-key: + description: API key whose suffix gets pre-approved. Pass from secrets at the call site. + required: false + default: '' +runs: + using: composite + steps: + - shell: bash + env: + ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} + run: | + node -e ' + const fs = require("fs"), os = require("os"), path = require("path"); + const p = path.join(os.homedir(), ".claude.json"); + const seed = fs.existsSync(p) ? JSON.parse(fs.readFileSync(p, "utf8")) : {}; + seed.hasCompletedOnboarding = true; + const key = process.env.ANTHROPIC_API_KEY || ""; + if (key) seed.customApiKeyResponses = { approved: [key.slice(-20)], rejected: [] }; + fs.writeFileSync(p, JSON.stringify(seed, null, 2)); + console.log("seeded", p); + ' diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index c7b4f1ef2..6e4ed013e 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -95,12 +95,7 @@ jobs: persist-credentials: false - name: Restore deps - run: | - if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then - cp -r /opt/node_modules_cache node_modules - else - bun install - fi + uses: ./.github/actions/restore-deps - name: Emit run manifest (ALL periodic tests minus reasoned excludes) env: @@ -144,71 +139,25 @@ jobs: persist-credentials: false - name: Fix bun temp - run: | - mkdir -p /home/runner/.cache/bun - { - echo "BUN_INSTALL_CACHE_DIR=/home/runner/.cache/bun" - echo "BUN_TMPDIR=/home/runner/.cache/bun" - echo "TMPDIR=/home/runner/.cache" - } >> "$GITHUB_ENV" + uses: ./.github/actions/fix-bun-temp - name: Restore deps - run: | - if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then - cp -r /opt/node_modules_cache node_modules - else - bun install - fi + uses: ./.github/actions/restore-deps - run: bun run build # Any slice can host a PTY test — seed + registration run - # unconditionally (idempotent; mirrors evals.yml's sliced lane). + # unconditionally (idempotent; mirrors evals.yml's sliced lane). The + # register composite carries the fail-fast dangling-symlink/frontmatter + # verification loop — this lane previously LACKED it, so a moved skill + # target surfaced as a silent "Unknown command" + wedged PTY session. - name: Seed claude interactive config - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - run: | - node -e ' - const fs = require("fs"), os = require("os"), path = require("path"); - const p = path.join(os.homedir(), ".claude.json"); - const seed = fs.existsSync(p) ? JSON.parse(fs.readFileSync(p, "utf8")) : {}; - seed.hasCompletedOnboarding = true; - const key = process.env.ANTHROPIC_API_KEY || ""; - if (key) seed.customApiKeyResponses = { approved: [key.slice(-20)], rejected: [] }; - fs.writeFileSync(p, JSON.stringify(seed, null, 2)); - console.log("seeded", p); - ' + uses: ./.github/actions/seed-claude-config + with: + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} - name: Register gstack skills for PTY tests - run: | - set -eu - SKILLS_DIR="$HOME/.claude/skills" - REPO="$GITHUB_WORKSPACE" - mkdir -p "$SKILLS_DIR" - ln -snf "$REPO" "$SKILLS_DIR/gstack" - for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do - rm -rf "${SKILLS_DIR:?}/$s" - mkdir -p "$SKILLS_DIR/$s" - cp "$REPO/$s/SKILL.md" "$SKILLS_DIR/$s/SKILL.md" - cp -R "$REPO/$s/sections" "$SKILLS_DIR/$s/sections" - done - PROJ_SKILLS="$REPO/.claude/skills" - mkdir -p "$PROJ_SKILLS" - for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do - rm -rf "${PROJ_SKILLS:?}/$s" - mkdir -p "$PROJ_SKILLS/$s" - cp "$REPO/$s/SKILL.md" "$PROJ_SKILLS/$s/SKILL.md" - cp -R "$REPO/$s/sections" "$PROJ_SKILLS/$s/sections" - done - mkdir -p "$HOME/.gstack" - touch "$HOME/.gstack/.activated" \ - "$HOME/.gstack/.first-loop-tip-shown" \ - "$HOME/.gstack/.telemetry-prompted" \ - "$HOME/.gstack/.proactive-prompted" \ - "$HOME/.gstack/.completeness-intro-seen" \ - "$HOME/.gstack/.plan-tune-nudge-shown" - touch "$SKILLS_DIR/gstack/.feature-prompted-continuous-checkpoint" \ - "$SKILLS_DIR/gstack/.feature-prompted-model-overlay" + uses: ./.github/actions/register-gstack-skills - uses: actions/download-artifact@v8 with: @@ -277,21 +226,10 @@ jobs: persist-credentials: false - name: Fix bun temp - run: | - mkdir -p /home/runner/.cache/bun - { - echo "BUN_INSTALL_CACHE_DIR=/home/runner/.cache/bun" - echo "BUN_TMPDIR=/home/runner/.cache/bun" - echo "TMPDIR=/home/runner/.cache" - } >> "$GITHUB_ENV" + uses: ./.github/actions/fix-bun-temp - name: Restore deps - run: | - if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then - cp -r /opt/node_modules_cache node_modules - else - bun install - fi + uses: ./.github/actions/restore-deps - run: bun run build diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index 897e90ca4..c299de6bc 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -524,12 +524,7 @@ jobs: persist-credentials: false - name: Restore deps - run: | - if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then - cp -r /opt/node_modules_cache node_modules - else - bun install - fi + uses: ./.github/actions/restore-deps - name: Emit run manifest env: @@ -577,72 +572,26 @@ jobs: persist-credentials: false - name: Fix bun temp - run: | - mkdir -p /home/runner/.cache/bun - { - echo "BUN_INSTALL_CACHE_DIR=/home/runner/.cache/bun" - echo "BUN_TMPDIR=/home/runner/.cache/bun" - echo "TMPDIR=/home/runner/.cache" - } >> "$GITHUB_ENV" + uses: ./.github/actions/fix-bun-temp - name: Restore deps - run: | - if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then - cp -r /opt/node_modules_cache node_modules - else - bun install - fi + uses: ./.github/actions/restore-deps - run: bun run build # Any slice can host a PTY smoke, so the seed/registration steps run # UNCONDITIONALLY (both are idempotent) — the old matrix keyed them on - # matrix.suite.name, which a sliced lane cannot do. + # matrix.suite.name, which a sliced lane cannot do. The register + # composite carries the fail-fast dangling-symlink/frontmatter + # verification loop, so a moved skill target fails HERE in seconds, + # not as a wedged PTY session at the shard wall. - name: Seed claude interactive config - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - run: | - node -e ' - const fs = require("fs"), os = require("os"), path = require("path"); - const p = path.join(os.homedir(), ".claude.json"); - const seed = fs.existsSync(p) ? JSON.parse(fs.readFileSync(p, "utf8")) : {}; - seed.hasCompletedOnboarding = true; - const key = process.env.ANTHROPIC_API_KEY || ""; - if (key) seed.customApiKeyResponses = { approved: [key.slice(-20)], rejected: [] }; - fs.writeFileSync(p, JSON.stringify(seed, null, 2)); - console.log("seeded", p); - ' + uses: ./.github/actions/seed-claude-config + with: + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} - name: Register gstack skills for PTY smokes - run: | - set -eu - SKILLS_DIR="$HOME/.claude/skills" - REPO="$GITHUB_WORKSPACE" - mkdir -p "$SKILLS_DIR" - ln -snf "$REPO" "$SKILLS_DIR/gstack" - for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do - rm -rf "${SKILLS_DIR:?}/$s" - mkdir -p "$SKILLS_DIR/$s" - cp "$REPO/$s/SKILL.md" "$SKILLS_DIR/$s/SKILL.md" - cp -R "$REPO/$s/sections" "$SKILLS_DIR/$s/sections" - done - PROJ_SKILLS="$REPO/.claude/skills" - mkdir -p "$PROJ_SKILLS" - for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do - rm -rf "${PROJ_SKILLS:?}/$s" - mkdir -p "$PROJ_SKILLS/$s" - cp "$REPO/$s/SKILL.md" "$PROJ_SKILLS/$s/SKILL.md" - cp -R "$REPO/$s/sections" "$PROJ_SKILLS/$s/sections" - done - mkdir -p "$HOME/.gstack" - touch "$HOME/.gstack/.activated" \ - "$HOME/.gstack/.first-loop-tip-shown" \ - "$HOME/.gstack/.telemetry-prompted" \ - "$HOME/.gstack/.proactive-prompted" \ - "$HOME/.gstack/.completeness-intro-seen" \ - "$HOME/.gstack/.plan-tune-nudge-shown" - touch "$SKILLS_DIR/gstack/.feature-prompted-continuous-checkpoint" \ - "$SKILLS_DIR/gstack/.feature-prompted-model-overlay" + uses: ./.github/actions/register-gstack-skills - uses: actions/download-artifact@v8 with: