From 96b5b03c5896b2ac58dec013a9d9b1fa2969cec5 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 18:43:22 +0000 Subject: [PATCH] ci-audit: fix free-lane failures (fence placeholder, runtime-root template, host metadata, paid registrations) - Step 1 fence no longer carries a placeholder; prose names the flags. - Saturday template moves to lib/ci-audit/templates/ so codex and factory runtime roots (which link lib/ whole) carry it. - ci-audit-planted is adapter-only on the Codex host (final output + git state). - Paid file count 114; HARNESS_VERSION records test-pr-profile.ts as non-behavioral. - paid-test-durations.json seeds the two new case keys from the local paid run 2026-10-31-1818 shard walls (max of 3 trials: 38.1 s skill arm, 92.7 s no-gstack arm), a packing hint until the CI refresh rewrites it. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- changelog.d/garrytan-gsta-204-ci-audit.md | 4 ++-- ci-audit/SKILL.md | 5 +++-- ci-audit/SKILL.md.tmpl | 5 +++-- docs/ci-audit.md | 2 +- {ci-audit => lib/ci-audit}/templates/ci-audit-weekly.yml | 0 scripts/harness-version.json | 2 +- scripts/paid-test-durations.json | 6 ++++-- test/ci-audit.test.ts | 2 +- test/codex-host.test.ts | 4 ++-- test/eng-finding-retry-budget.test.ts | 2 +- test/helpers/e2e-hosts.ts | 1 + 11 files changed, 19 insertions(+), 14 deletions(-) rename {ci-audit => lib/ci-audit}/templates/ci-audit-weekly.yml (100%) diff --git a/changelog.d/garrytan-gsta-204-ci-audit.md b/changelog.d/garrytan-gsta-204-ci-audit.md index f68e983ab..3a83b4548 100644 --- a/changelog.d/garrytan-gsta-204-ci-audit.md +++ b/changelog.d/garrytan-gsta-204-ci-audit.md @@ -5,9 +5,9 @@ bump: minor ### Added - **`bin/gstack-ci-audit run`** (lib/ci-audit/): push to all checks green over all-green, first-attempt PR heads (cancelled, superseded, rerun, unsettled and window-edge heads are left out and counted), time to first failure, per-job waiting/setup/running with n, and cost from billed minutes scaled by per-workflow run counts (GitHub's runs list stops at 1,000 runs; the report says when the timing window is shorter). Ubicloud is priced by the VM family read from sampled logs, and cache hit rate is claimed only from restore lines in logs (`--logs N`, at most 20). Exit codes 0 complete, 1 usage, 3 unavailable, 4 partial. Errors are one `CI_AUDIT: unavailable ()` line; ten `ci_audit_*` codes in `lib/gate-outcomes.ts` with docs/troubleshooting.md sections. - **`gstack-ci-audit ship-check`**: a few-second, report-only check for /ship. It checks CI-shaping files in the diff against the lessons, and with `--pr` compares the PR's previous settled push with up to 40 comparable all-green PR heads. "Slower" means above their p95 and at least `ci_audit_slower_min_s` (default 120) over their median; under 8 comparable heads it prints `unknown thin baseline`. About 6 GitHub calls, 10 s deadline. `gstack-config set ci_audit_ship off` turns it off. (The /ship step that calls it lands separately.) -- **Saturday audit**: `ci-audit/templates/ci-audit-weekly.yml` runs the report every Saturday at 15:00 UTC and `gstack-ci-audit publish` keeps one "Weekly CI audit" issue current. Only a complete audit replaces the issue or closes it; a partial or failed one comments and keeps the last findings. +- **Saturday audit**: `lib/ci-audit/templates/ci-audit-weekly.yml` runs the report every Saturday at 15:00 UTC and `gstack-ci-audit publish` keeps one "Weekly CI audit" issue current. Only a complete audit replaces the issue or closes it; a partial or failed one comments and keeps the last findings. - **`--record` / `--fixture`** (or `GSTACK_CI_AUDIT_FIXTURE`) save and replay GitHub responses with allowlisted fields only, plus the recording's clock; `test/fixtures/ci-audit/gstack` is a recorded gstack window. -- **Paid case `ci-audit-planted`** (gate, behavior) runs /ci-audit on a synthetic repo with three planted problems and two decoys; `ci-audit-planted-no-gstack` (periodic) gives the same task and data with no skill text. +- **Paid case `ci-audit-planted`** (gate, behavior) runs /ci-audit on a synthetic repo with three planted problems and two decoys; `ci-audit-planted-no-gstack` (marathon lane) gives the same task and data with no skill text. - **docs/ci-audit.md**, catalog rows, and the doctor's `ci-audit` row. ### Changed diff --git a/ci-audit/SKILL.md b/ci-audit/SKILL.md index 256522c12..fc5ce81d2 100644 --- a/ci-audit/SKILL.md +++ b/ci-audit/SKILL.md @@ -126,10 +126,11 @@ GSTACK_STATE_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get GSTACK_STATE_ GSTACK_STATE_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get GSTACK_STATE_ROOT); : "${GSTACK_STATE_ROOT:?gstack-paths failed; reinstall with ./setup or /gstack-upgrade}" SLUG=$(~/.claude/skills/gstack/bin/gstack-slug --get SLUG 2>/dev/null) && mkdir -p "$GSTACK_STATE_ROOT/projects/$SLUG" && echo "PROJECT_DIR: $GSTACK_STATE_ROOT/projects/$SLUG" OUT="$GSTACK_STATE_ROOT"/projects/$SLUG/ci-audit-$(date +%Y%m%d-%H%M%S) -~/.claude/skills/gstack/bin/gstack-ci-audit run --logs 10 --out "$OUT.json" > "$OUT.md" +~/.claude/skills/gstack/bin/gstack-ci-audit run --logs 10 --out "$OUT.json" > "$OUT.md" echo "EXIT: $?"; echo "REPORT: $OUT.md" ``` +Add the user's `--repo`, `--since-days` or `--workflows` flags to the run line. Exit 0 is complete. Exit 4 is partial: the report says what was cut; label every number from it partial. Exit 3 printed `CI_AUDIT: unavailable … ()` and a `fix:` line: relay both and stop. Exit 1 is a usage error: fix the flags once. @@ -176,7 +177,7 @@ It prints before and after for push to green, per-job phases and dollars. ## Saturday audit When the user wants this every week, offer (A/B) to copy -`~/.claude/skills/gstack/ci-audit/templates/ci-audit-weekly.yml` to +`~/.claude/skills/gstack/lib/ci-audit/templates/ci-audit-weekly.yml` to `.github/workflows/ci-audit-weekly.yml`. It runs the same report every Saturday morning, report-only, and keeps one "Weekly CI audit" issue current: a complete run replaces or closes it, and a partial or failed run leaves the last findings and says diff --git a/ci-audit/SKILL.md.tmpl b/ci-audit/SKILL.md.tmpl index f92eb7adf..6c2c44357 100644 --- a/ci-audit/SKILL.md.tmpl +++ b/ci-audit/SKILL.md.tmpl @@ -49,10 +49,11 @@ GitHub Actions only. GSTACK_STATE_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get GSTACK_STATE_ROOT); : "${GSTACK_STATE_ROOT:?gstack-paths failed; reinstall with ./setup or /gstack-upgrade}" {{SLUG_SETUP}} OUT="$GSTACK_STATE_ROOT"/projects/$SLUG/ci-audit-$(date +%Y%m%d-%H%M%S) -~/.claude/skills/gstack/bin/gstack-ci-audit run --logs 10 --out "$OUT.json" > "$OUT.md" +~/.claude/skills/gstack/bin/gstack-ci-audit run --logs 10 --out "$OUT.json" > "$OUT.md" echo "EXIT: $?"; echo "REPORT: $OUT.md" ``` +Add the user's `--repo`, `--since-days` or `--workflows` flags to the run line. Exit 0 is complete. Exit 4 is partial: the report says what was cut; label every number from it partial. Exit 3 printed `CI_AUDIT: unavailable … ()` and a `fix:` line: relay both and stop. Exit 1 is a usage error: fix the flags once. @@ -99,7 +100,7 @@ It prints before and after for push to green, per-job phases and dollars. ## Saturday audit When the user wants this every week, offer (A/B) to copy -`~/.claude/skills/gstack/ci-audit/templates/ci-audit-weekly.yml` to +`~/.claude/skills/gstack/lib/ci-audit/templates/ci-audit-weekly.yml` to `.github/workflows/ci-audit-weekly.yml`. It runs the same report every Saturday morning, report-only, and keeps one "Weekly CI audit" issue current: a complete run replaces or closes it, and a partial or failed run leaves the last findings and says diff --git a/docs/ci-audit.md b/docs/ci-audit.md index e15ad5848..16c38f31e 100644 --- a/docs/ci-audit.md +++ b/docs/ci-audit.md @@ -79,7 +79,7 @@ Turn it off with `gstack-config set ci_audit_ship off`. ## The Saturday audit -`ci-audit/templates/ci-audit-weekly.yml` runs the full report every Saturday at 15:00 UTC +`lib/ci-audit/templates/ci-audit-weekly.yml` runs the full report every Saturday at 15:00 UTC and keeps one open "Weekly CI audit" issue current. A complete audit creates or replaces the issue, or closes it when nothing is left to fix. A partial or failed audit comments on the issue and keeps the last complete findings. The issue is matched by its title and diff --git a/ci-audit/templates/ci-audit-weekly.yml b/lib/ci-audit/templates/ci-audit-weekly.yml similarity index 100% rename from ci-audit/templates/ci-audit-weekly.yml rename to lib/ci-audit/templates/ci-audit-weekly.yml diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 420cf7e06..0cd37e66e 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -17,7 +17,7 @@ "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"}, "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, - "scripts/test-pr-profile.ts": {"blob":"911e393d0c36c050c3a68db5a81ef21701bfd5ae","since":6,"nonBehavioral":"selection only: fragment exemption narrowed to .md files directly in the fragments dir; changed-file lists include a rename's old path; no case prompt, fixture, judge or verdict logic changes (GST-400)"}, + "scripts/test-pr-profile.ts": {"blob":"a43311e4e16b76ea68bf1cba7a59b2a9ba9c026b","since":6,"nonBehavioral":"scripts/test-pr-profile.ts adds the new ci-audit-planted case to the PR profile list and file map; no existing case's selection or verdict changes (GSTA-204)"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6}, diff --git a/scripts/paid-test-durations.json b/scripts/paid-test-durations.json index f7d656e8b..fa0efad18 100644 --- a/scripts/paid-test-durations.json +++ b/scripts/paid-test-durations.json @@ -105,7 +105,8 @@ "test/skill-e2e-triage.test.ts": 64816, "test/skill-e2e-workflow.test.ts": 265175, "test/skill-llm-eval.test.ts": 354000, - "test/skill-routing-e2e.test.ts": 1000 + "test/skill-routing-e2e.test.ts": 1000, + "test/skill-e2e-ci-audit.test.ts#ci-audit-planted": 38100 }, "periodic": { "test/carve-section-loading.test.ts#carve-section-loading-browse": 92911, @@ -216,7 +217,8 @@ "test/skill-e2e-triage.test.ts": 1000, "test/skill-e2e-workflow.test.ts": 121594, "test/skill-llm-eval.test.ts": 339704, - "test/skill-routing-e2e.test.ts": 81360 + "test/skill-routing-e2e.test.ts": 81360, + "test/skill-e2e-ci-audit.test.ts#ci-audit-planted-no-gstack": 92730 } } } diff --git a/test/ci-audit.test.ts b/test/ci-audit.test.ts index 0ebd7f016..296e4df07 100644 --- a/test/ci-audit.test.ts +++ b/test/ci-audit.test.ts @@ -266,7 +266,7 @@ describe('weekly issue', () => { }); describe('Saturday workflow template', () => { - const file = path.join(ROOT, 'ci-audit/templates/ci-audit-weekly.yml'); + const file = path.join(ROOT, 'lib/ci-audit/templates/ci-audit-weekly.yml'); const doc = Bun.YAML.parse(fs.readFileSync(file, 'utf8')) as any; test('runs Saturday morning and on demand, reads Actions, writes only issues', () => { diff --git a/test/codex-host.test.ts b/test/codex-host.test.ts index e41e55ef3..c2f3ff425 100644 --- a/test/codex-host.test.ts +++ b/test/codex-host.test.ts @@ -128,10 +128,10 @@ describe('per-case host metadata', () => { for (const id of Object.keys(E2E_HOSTS)) expect(Object.hasOwn(E2E_TOUCHFILES, id), id).toBe(true); }); - test('classifies 52 adapter-only, 34 Claude-specific, 26 rework and 1 host-neutral case', () => { + test('classifies 53 adapter-only, 34 Claude-specific, 26 rework and 1 host-neutral case', () => { const counts: Record = {}; for (const support of Object.values(E2E_HOSTS)) counts[support] = (counts[support] ?? 0) + 1; - expect(counts.both).toBe(52); + expect(counts.both).toBe(53); expect(counts.rework).toBe(26); expect(counts['host-neutral']).toBe(1); expect(['pty', 'sdk', 'subagents', 'skill-tool', 'hermetic', 'self-host'].reduce((sum, reason) => sum + (counts[reason] ?? 0), 0)).toBe(34); diff --git a/test/eng-finding-retry-budget.test.ts b/test/eng-finding-retry-budget.test.ts index c390233f7..0e294683d 100644 --- a/test/eng-finding-retry-budget.test.ts +++ b/test/eng-finding-retry-budget.test.ts @@ -141,7 +141,7 @@ test('live periodic census fits the declared CI wall including setup', () => { test('registered allocation is deterministic and preserves every discovered file', () => { const files = collectPaidTestFiles(); - expect(files).toHaveLength(113); // + skill-e2e-model-panel (GSTA-173); + skill-e2e-write-skill (GSTA-133); + skill-e2e-ship-frozen-scope (frozen review scope, GSTA-113 PR 2); + skill-e2e-failure-routing (#3157/#3158); + skill-e2e-dx-error-contract (#3226); + skill-e2e-gate-integrity (HONEST_WORK_GATE_INTEGRITY W8); + skill-e2e-autoplan-schema-canary, -guard-pty and -long-session (Oct 7 wave, CEO-13, ENG-18, CEO-15); + skill-e2e-ship-measure-seeded-flake (v1.91.32.0); W5a: 15 carve wrappers became one case-sharded file; + codex-e2e-multiblock-live and skill-e2e-design-model-smoke (v1.91.30.0); + expect(files).toHaveLength(114); // + skill-e2e-ci-audit (GSTA-204); + skill-e2e-model-panel (GSTA-173); + skill-e2e-write-skill (GSTA-133); + skill-e2e-ship-frozen-scope (frozen review scope, GSTA-113 PR 2); + skill-e2e-failure-routing (#3157/#3158); + skill-e2e-dx-error-contract (#3226); + skill-e2e-gate-integrity (HONEST_WORK_GATE_INTEGRITY W8); + skill-e2e-autoplan-schema-canary, -guard-pty and -long-session (Oct 7 wave, CEO-13, ENG-18, CEO-15); + skill-e2e-ship-measure-seeded-flake (v1.91.32.0); W5a: 15 carve wrappers became one case-sharded file; + codex-e2e-multiblock-live and skill-e2e-design-model-smoke (v1.91.30.0); expect(files).toContain('test/skill-e2e-ship-skip.test.ts'); const m = livePlan(files); expect(livePlan([...files].reverse())).toEqual(m); diff --git a/test/helpers/e2e-hosts.ts b/test/helpers/e2e-hosts.ts index ed2997e46..2afd11b35 100644 --- a/test/helpers/e2e-hosts.ts +++ b/test/helpers/e2e-hosts.ts @@ -40,6 +40,7 @@ const ADAPTER_ONLY: readonly string[] = [ 'browse-basic', 'browse-snapshot', 'canary-workflow', + 'ci-audit-planted', 'context-recovery-artifacts', 'context-restore-loads-latest', 'context-save-writes-file',