Merge oct7-guard: /autoplan guard on Claude Code 2.1.292, bounded journal read, schema canary (A1, A2)

# Conflicts:
#	test/helpers/carve-guards.ts
This commit is contained in:
garrytan committed 2026-10-07 21:34:36 +00:00
commit a349219882
61 files changed
+3448 -997

No files matched your search

+9 -8
View File
@@ -128,14 +128,15 @@ RUN case "${TARGETARCH:-amd64}" in \
# skillify HOME discovery on 2.1.237, guard/freeze hooks on 2.1.162).
# Bump deliberately, via a PR that runs the PTY gate against the new TUI.
# test/ci-image-cli-pin.test.ts fails the free suite if this pin is removed.
# 2.1.284 is the first pin that recognizes the eval model claude-fable-5-1.
# 2.1.251 already sent it effort "high", but ran it as an unknown model with
# a generic system prompt. Users on stable (2.1.280) and latest (2.1.285) get
# the fable-5-1 profile: its own system prompt, 64k max_tokens and per-turn
# effort, which only moves the same effort value into the conversation.
# Census 36626737820 on 2.1.284 looked slower mostly because the API was:
# its SDK-only judge evals, which never start this CLI, were 25% slower too.
RUN npm i -g @anthropic-ai/claude-code@2.1.284
# 2.1.292 is the npm latest users run (October 2026). On it the fork-subagent
# gate is on by default in interactive sessions: the Agent schema drops
# run_in_background and reviewers run in the background, which broke the
# /autoplan publication guard while CI still ran 2.1.284 and never saw it.
# Pinning the version users run is the point; the daily
# autoplan-schema-canary case watches the latest release ahead of the pin.
# (2.1.284 was the first pin to recognize the eval model claude-fable-5-1;
# 2.1.292 keeps that profile.)
RUN npm i -g @anthropic-ai/claude-code@2.1.292
# Codex CLI — exact pin, same discipline as the claude pin above, so the
# Codex eval files execute in CI instead of self-skipping. Installed OFF PATH:
+59 -1
View File
@@ -20,6 +20,9 @@ run-name: ${{ inputs.nonce && format('Periodic Evals [eval-bg {0}]', inputs.nonc
on:
schedule:
- cron: '0 6 * * 1' # Monday 6 AM UTC; build-image below builds or reuses the CI image
# Daily, for autoplan-schema-canary alone (latest Claude Code vs the /autoplan
# guard's payload assumptions); every census job below skips this schedule.
- cron: '30 7 * * *'
workflow_dispatch:
inputs:
redispatch_of:
@@ -38,13 +41,17 @@ on:
description: 'eval-bg: dispatch nonce carried in the run name'
type: string
default: ''
schema_canary_only:
description: 'Run only autoplan-schema-canary (the daily latest-Claude-Code canary), not the census'
type: boolean
default: false
# One group per ref AND event: a branch dispatch never cancels main's census,
# and a manual main dispatch never cancels the scheduled one. A re-dispatch
# runs in its own group so it never cancels the run that dispatched it; both
# runs are reported.
concurrency:
group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}
group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}${{ (github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary' || '' }}
cancel-in-progress: true
env:
@@ -110,6 +117,9 @@ jobs:
plan-slices:
runs-on: ubicloud-standard-8
timeout-minutes: 10
# The daily schedule and a canary-only dispatch run schema-canary alone;
# every census job needs this one, so they all skip with it.
if: ${{ github.event.schedule != '30 7 * * *' && !inputs.schema_canary_only }}
permissions:
contents: read
outputs:
@@ -176,6 +186,54 @@ jobs:
path: /tmp/gate-census-plan/manifest.json
retention-days: 30
# autoplan-schema-canary: installs the latest published Claude Code with
# install scripts off, audits registry signatures, and checks the Agent and
# Read hook payloads against the journal with the /autoplan guard's own
# comparison. Daily, so a schema strip reds within a day of a release; the
# weekly census also runs it. Read-only token; the key's spend is capped by
# the case (one haiku session, about $0.02).
schema-canary:
runs-on: ubicloud-standard-2
needs: build-image
if: ${{ github.event.schedule == '30 7 * * *' || inputs.schema_canary_only }}
timeout-minutes: 20
permissions:
contents: read
packages: read
container:
image: ${{ needs.build-image.outputs.image-tag }}
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
options: --user runner
env:
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-schema-canary
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Fix bun temp
uses: ./.github/actions/fix-bun-temp
- name: Restore deps
uses: ./.github/actions/restore-deps
- name: Run autoplan-schema-canary on the latest Claude Code
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
GSTACK_EVAL_DIR: /tmp/schema-canary-results
run: EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --case autoplan-schema-canary --trials 1
- name: Upload canary results
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: schema-canary-a${{ github.run_attempt }}
path: /tmp/schema-canary-results
if-no-files-found: ignore
retention-days: 90
eval-slices:
runs-on: ubicloud-standard-8
needs: [build-image, plan-slices]
@@ -0,0 +1,39 @@
name: Measure journal read
# CEO-2 / CEO-6: one read plus evaluation of the /autoplan publication guard on
# synthetic 120 MiB parent journals, on Linux and macOS, while the journal keeps
# growing. Dispatched by hand on the A2 branch; never on every PR. The Linux
# numbers are the acceptance record (<= 1000 ms, <= 256 MiB peak RSS); the
# macOS numbers are recorded and do not gate. Results go to the job summary.
on:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: measure-journal-read-${{ github.ref }}
cancel-in-progress: true
jobs:
measure:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
runs-on: ${{ matrix.os }}
timeout-minutes: 20
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.2
- name: Install dependencies
run: bun install --frozen-lockfile
- name: Measure one guarded read (3 runs per journal)
run: |
bun scripts/measure-journal-read.ts --runs 3 | tee "$RUNNER_TEMP/measure.md"
cat "$RUNNER_TEMP/measure.md" >> "$GITHUB_STEP_SUMMARY"