mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-10 18:50:18 +01:00
Merge oct7-guard: /autoplan guard on Claude Code 2.1.292, bounded journal read, schema canary (A1, A2)
# Conflicts: # test/helpers/carve-guards.ts
This commit is contained in:
commit
a349219882
61 files changed
+3448
-997
No files matched your search
@@ -128,14 +128,15 @@ RUN case "${TARGETARCH:-amd64}" in \
|
||||
# skillify HOME discovery on 2.1.237, guard/freeze hooks on 2.1.162).
|
||||
# Bump deliberately, via a PR that runs the PTY gate against the new TUI.
|
||||
# test/ci-image-cli-pin.test.ts fails the free suite if this pin is removed.
|
||||
# 2.1.284 is the first pin that recognizes the eval model claude-fable-5-1.
|
||||
# 2.1.251 already sent it effort "high", but ran it as an unknown model with
|
||||
# a generic system prompt. Users on stable (2.1.280) and latest (2.1.285) get
|
||||
# the fable-5-1 profile: its own system prompt, 64k max_tokens and per-turn
|
||||
# effort, which only moves the same effort value into the conversation.
|
||||
# Census 36626737820 on 2.1.284 looked slower mostly because the API was:
|
||||
# its SDK-only judge evals, which never start this CLI, were 25% slower too.
|
||||
RUN npm i -g @anthropic-ai/claude-code@2.1.284
|
||||
# 2.1.292 is the npm latest users run (October 2026). On it the fork-subagent
|
||||
# gate is on by default in interactive sessions: the Agent schema drops
|
||||
# run_in_background and reviewers run in the background, which broke the
|
||||
# /autoplan publication guard while CI still ran 2.1.284 and never saw it.
|
||||
# Pinning the version users run is the point; the daily
|
||||
# autoplan-schema-canary case watches the latest release ahead of the pin.
|
||||
# (2.1.284 was the first pin to recognize the eval model claude-fable-5-1;
|
||||
# 2.1.292 keeps that profile.)
|
||||
RUN npm i -g @anthropic-ai/claude-code@2.1.292
|
||||
|
||||
# Codex CLI — exact pin, same discipline as the claude pin above, so the
|
||||
# Codex eval files execute in CI instead of self-skipping. Installed OFF PATH:
|
||||
|
||||
@@ -20,6 +20,9 @@ run-name: ${{ inputs.nonce && format('Periodic Evals [eval-bg {0}]', inputs.nonc
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 6 * * 1' # Monday 6 AM UTC; build-image below builds or reuses the CI image
|
||||
# Daily, for autoplan-schema-canary alone (latest Claude Code vs the /autoplan
|
||||
# guard's payload assumptions); every census job below skips this schedule.
|
||||
- cron: '30 7 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
redispatch_of:
|
||||
@@ -38,13 +41,17 @@ on:
|
||||
description: 'eval-bg: dispatch nonce carried in the run name'
|
||||
type: string
|
||||
default: ''
|
||||
schema_canary_only:
|
||||
description: 'Run only autoplan-schema-canary (the daily latest-Claude-Code canary), not the census'
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
# One group per ref AND event: a branch dispatch never cancels main's census,
|
||||
# and a manual main dispatch never cancels the scheduled one. A re-dispatch
|
||||
# runs in its own group so it never cancels the run that dispatched it; both
|
||||
# runs are reported.
|
||||
concurrency:
|
||||
group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}
|
||||
group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}${{ (github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary' || '' }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
@@ -110,6 +117,9 @@ jobs:
|
||||
plan-slices:
|
||||
runs-on: ubicloud-standard-8
|
||||
timeout-minutes: 10
|
||||
# The daily schedule and a canary-only dispatch run schema-canary alone;
|
||||
# every census job needs this one, so they all skip with it.
|
||||
if: ${{ github.event.schedule != '30 7 * * *' && !inputs.schema_canary_only }}
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
@@ -176,6 +186,54 @@ jobs:
|
||||
path: /tmp/gate-census-plan/manifest.json
|
||||
retention-days: 30
|
||||
|
||||
# autoplan-schema-canary: installs the latest published Claude Code with
|
||||
# install scripts off, audits registry signatures, and checks the Agent and
|
||||
# Read hook payloads against the journal with the /autoplan guard's own
|
||||
# comparison. Daily, so a schema strip reds within a day of a release; the
|
||||
# weekly census also runs it. Read-only token; the key's spend is capped by
|
||||
# the case (one haiku session, about $0.02).
|
||||
schema-canary:
|
||||
runs-on: ubicloud-standard-2
|
||||
needs: build-image
|
||||
if: ${{ github.event.schedule == '30 7 * * *' || inputs.schema_canary_only }}
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
contents: read
|
||||
packages: read
|
||||
container:
|
||||
image: ${{ needs.build-image.outputs.image-tag }}
|
||||
credentials:
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
options: --user runner
|
||||
env:
|
||||
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-schema-canary
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Fix bun temp
|
||||
uses: ./.github/actions/fix-bun-temp
|
||||
|
||||
- name: Restore deps
|
||||
uses: ./.github/actions/restore-deps
|
||||
|
||||
- name: Run autoplan-schema-canary on the latest Claude Code
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
GSTACK_EVAL_DIR: /tmp/schema-canary-results
|
||||
run: EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --case autoplan-schema-canary --trials 1
|
||||
|
||||
- name: Upload canary results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: schema-canary-a${{ github.run_attempt }}
|
||||
path: /tmp/schema-canary-results
|
||||
if-no-files-found: ignore
|
||||
retention-days: 90
|
||||
|
||||
eval-slices:
|
||||
runs-on: ubicloud-standard-8
|
||||
needs: [build-image, plan-slices]
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
name: Measure journal read
|
||||
|
||||
# CEO-2 / CEO-6: one read plus evaluation of the /autoplan publication guard on
|
||||
# synthetic 120 MiB parent journals, on Linux and macOS, while the journal keeps
|
||||
# growing. Dispatched by hand on the A2 branch; never on every PR. The Linux
|
||||
# numbers are the acceptance record (<= 1000 ms, <= 256 MiB peak RSS); the
|
||||
# macOS numbers are recorded and do not gate. Results go to the job summary.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: measure-journal-read-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
measure:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.4.2
|
||||
- name: Install dependencies
|
||||
run: bun install --frozen-lockfile
|
||||
- name: Measure one guarded read (3 runs per journal)
|
||||
run: |
|
||||
bun scripts/measure-journal-read.ts --runs 3 | tee "$RUNNER_TEMP/measure.md"
|
||||
cat "$RUNNER_TEMP/measure.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
Reference in new issue
Block a user