mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
Merge wave/w-followups
This commit is contained in:
garrytan committed
commit
4ea2c6eed7
16 files changed
+1056
-177
No files matched your search
@@ -283,25 +283,47 @@ jobs:
|
||||
# PIPESTATUS[0], NOT $?: the default step shell has no pipefail.
|
||||
echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# One tracking issue for the whole lane (never one per week).
|
||||
- name: Upsert tracking issue on failure
|
||||
# A red marathon run writes its report to the step summary and the
|
||||
# marathon-report artifact on every ref. Only main upserts the lane's ONE
|
||||
# tracking issue (never one per week) and closes it on the next green
|
||||
# run; a branch-dispatched run never touches main's issue.
|
||||
- name: Write the marathon report
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success')
|
||||
run: |
|
||||
set -euo pipefail
|
||||
{
|
||||
echo "Marathon report (non-blocking lane): branch ${GITHUB_REF_NAME} @ ${GITHUB_SHA}, event ${GITHUB_EVENT_NAME}, run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo
|
||||
echo "A census went red: triage with ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${GITHUB_SHA}/docs/evals/census-red.md"
|
||||
echo
|
||||
echo "- reconciliation exit: ${{ steps.reconcile.outputs.exit }}"
|
||||
echo "- marathon slices job: ${{ needs.eval-slices.result }}"
|
||||
echo
|
||||
{ tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"; } \
|
||||
| bun --no-install -e 'import { publishedFence } from "./scripts/lib/published-text";
|
||||
const lines = (await Bun.stdin.text()).split("\n").filter(Boolean);
|
||||
console.log(publishedFence(lines.length ? lines : ["(none)"]).join("\n"));'
|
||||
} > /tmp/marathon-report.md
|
||||
cat /tmp/marathon-report.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload the marathon report
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success')
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: marathon-report-a${{ github.run_attempt }}
|
||||
path: /tmp/marathon-report.md
|
||||
if-no-files-found: ignore
|
||||
retention-days: 90
|
||||
|
||||
- name: Upsert tracking issue on failure
|
||||
if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success')
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TITLE="Weekly marathon evals: red lane needs triage"
|
||||
BODY_FILE=/tmp/issue-body.md
|
||||
{
|
||||
echo "Automated weekly marathon report (non-blocking lane) — run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo
|
||||
echo "- reconciliation exit: ${{ steps.reconcile.outputs.exit }}"
|
||||
echo "- marathon slices job: ${{ needs.eval-slices.result }}"
|
||||
echo
|
||||
echo '```'
|
||||
tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"
|
||||
echo '```'
|
||||
} > "$BODY_FILE"
|
||||
BODY_FILE=/tmp/marathon-report.md
|
||||
[ -s "$BODY_FILE" ] || echo "Marathon report missing; run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" > "$BODY_FILE"
|
||||
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
|
||||
if [ -n "$EXISTING" ]; then
|
||||
gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE"
|
||||
@@ -310,6 +332,18 @@ jobs:
|
||||
gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body-file "$BODY_FILE"
|
||||
fi
|
||||
|
||||
- name: Close the tracking issue on a green run
|
||||
if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && needs.eval-slices.result == 'success'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TITLE="Weekly marathon evals: red lane needs triage"
|
||||
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
|
||||
if [ -n "$EXISTING" ]; then
|
||||
gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --comment "Green marathon run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
fi
|
||||
|
||||
- name: Fail the workflow when reconciliation failed
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success')
|
||||
run: exit 1
|
||||
@@ -627,7 +627,7 @@ jobs:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
bun run eval:pass-rates --gate --runs 10 > /tmp/pass-rates.txt 2>&1
|
||||
bun run eval:pass-rates --gate --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates.txt 2>&1
|
||||
echo "exit=$?" >> "$GITHUB_OUTPUT"
|
||||
cat /tmp/pass-rates.txt
|
||||
|
||||
@@ -652,10 +652,13 @@ jobs:
|
||||
echo "redispatch=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# A red weekly lane nobody must action is waste — upsert ONE tracking
|
||||
# issue (never a new issue per week) with the headline and failure block
|
||||
# of both lanes, and close it on the next green run.
|
||||
- name: Upsert tracking issue on failure
|
||||
# A red census writes one report on every ref: the step summary and the
|
||||
# census-report artifact. Only main (scheduled weekly, or dispatched on
|
||||
# main) upserts ONE tracking issue (never a new issue per week) and
|
||||
# closes it on the next green run; a branch-dispatched census never
|
||||
# comments on or closes main's issue. Every report-derived line goes
|
||||
# through the one published-text sanitizer (scripts/lib/published-text.ts).
|
||||
- name: Write the census report
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -663,17 +666,28 @@ jobs:
|
||||
REDISPATCH_OF: ${{ inputs.redispatch_of }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TITLE="Weekly periodic evals: red lane needs triage"
|
||||
BODY_FILE=/tmp/issue-body.md
|
||||
published() { # $1 fence|line: stdin lines through the published-text sanitizer
|
||||
bun --no-install -e 'import { publishedFence, sanitizePublishedText } from "./scripts/lib/published-text";
|
||||
const lines = (await Bun.stdin.text()).split("\n").filter(Boolean);
|
||||
console.log(process.argv[1] === "line" ? sanitizePublishedText(lines.join(" ")) : publishedFence(lines.length ? lines : ["(none)"]).join("\n"));' "$1"
|
||||
}
|
||||
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
GUIDE_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${GITHUB_SHA}/docs/evals/census-red.md"
|
||||
bun run eval:pass-rates --reds --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates-reds.txt 2>&1 \
|
||||
|| echo "history unavailable: eval:pass-rates --reds exited $?" >> /tmp/pass-rates-reds.txt
|
||||
bun run eval:pass-rates --headroom --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates-headroom.txt 2>&1 \
|
||||
|| echo "history unavailable: eval:pass-rates --headroom exited $?" >> /tmp/pass-rates-headroom.txt
|
||||
{
|
||||
echo "Automated weekly report — run: ${RUN_URL}"
|
||||
echo "Periodic census report: branch ${GITHUB_REF_NAME} @ ${GITHUB_SHA}, event ${GITHUB_EVENT_NAME}, run ${RUN_URL}"
|
||||
echo
|
||||
echo "A census went red: triage with ${GUIDE_URL}"
|
||||
if [ -n "$REDISPATCH_OF" ]; then echo; echo "This run is the one INFRA/INCOMPLETE re-dispatch of run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${REDISPATCH_OF}; both runs are reported."; fi
|
||||
if [ "$REDISPATCH" = "true" ]; then echo; echo "Every red verdict is machine-classified INFRA/INCOMPLETE: re-dispatching once as a new run (EVAL_POLICY.infraRedispatch). This run stays red and reported."; fi
|
||||
echo
|
||||
echo "- periodic reconciliation exit: ${{ steps.reconcile.outputs.exit }} (slices job: ${{ needs.eval-slices.result }}, Codex slices job: ${{ needs.eval-codex-slices.result }})"
|
||||
echo "- gate census reconciliation exit: ${{ steps.gate-reconcile.outputs.exit }} (census job: ${{ needs.gate-census.result }})"
|
||||
echo "- pass-rate history gate exit: ${{ steps.pass-rates.outputs.exit }}"
|
||||
echo "- $({ grep -E '^all-green probability' /tmp/pass-rates-reds.txt || echo 'all-green probability: unavailable (see the red ledger)'; } | published line)"
|
||||
echo
|
||||
echo "### Periodic lane"
|
||||
cat /tmp/paid-report/report-summary.md 2>/dev/null || echo "(no periodic report summary)"
|
||||
@@ -682,23 +696,44 @@ jobs:
|
||||
cat /tmp/gate-census-report/report-summary.md 2>/dev/null || echo "(no gate census report summary)"
|
||||
echo
|
||||
echo "### Pass-rate history (ACTION REQUIRED)"
|
||||
echo '```'
|
||||
{ grep -E 'ACTION REQUIRED|history unavailable' /tmp/pass-rates.txt || echo "(no pass-rate alarms)"; } | sed 's/@/@\xe2\x80\x8b/g' | head -c 6000
|
||||
echo '```'
|
||||
{ awk '/^ACTION REQUIRED/ { on = 1; print; next } on && /^ \[/ { print; next } { on = 0 } /history unavailable/ { print }' /tmp/pass-rates.txt 2>/dev/null \
|
||||
| grep . || echo "(no pass-rate alarms)"; } | head -c 6000 | published fence
|
||||
echo
|
||||
echo "### Red ledger: bun run eval:pass-rates --reds --runs 10 --branch ${GITHUB_REF_NAME}"
|
||||
{ head -n 1 /tmp/pass-rates-reds.txt; tail -n +2 /tmp/pass-rates-reds.txt | tail -n 60; } | cut -c1-400 | published fence
|
||||
echo
|
||||
echo "### Session headroom: bun run eval:pass-rates --headroom --runs 10 --branch ${GITHUB_REF_NAME}"
|
||||
head -n 40 /tmp/pass-rates-headroom.txt | cut -c1-400 | published fence
|
||||
echo
|
||||
echo "<details><summary>Full reconciliation output</summary>"
|
||||
echo
|
||||
echo '```'
|
||||
tail -c 6000 /tmp/report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no reconciliation output)"
|
||||
echo '```'
|
||||
{ tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"; } | published fence
|
||||
echo
|
||||
echo '```'
|
||||
tail -c 6000 /tmp/gate-report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no gate census reconciliation output)"
|
||||
echo '```'
|
||||
{ tail -c 6000 /tmp/gate-report.txt 2>/dev/null || echo "(no gate census reconciliation output)"; } | published fence
|
||||
echo "</details>"
|
||||
echo
|
||||
echo "Policy: EVAL_POLICY and CASE_QUARANTINE in test/helpers/periodic-exclude-data.ts; history: \`bun run eval:pass-rates\`."
|
||||
} > "$BODY_FILE"
|
||||
} > /tmp/census-report.md
|
||||
cat /tmp/census-report.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload the census report
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: census-report-a${{ github.run_attempt }}
|
||||
path: /tmp/census-report.md
|
||||
if-no-files-found: ignore
|
||||
retention-days: 90
|
||||
|
||||
- name: Upsert tracking issue on failure
|
||||
if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TITLE="Weekly periodic evals: red lane needs triage"
|
||||
BODY_FILE=/tmp/census-report.md
|
||||
[ -s "$BODY_FILE" ] || echo "Periodic census report missing; run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" > "$BODY_FILE"
|
||||
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
|
||||
if [ -n "$EXISTING" ]; then
|
||||
gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE"
|
||||
@@ -708,7 +743,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Close the tracking issue on a green run
|
||||
if: always() && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success'
|
||||
if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REDISPATCH_OF: ${{ inputs.redispatch_of }}
|
||||
|
||||
@@ -274,6 +274,13 @@ this policy. Quarantine (`CASE_QUARANTINE`) and history are described in
|
||||
`docs/TESTING_INTERNALS.md`; `bun run eval:pass-rates --case <id>` shows a
|
||||
case's per-trial pass rate with its Wilson interval.
|
||||
|
||||
**A census went red?** Follow [docs/evals/census-red.md](docs/evals/census-red.md):
|
||||
read the red line, inspect its evidence with `bun run eval:pass-rates --run <id>`,
|
||||
check `--reds` and `--headroom`, repair with a free regression test, and only
|
||||
then spend one paid run with the line's `after a repair:` command. A red census
|
||||
on `main` is reported on the weekly tracking issue; a census dispatched on a
|
||||
branch writes the same report to its run summary and `census-report` artifact.
|
||||
|
||||
CI enables verified first-attempt reuse for 16 workflow quality judges for
|
||||
24 hours within the same PR. The cookie workflow's custom input and the other 11
|
||||
quality cases stay fresh. PR-profile E2E shards that run once (no retry, so the
|
||||
@@ -446,6 +453,9 @@ bun run eval:list # list all eval runs (turns, duration, cost per run
|
||||
bun run eval:compare # compare two runs — shows per-test deltas + Takeaway commentary
|
||||
bun run eval:summary # aggregate stats + per-test efficiency averages across runs
|
||||
bun run eval:pass-rates # per-case trial pass rates + Wilson intervals from recent weekly runs (--case, --runs, --dir, --backfill, --json, --gate); eval:flake-rank is an alias
|
||||
bun run eval:pass-rates --reds # verdict reds per census by failure class and cause, all-green probability
|
||||
bun run eval:pass-rates --headroom # slowest session per case vs its armed budget (alarm above 85% in --gate)
|
||||
bun run eval:pass-rates --run <id> # one census's reds with their values and fetched transcript evidence
|
||||
```
|
||||
|
||||
**Detached runs for agents and long suites.** When an agent (or you, for a run
|
||||
|
||||
@@ -186,8 +186,12 @@ for capture, judge, and benchmark model selection.
|
||||
The design binary (`$D`) uses OpenAI's `gpt-5.5` by default, with image
|
||||
generation through `gpt-image-2`. Set `GSTACK_DESIGN_MODEL=<model>` to use
|
||||
another gpt-5-class model for both image generation and screenshot analysis;
|
||||
if OpenAI rejects it, the error names `GSTACK_DESIGN_MODEL`. Check a key
|
||||
against the defaults with `bun run design/scripts/live-model-check.ts`.
|
||||
if OpenAI rejects it, the error names `GSTACK_DESIGN_MODEL`. Set
|
||||
`GSTACK_DESIGN_IMAGE_MODEL=<gpt-image model>` to change the image tool model; a
|
||||
value that is not a gpt-image model name is refused before any request. Check a
|
||||
key against the defaults with `bun run design/scripts/live-model-check.ts`,
|
||||
which always tests the default models and ignores both overrides; the weekly
|
||||
periodic census runs the same check.
|
||||
|
||||
**Want to add support for another agent?** See [docs/ADDING_A_HOST.md](docs/ADDING_A_HOST.md).
|
||||
Rendering a new agent is one TypeScript config file; installing it also needs a
|
||||
|
||||
+107
-8
@@ -96,9 +96,13 @@ against the previous finalized run (in-flight `_partial` files are never used as
|
||||
a baseline, so a run can't compare against itself).
|
||||
|
||||
The periodic overlay fixtures use a versioned behavior gate with efficacy
|
||||
reported separately. See [Overlay benchmark contract v2](OVERLAY_BENCHMARK_CONTRACT.md)
|
||||
reported separately. See [Overlay benchmark contract v4](OVERLAY_BENCHMARK_CONTRACT.md)
|
||||
for exact correctness requirements, retired fanout cases, immutable evidence,
|
||||
and the limits of a passing result.
|
||||
and the limits of a passing result. Under v4 a completed wrong answer is a
|
||||
measured correctness result in either arm (`taskCorrect: false` with
|
||||
`answerError`); only an overlay-ON wrong answer fails the case, so an
|
||||
overlay-OFF wrong answer stays in the comparison. v3 verdicts keep their
|
||||
original meaning.
|
||||
|
||||
## Coverage ownership
|
||||
|
||||
@@ -365,7 +369,22 @@ report and tracking issue, and nothing requires it. evals-periodic.yml runs ALL
|
||||
periodic-tier files weekly (the coverage contract) minus the reasoned
|
||||
exclusions in `test/helpers/periodic-exclude-data.ts` (reason + tracking
|
||||
required per entry; removal re-activates the file), plus a weekly
|
||||
`EVALS_ALL` gate census, plus a tracking-issue UPSERT on red weeks. The CI
|
||||
`EVALS_ALL` gate census. A red census writes one report on every ref, to the
|
||||
step summary and the `census-report-a<attempt>` artifact: the run's branch, sha
|
||||
and event, both lanes' summaries, every pass-rate alarm line, the red ledger
|
||||
and session headroom (`eval:pass-rates --reds` / `--headroom --runs 10 --branch
|
||||
<ref>`), the all-green estimate and a link to the red-census guide, with
|
||||
report-derived text passed through `scripts/lib/published-text.ts`. Only a run
|
||||
on `refs/heads/main` comments on (or opens) the one tracking issue and closes it
|
||||
on the next green run; the marathon lane does the same with its own issue and
|
||||
`marathon-report-a<attempt>` artifact. `test/evals-tracking-issue.test.ts`
|
||||
pins the main-only guard and runs both report scripts. The host-run Codex job
|
||||
also carries the live Codex multi-block case (`codex-multiblock-live`: `./setup
|
||||
--host codex` from the job's checkout into a fresh `CODEX_HOME`, the router in
|
||||
`codex debug prompt-input`, and a later /learn block rooted by its own
|
||||
prelude), and the periodic lane runs the design default-model smoke
|
||||
(`design-model-smoke`), a visible SKIPPED with no coverage credit when
|
||||
`OPENAI_API_KEY` is absent. The CI
|
||||
image pins the claude CLI to an exact version (`.github/docker/Dockerfile.ci`,
|
||||
enforced by `test/ci-image-cli-pin.test.ts` — bumps ride PRs that run the PTY
|
||||
gate), and every eval-store run records `claude --version`, resolved once in
|
||||
@@ -373,6 +392,7 @@ the runner parent and handed to shard children as `GSTACK_CLAUDE_CLI_VERSION`
|
||||
(never spawned on a test thread), so a TUI-drift flake hunt is a grep, not
|
||||
archaeology.
|
||||
|
||||
<a id="eval-verdict-policy"></a>
|
||||
**Eval verdict policy** (`EVAL_POLICY` version 1 in
|
||||
`test/helpers/periodic-exclude-data.ts`, pre-registered 2026-09-29). Paid evals
|
||||
never retry. Each live case has exactly one kind in `E2E_KINDS`
|
||||
@@ -397,6 +417,11 @@ and the kind fixes its trials before the run:
|
||||
refusal counts as an unscored panel only when every sample refused.
|
||||
`callJudge`'s 429 backoff happens before any model output and is transport,
|
||||
not a verdict retry. The workflow-judge cache stores whole panels only.
|
||||
The `rule` case auq-matrix uses the same panel inside its one execution: each
|
||||
skill's single native capture has its recommendation substance scored by 3
|
||||
judge samples, gated on their mean against the unchanged minimum of 4, and a
|
||||
failed sample fails the panel without resampling (approved 2026-10-04 for
|
||||
censuses after the change).
|
||||
|
||||
`panelVerdict()` (`test/helpers/eval-store.ts`) is the single verdict
|
||||
function the report, `collector-outcomes.json`, the PR comment and pass-rates
|
||||
@@ -414,6 +439,58 @@ reported. Changing any `EVAL_POLICY` constant after seeing census results needs
|
||||
Garry's re-approval, a `version` bump and a fresh census;
|
||||
`test/periodic-exclude-policy.test.ts` pins the approved values.
|
||||
|
||||
<a id="failure-causes"></a>
|
||||
**Failure causes and details** (diagnostic only; verdicts unchanged under v1).
|
||||
Every failed trial record keeps `failure_class` (`assertion`, `contract`,
|
||||
`timeout`, `infra`: the field verdicts, INFRA re-dispatch and pass rates read)
|
||||
and its 300-character `error` exactly as before. `failureCauseOf()`
|
||||
(`test/helpers/eval-store.ts`, called by both record builders, including the
|
||||
JUnit census for PTY and JUnit-only cases) adds an optional `failure_cause`
|
||||
with a one-line `failure_cause_evidence`. It never changes `failure_class`.
|
||||
The highest-precedence cause with evidence wins:
|
||||
|
||||
| `failure_cause` | Recorded when | First step |
|
||||
|---|---|---|
|
||||
| `contract` | `expectContract()` stamped a contract violation | fix the product; a violation fails a behavior panel outright |
|
||||
| `pre_turn_infra` | the trial failed before the first model turn (`failure_class: 'infra'`) | check the runner and API status; an all-INFRA census is re-dispatched once |
|
||||
| `api_error` | a session ended on an API error (for PTY, the `API Error:` panel on an idle turn) or the runner's exit reason was `error_api` | read the evidence line; a provider outage is a named red |
|
||||
| `refusal` | the structured stop reason was a refusal (phrases only as a last resort; quoted refusal text in a tool result does not count) | check whether the fixture prompt invites the refusal |
|
||||
| `provider_stall` | a session that streamed partial messages saw no event of any kind, thinking deltas included, for `STALL_WINDOW_MS` (120 s) while a model request was in flight and no tool, permission prompt, hook or subagent was outstanding; sessions without partial messages are never stalls | read the session ledger row; still a failed trial under v1 |
|
||||
| `session_timeout` | a runner's armed session timeout fired | `eval:pass-rates --headroom`; cut work, never raise the budget |
|
||||
| `observer_timeout` | the test's observer gave up waiting for an outcome (`outcome=timeout`) | read the transcript tail: a missed detector or a product that never answered |
|
||||
| `assertion` | a check failed after the session ran (`session completed; check failed` when it ended normally) | the Expected/Received values or judge dimensions |
|
||||
| `unknown` | the trial failed with no evidence for any cause above | inspect the shard directory |
|
||||
|
||||
A detector, harness or product fault is a human diagnosis, never a cause.
|
||||
Readers before this field accept the new records (a frozen v1 reader runs over
|
||||
them in `test/eval-store.test.ts`); an unknown cause from a newer gstack is
|
||||
rejected with `unknown failure_cause "<x>" (written by a newer gstack; update
|
||||
this checkout to read it)`.
|
||||
|
||||
`failure_detail` carries the values `error` cannot hold: a failed Bun matcher's
|
||||
`{expected, received}`, or for a judge one `{dimension, mean, threshold,
|
||||
samples, rationale}` entry per failing dimension, each part capped at 200
|
||||
characters. `sessions[]` (at most 32) summarizes the trial's session-ledger
|
||||
rows. `cost_known: false` marks a record or trial whose harness captured no
|
||||
billing: PTY and Codex sessions never bill, and an SDK session bills only with a
|
||||
terminal `total_cost_usd`. An absent field means known. Census and marathon
|
||||
cost lines print the known sum and `N of M trial(s) cost unknown`, never a bare
|
||||
total over unknowns.
|
||||
|
||||
<a id="session-ledger"></a>
|
||||
**Session ledger.** Every runner (session-runner, the Agent SDK runner
|
||||
including `runSharedInteractive`, the PTY session and `runRecordedCodexEval`)
|
||||
appends one best-effort line per armed session to
|
||||
`$GSTACK_EVAL_DIR/session-ledger.jsonl` (`test/helpers/session-ledger.ts`):
|
||||
a stable key `<runner>:<label>#n`, the exact `budget_ms` it armed, monotonic
|
||||
`elapsed_ms`, the `end` reason, sanitized evidence, `billed`, and a compact
|
||||
liveness summary (longest in-request silence, the last event kind, any tool open
|
||||
at the end). `claude -p` sessions stream partial messages for this summary;
|
||||
transcripts keep their shape and size (full stream diagnostics only with
|
||||
`publicStreamDiagnostics`). A failed ledger write never fails a trial; a trial
|
||||
without ledger rows has no `sessions` and its headroom is `unknown`, never
|
||||
scored. The ledger rides in each slice artifact beside the shard results.
|
||||
|
||||
**Quarantine** (`CASE_QUARANTINE`, same file). An entry needs: a per-trial rate
|
||||
below 95% over at least 10 post-policy trials of the case's current input
|
||||
identity (pre-policy backfill may justify only an initial entry, labeled as
|
||||
@@ -430,8 +507,11 @@ weekly runs old, or when a tier is over its cap.
|
||||
|
||||
**Pass-rate history** (`bun run eval:pass-rates`, `scripts/eval-flake-rank.ts`).
|
||||
It reads the `trial-outcomes` artifact of the last N completed
|
||||
`evals-periodic.yml` runs on the current branch and `main` (flags: `--case`,
|
||||
`--runs N`, `--branch`, `--dir`, `--backfill`, `--json`, `--gate`) and prints
|
||||
`evals-periodic.yml` runs on the given `--branch` (default: the current branch)
|
||||
and `main` (flags: `--case`, `--runs N`, `--branch`, `--repo`, `--workflow`,
|
||||
`--dir`, `--backfill`, `--json`, `--gate`, and the views `--headroom`, `--reds`,
|
||||
`--run <id>`; `--help` works offline, and an unknown flag or case id exits 2
|
||||
before anything is fetched; every view prints its `scope:` line) and prints
|
||||
per-case per-trial pass rates with 95% Wilson intervals. A series is one case
|
||||
under one input identity, the hash of its own touchfiles minus
|
||||
`GLOBAL_TOUCHFILES` (harness edits do not restart it), per model, Claude CLI
|
||||
@@ -447,8 +527,26 @@ only, when a non-quarantined blocking case meets the entry rule (proposing an
|
||||
entry), when a `rule` case does (rule case behaving like behavior: fix or
|
||||
reclassify), when a blocking case's current identity is significantly below its
|
||||
previous one (one-sided Fisher exact, α = 0.05, at least 6 trials each side,
|
||||
Holm-controlled across the cases tested), and on the quarantine rules above.
|
||||
History that cannot be fetched fails the gate closed.
|
||||
Holm-controlled across the cases tested), on the quarantine rules above, and
|
||||
with a `[headroom]` alarm when a case's slowest session ran above
|
||||
`HEADROOM_FAIL` (85%) of the budget it armed. History that cannot be fetched
|
||||
fails the gate closed.
|
||||
|
||||
The views (`scripts/lib/eval-history.ts`): `--headroom` lists each case's
|
||||
slowest armed session against its budget (warn above `HEADROOM_WARN`, 75%;
|
||||
`insufficient` below 3 samples; timed-out samples marked censored; `unknown`
|
||||
without a session ledger, with the case wall beside it as a labelled upper
|
||||
bound) and the census critical path from the slice job timings. `--reds` lists
|
||||
verdict reds per census lane by failure class and cause, pooled and shrunk
|
||||
per-case red rates, and the all-green probability Π(1 − p_i) over the latest
|
||||
census's cases with an interval, labelled an approximation because verdicts are
|
||||
not proven independent. `--run <id>` is one census's triage: each red with its
|
||||
values and history, and the transcript evidence it downloads from only the
|
||||
slices its reds name (a slice artifact over 64 MB is reported, not fetched).
|
||||
The three thresholds `HEADROOM_WARN`, `HEADROOM_FAIL` and `STALL_WINDOW_MS`
|
||||
live in `test/helpers/eval-budgets.ts`, outside `EVAL_POLICY`; a slow case gets
|
||||
less work, never a larger budget. The red-census walkthrough is
|
||||
[docs/evals/census-red.md](evals/census-red.md).
|
||||
|
||||
**The arithmetic.** With per-trial pass rate p, the chance a single case goes
|
||||
red (a false red while the product works, the catch rate once it has
|
||||
@@ -483,7 +581,8 @@ and judge verdicts at p_rule:
|
||||
The rule term dominates: a green lane on a working product needs rule cases to
|
||||
be near-deterministic (0.999), which is why failing detectors are converted to
|
||||
outcome checks and product defects are fixed or named, and why each census
|
||||
reports its expected lane false-red from the measured rates.
|
||||
reports its all-green probability from the measured rates (`eval:pass-rates
|
||||
--reds`, carried into the census report).
|
||||
|
||||
**Timeout policy.** Paid tests use the tiers in
|
||||
`test/helpers/eval-budgets.ts` (JUDGE/CAPTURE/CAPTURE_LONG/PTY/PTY_LONG);
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
# A census went red
|
||||
|
||||
A periodic census (`evals-periodic.yml`: the weekly periodic lane plus the
|
||||
gate census) or a marathon run reports each red verdict as one line. This page
|
||||
takes one red line to a diagnosis, using free commands first. A paid run is
|
||||
allowed only after a repair.
|
||||
|
||||
Where the report is:
|
||||
|
||||
- **Main** (the scheduled weekly run, or a dispatch on `main`): a comment on the
|
||||
open "Weekly periodic evals: red lane needs triage" issue (marathon: "Weekly
|
||||
marathon evals: red lane needs triage").
|
||||
- **Any branch**: the run's step summary and its `census-report-a<attempt>`
|
||||
artifact (marathon: `marathon-report-a<attempt>`). A branch census never
|
||||
comments on or closes main's issue.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- A checkout of the census branch with `bun install` done. Bun 1.4.0 is the
|
||||
tested version.
|
||||
- `gh` authenticated with read access to the repository's Actions (or
|
||||
`GH_TOKEN` set). `eval:pass-rates` downloads artifacts through the REST zip
|
||||
route, which also works where `gh run download` is blocked.
|
||||
- The run id: the number at the end of the run URL in the report's first line.
|
||||
|
||||
## 1. Read the red line
|
||||
|
||||
A red line in the census report looks like this (one line; a behavior panel
|
||||
with one failed trial shown):
|
||||
|
||||
```
|
||||
✗ plan-review-prosons-neutral-neg behavior FAIL 1/3 (✗✗✓) t1: assertion (session completed; check failed) — expect(received).not.toMatch(expected) · cause assertion: session completed; check failed Expected: not /taste call/i · Received: "a taste call"; t2: assertion [t1@slice 7, t2@slice 9, t3@slice 9, attempt 1] evidence: paid-slice-7-a1/shards/skill-e2e-plan-prosons--plan-review-prosons-neutral-neg.t1 (fetch: bun run eval:pass-rates --run 37193478719 --case plan-review-prosons-neutral-neg) after a repair: bun run scripts/test-paid-shards.ts --tier periodic --case plan-review-prosons-neutral-neg --trials 3
|
||||
```
|
||||
|
||||
| Field | What it says |
|
||||
|---|---|
|
||||
| case, kind, verdict | the registry id, its kind (`rule`, `behavior`, `judge`) and the verdict with each trial's outcome; the brackets name the slice and attempt of each trial |
|
||||
| `failure_class` | the class verdicts read: `assertion`, `contract`, `timeout` or `infra` |
|
||||
| `cause` | what the machine observed (`failure_cause`, glossary in [TESTING_INTERNALS](../TESTING_INTERNALS.md#failure-causes)), with its evidence line |
|
||||
| Expected / Received | the failed matcher's values; for a judge, each failing dimension's mean, threshold, sample count and lowest rationale |
|
||||
| `evidence:` | the slice artifact and shard directory holding the transcript |
|
||||
| `after a repair:` | the command that runs only this case, once a repair exists |
|
||||
|
||||
The cause is a label. Under EVAL_POLICY v1 it never changes a verdict: a
|
||||
`provider_stall` or `refusal` is still a failed trial. Whether an assertion
|
||||
failure was a detector, harness or product fault is your diagnosis, not a
|
||||
machine cause.
|
||||
|
||||
## 2. Inspect the evidence (free)
|
||||
|
||||
```bash
|
||||
bun run eval:pass-rates --run <run-id> # every red of that census
|
||||
bun run eval:pass-rates --run <run-id> --case <id> # one case
|
||||
```
|
||||
|
||||
Expected output: one block per red, with its history and a local evidence path.
|
||||
|
||||
```
|
||||
run 37198445662 (main @ 1a2b3c4d5, 2026-10-04T11:29:00Z): 2 verdict red(s)
|
||||
✗ qa-only/SKILL.md workflow periodic/full FAIL (history: 7/8 verdicts green)
|
||||
t1: assertion / cause assertion: session completed; check failed clarity 3.67 < 4 (3 samples) "…"
|
||||
evidence: ~/.gstack/eval-pass-rates-cache/garrytan-gstack/37198445662/paid-slice-5-a1/shards/skill-llm-eval
|
||||
```
|
||||
|
||||
Only the slices the reds name are downloaded, into
|
||||
`<state root>/eval-pass-rates-cache/<owner>-<repo>/<run-id>/`. The shard
|
||||
directory holds the eval record JSON (transcript, judge scores), `junit.xml`
|
||||
and, for runs after the session ledger landed, `session-ledger.jsonl` (each
|
||||
session's armed budget, elapsed time, end reason and liveness summary). A slice
|
||||
artifact over 64 MB is reported as `too large to fetch`; download it from the
|
||||
run page or with `gh api repos/<owner>/<repo>/actions/artifacts/<id>/zip > slice.zip`.
|
||||
|
||||
## 3. Put it in context (free)
|
||||
|
||||
```bash
|
||||
bun run eval:pass-rates --reds --runs 10 --branch <branch> # reds per census by class and cause, all-green probability
|
||||
bun run eval:pass-rates --headroom --runs 10 --branch <branch> # slowest session per case vs its armed budget
|
||||
bun run eval:pass-rates --case <id> --runs 10 # the case's pass rate per input series
|
||||
```
|
||||
|
||||
`--reds` ends with the all-green probability: Π(1 − p_i) over the latest
|
||||
census's cases, with per-case red rates shrunk toward the pooled rate. It is an
|
||||
approximation (verdicts are not proven independent). A case that is red in
|
||||
several censuses is a repeat offender; its fix is structural (a replay corpus,
|
||||
a structured signal or a speedup), not a one-off spelling patch.
|
||||
|
||||
What incomplete data means:
|
||||
|
||||
| Shown | Meaning |
|
||||
|---|---|
|
||||
| headroom `unknown (no session ledger)` | the trials predate the session ledger, or the runner wrote none; the case wall beside it is an upper bound, not a session clock |
|
||||
| headroom `insufficient` | fewer than 3 session samples; never read as a pass |
|
||||
| headroom `censored` | a sample timed out, so its real duration is at least the budget |
|
||||
| cause `unrecorded` | the trial was written before `failure_cause` existed |
|
||||
| cause `unknown` | the trial failed with no evidence for any other cause |
|
||||
| `cost unknown` | the harness (PTY, Codex) records no billing; only the known sum is shown |
|
||||
| `history unavailable (...)` | the GitHub history could not be read; `--gate` fails closed |
|
||||
|
||||
## 4. Replay and repair (free)
|
||||
|
||||
Diagnose before changing code: product defect, test or detector defect, harness
|
||||
defect, or infrastructure. Then reproduce with the smallest free check:
|
||||
|
||||
```bash
|
||||
bun test test/detector-corpus-<case>.test.ts # cases with a replay corpus
|
||||
bun run eval:pass-rates --reds --dir <evidence dir> # re-read downloaded artifacts offline
|
||||
```
|
||||
|
||||
The repair lands with a free regression test that fails before it and passes
|
||||
after it (`AGENTS.md`, "Validation discipline").
|
||||
|
||||
## 5. After a repair: one paid run
|
||||
|
||||
Run the line's `after a repair:` command, which selects only that case:
|
||||
|
||||
```bash
|
||||
bun run scripts/test-paid-shards.ts --tier periodic --case <id> # a rule case: one diagnostic trial
|
||||
bun run scripts/test-paid-shards.ts --tier periodic --case <id> --trials 3 # a behavior case: its 3-trial panel
|
||||
```
|
||||
|
||||
A judge prints an `EVALS_JUDGE_SELECTION_JSON=... bun test <file> -t '<pattern>'`
|
||||
command instead; a changed judge gets one 3-sample panel.
|
||||
|
||||
Rules (see [TESTING_INTERNALS](../TESTING_INTERNALS.md#eval-verdict-policy)):
|
||||
|
||||
- A paid run follows a concrete repair. A red census is never rerun on
|
||||
unchanged inputs.
|
||||
- A diagnostic run never forms a verdict. The next census on the repaired head
|
||||
decides.
|
||||
- A census whose every red is machine-classified INFRA or INCOMPLETE is
|
||||
re-dispatched once automatically; both runs stay reported.
|
||||
- An unresolved red is listed as a named red with its diagnosis. A case enters
|
||||
`CASE_QUARANTINE` only under its entry rule, and a product defect is never
|
||||
quarantined.
|
||||
+168
-132
@@ -242,6 +242,35 @@ contains no parsable version. /ship stops instead of inventing `0.0.0.0`.
|
||||
|
||||
**Expected result.** `gstack-version-bump classify` prints the current version.
|
||||
|
||||
<a id="review-base-stale"></a>
|
||||
### `BASE_REFRESH: stale <rev>` / `Base coverage: stale at <rev>`
|
||||
|
||||
**Meaning.** /review could not fetch the base branch (offline, no credentials,
|
||||
or a read-only `.git` such as Codex's sandbox), so it reviewed against your
|
||||
local `origin/<base>` at `<rev>`.
|
||||
|
||||
**Fix.** Run `git fetch origin <base>` outside the sandbox and rerun /review if
|
||||
the base has moved.
|
||||
|
||||
<a id="review-fingerprint-tmpdir"></a>
|
||||
### `gstack-wtree: cannot create a temp index under <dir>; set TMPDIR to a writable directory` / `gstack-review-log: cannot create a private object directory under <dir>; no working-tree fingerprint. Set TMPDIR to a writable directory.`
|
||||
|
||||
**Meaning.** /review fingerprints the working tree in a private temporary
|
||||
object directory (Codex's sandbox keeps `.git` read-only). The temp directory
|
||||
could not be created, so this review has no fingerprint and later steps cannot
|
||||
reuse its evidence.
|
||||
|
||||
**Fix.** `export TMPDIR=<a writable directory>`, then run /review again.
|
||||
|
||||
<a id="review-fingerprint-stage"></a>
|
||||
### `gstack-wtree: cannot stage the working tree: <git error>`
|
||||
|
||||
**Meaning.** The fingerprint's objects could not be written. Standalone
|
||||
`gstack-wtree` writes into the repository's object store.
|
||||
|
||||
**Fix.** Run it through /review (`gstack-review-log` supplies a private object
|
||||
directory), or make `.git/objects` writable.
|
||||
|
||||
<a id="plan-audit-not-run"></a>
|
||||
### `Plan completion audit: not run (no plan is bound to this branch and no docs/designs/ file matches). Fix: ...`
|
||||
|
||||
@@ -307,30 +336,6 @@ Or stop the reminder: `gstack-slug --adopt-legacy --dismiss <slug>`.
|
||||
|
||||
---
|
||||
|
||||
<a id="review-base-stale"></a>
|
||||
### `BASE_REFRESH: stale <rev>` / `Base coverage: stale at <rev>`
|
||||
|
||||
**Meaning.** /review could not fetch the base branch (offline, no credentials,
|
||||
or a read-only `.git` such as Codex's sandbox), so it reviewed against your
|
||||
local `origin/<base>` at `<rev>`.
|
||||
|
||||
**Fix.** Run `git fetch origin <base>` outside the sandbox and rerun /review if
|
||||
the base has moved.
|
||||
|
||||
<a id="review-fingerprint-objects"></a>
|
||||
### `gstack-review-log: cannot create a private object directory under <dir>; no working-tree fingerprint.` / `gstack-wtree: cannot create a temp index under <dir>`
|
||||
|
||||
**Meaning.** The working-tree fingerprint needs a writable temp directory.
|
||||
|
||||
**Fix.** `export TMPDIR=<writable dir>` and rerun.
|
||||
|
||||
### `gstack-wtree: cannot stage the working tree: ...` / `gstack-wtree: cannot write the tree: ...`
|
||||
|
||||
**Meaning.** The object directory could not be written. `gstack-review-log`
|
||||
supplies a private one; standalone `gstack-wtree` writes to `.git/objects`.
|
||||
|
||||
**Fix.** Run it through `gstack-review-log`, or make `.git/objects` writable.
|
||||
|
||||
## Design skills
|
||||
|
||||
<a id="design-not-available"></a>
|
||||
@@ -353,6 +358,15 @@ If the message names a missing timeout tool, install coreutils (`gtimeout`) or p
|
||||
|
||||
**Expected result.** The next design skill prints `DESIGN_READY`.
|
||||
|
||||
<a id="design-image-model-invalid"></a>
|
||||
### `GSTACK_DESIGN_IMAGE_MODEL="<value>" is not a gpt-image model name (expected something like gpt-image-2); fix it or unset it to use gpt-image-2`
|
||||
|
||||
**Meaning.** The design binary refuses an image tool model override that is not
|
||||
a `gpt-image-*` model name, before sending any request.
|
||||
|
||||
**Fix.** `export GSTACK_DESIGN_IMAGE_MODEL=gpt-image-2` (or another gpt-image
|
||||
model your key can use), or `unset GSTACK_DESIGN_IMAGE_MODEL`.
|
||||
|
||||
<a id="design-taste-profile-unavailable"></a>
|
||||
### `TASTE_PROFILE_UNAVAILABLE: could not resolve the project slug (gstack-slug failed). Fix: run ./setup.`
|
||||
|
||||
@@ -375,15 +389,6 @@ pick a new `--output`), then copy the recovery file to where you want it.
|
||||
|
||||
---
|
||||
|
||||
<a id="design-image-model-invalid"></a>
|
||||
### `GSTACK_DESIGN_IMAGE_MODEL="..." is not a gpt-image model name`
|
||||
|
||||
**Meaning.** The image model override must look like `gpt-image-2`; anything
|
||||
else is refused before a request is sent. The live model check always tests
|
||||
the defaults, whatever this is set to.
|
||||
|
||||
**Fix.** Set a gpt-image model name, or `unset GSTACK_DESIGN_IMAGE_MODEL`.
|
||||
|
||||
## Codex and other env-var hosts
|
||||
|
||||
<a id="gstack-no-install-found"></a>
|
||||
@@ -446,30 +451,35 @@ stopped instead of touching your project.
|
||||
## Setup and auto-update
|
||||
|
||||
<a id="bun-too-old"></a>
|
||||
### `gstack needs Bun 1.3.3 or newer (1.4.0 recommended); found X at <path>. Nothing was installed or changed.`
|
||||
### `gstack needs Bun 1.3.3 or newer (1.4.0 recommended); found <version> at <path>. Nothing was installed or changed.`
|
||||
|
||||
Also: `warning: gstack is tested on Bun 1.4.0 (CI pin); found X` and
|
||||
`warning: could not read the Bun version (...); continuing.`
|
||||
|
||||
**Meaning.** Bun older than 1.3.3 accepts but ignores the flags that stop
|
||||
**Meaning.** Bun older than 1.3.3 silently ignores the build flags that stop
|
||||
gstack's compiled tools from reading a project's `.env`, so setup refuses it
|
||||
before writing anything. Bun 1.3.3 to 1.3.x works but is untested; an
|
||||
unreadable version only warns.
|
||||
before writing anything. The same link appears on two warnings that do not stop
|
||||
setup: `warning: gstack is tested on Bun 1.4.0 (CI pin); found <version>` (1.3.3
|
||||
up to 1.4.0 works but is untested) and `warning: could not read the Bun version
|
||||
(...)` (setup continued; check `bun --version`).
|
||||
|
||||
**Fix.** `bun upgrade`, then `./setup`.
|
||||
**Fix.**
|
||||
|
||||
```bash
|
||||
bun upgrade
|
||||
./setup
|
||||
```
|
||||
|
||||
**Expected result.** `bun --version` prints 1.4.0 or newer and setup finishes
|
||||
with no Bun warning.
|
||||
|
||||
<a id="auto-update-bun-too-old"></a>
|
||||
### `gstack auto-update: update held (bun-too-old: found Bun X at <path>; gstack V needs F or newer); nothing was installed or changed.`
|
||||
### `gstack auto-update: update held (bun-too-old: found Bun <version> at <path>; gstack <version> needs <floor> or newer); nothing was installed or changed. Fix now: ...`
|
||||
|
||||
Also `/gstack-upgrade`'s `BUN_TOO_OLD: bun-too-old: ...; nothing was changed`.
|
||||
**Meaning.** Team-mode auto-update fetched a release whose setup needs a newer
|
||||
Bun than the one setup would run. It left your checkout and installed skills at
|
||||
the current revision and prints this line once per session start. (An older
|
||||
auto-updater running its first update cannot make this check.)
|
||||
|
||||
**Meaning.** The incoming release needs a newer Bun than the one setup would
|
||||
run, so the update stopped before moving your checkout. Your installed version
|
||||
keeps working. The first update run by an auto-updater older than this check
|
||||
cannot do it.
|
||||
|
||||
**Fix.** `bun upgrade`. The next session start resumes the update, or run
|
||||
`cd <gstack checkout> && git pull --ff-only && ./setup` (or `/gstack-upgrade`).
|
||||
**Fix.** `bun upgrade`; the next session start resumes the update. To update
|
||||
now: `bun upgrade && cd <gstack checkout> && git pull --ff-only && ./setup`.
|
||||
|
||||
<a id="auto-update-incomplete"></a>
|
||||
### `gstack auto-update: setup did not finish (<reason>); installed skills may be out of date. gstack retries automatically. Fix now: cd <dir> && ./setup`
|
||||
@@ -581,16 +591,19 @@ after `--`).
|
||||
|
||||
**Fix.** `cd <gstack checkout> && ./setup` (or `/unfreeze`).
|
||||
|
||||
---
|
||||
|
||||
<a id="browse-chromium-pid-unrecorded"></a>
|
||||
### `[browse] Could not record the Chromium PID (CDP SystemInfo.getProcessInfo: ...); browse stop cannot reap a surviving Chromium.`
|
||||
### `[browse] Could not record the Chromium PID (CDP SystemInfo.getProcessInfo: <error>); browse stop cannot reap a surviving Chromium.`
|
||||
|
||||
**Meaning.** gstack reads the PID of a browser it launched over CDP so `browse
|
||||
stop` can clean up a Chromium that outlives the server. That read failed, so
|
||||
this session's browser is not reaped automatically.
|
||||
**Meaning.** For a headless Chromium it launched, browse reads the browser's
|
||||
process id over CDP so `browse stop` can kill a browser that outlives the
|
||||
server. That read failed or took longer than 2 seconds, so this session's
|
||||
browser is not recorded. Browsers gstack did not launch are never recorded or
|
||||
killed.
|
||||
|
||||
**Fix.** After `browse stop`, check `ps` for a leftover Chromium and kill it by hand.
|
||||
**Fix.** After `browse stop`, check for a leftover browser with
|
||||
`ps aux | grep -i chrom` and end it with `kill <pid>`.
|
||||
|
||||
---
|
||||
|
||||
## Memory and gbrain
|
||||
|
||||
@@ -665,6 +678,104 @@ gstack-gbrain-sync --prune-gone-worktrees
|
||||
|
||||
---
|
||||
|
||||
## Eval reports and pass-rates
|
||||
|
||||
These come from the paid eval census reports and `bun run eval:pass-rates`.
|
||||
The walkthrough for a red census is [docs/evals/census-red.md](evals/census-red.md).
|
||||
|
||||
<a id="pass-rates-unknown-flag"></a>
|
||||
### `eval:pass-rates: unknown flag <flag>` / `eval:pass-rates: unknown case <id> (not an E2E, judge or paid test file id)`
|
||||
|
||||
**Meaning.** The flag or case id is not recognized. The command exits 2 before
|
||||
fetching anything.
|
||||
|
||||
**Fix.** `bun run eval:pass-rates --help` lists the flags. `--case` takes a
|
||||
registry id from `test/helpers/touchfiles-data.ts` or a paid test file path.
|
||||
|
||||
<a id="failure-cause-newer"></a>
|
||||
### `unknown failure_cause "<x>" (written by a newer gstack; update this checkout to read it)`
|
||||
|
||||
**Meaning.** A trial record names a failure cause this checkout does not know.
|
||||
A newer gstack wrote it.
|
||||
|
||||
**Fix.** `git pull` (or rebase your branch onto `main`), then rerun the command.
|
||||
|
||||
<a id="paid-case-several-owners"></a>
|
||||
### `--case <id>: registered by <file>, <file>; it needs exactly one`
|
||||
|
||||
**Meaning.** More than one paid test file registers the case, so
|
||||
`test-paid-shards.ts --case` cannot pick one file.
|
||||
|
||||
**Fix.** Run the red shard's file directly; the report's `after a repair:` line
|
||||
prints it: `EVALS=1 EVALS_TIER=<tier> bun test <file>`.
|
||||
|
||||
<a id="evidence-too-large"></a>
|
||||
### `evidence: <artifact> too large to fetch (<N> MB)`
|
||||
|
||||
**Meaning.** The slice artifact holding this red's transcript is over 64 MB, so
|
||||
`eval:pass-rates --run` does not download it.
|
||||
|
||||
**Fix.** Download it from the run page, or
|
||||
`gh api repos/<owner>/<repo>/actions/artifacts/<artifact id>/zip > slice.zip`.
|
||||
|
||||
<a id="headroom-alarm"></a>
|
||||
### `[headroom] <case> session <key>: max <s> of <s> (<pct>) over <n> sample(s), above the 85% cap. Cut work in the skill or fixture; budgets are never raised ...`
|
||||
|
||||
**Meaning.** `eval:pass-rates --gate` (the weekly report) found a case whose
|
||||
slowest session used more than 85% of the timeout it armed. It is one timeout
|
||||
away from a red.
|
||||
|
||||
**Fix.** Cut work in the skill or fixture. Budgets are never raised. Check the
|
||||
case with `bun run eval:pass-rates --headroom --case <id>`.
|
||||
|
||||
<a id="cost-unknown"></a>
|
||||
### `cost unknown (no billing captured for any of <N> trial(s))` / `cost $<x> known + <N> of <M> trial(s) cost unknown`
|
||||
|
||||
**Meaning.** PTY and Codex sessions record no billing, so their cost is
|
||||
unknown. Only the known sum is shown.
|
||||
|
||||
**Fix.** None needed; this is informational.
|
||||
|
||||
<a id="cause-provider-stall"></a>
|
||||
### `· cause provider_stall: no stream event for <N>s ...`
|
||||
|
||||
**Meaning.** The session streamed partial messages, then received no event of
|
||||
any kind for at least 120 seconds while a model request was in flight and no
|
||||
tool, permission prompt, hook or subagent was outstanding. Under EVAL_POLICY v1
|
||||
it is still a failed trial.
|
||||
|
||||
**Fix.** Inspect it with `bun run eval:pass-rates --run <run id> --case <id>`.
|
||||
A paid rerun happens only after a repair.
|
||||
|
||||
<a id="overlay-wrong-answer"></a>
|
||||
### Overlay record with `taskCorrect: false` and `answerError` (contract v4)
|
||||
|
||||
**Meaning.** The trial completed with a wrong answer. Under overlay contract v4
|
||||
that is a valid measurement in the comparison; only an overlay-ON wrong answer
|
||||
fails the case.
|
||||
|
||||
**Fix.** Read `answerError` in the trial JSON. No rerun is needed.
|
||||
|
||||
<a id="detector-corpora-stale"></a>
|
||||
### `[detector-corpora] <N> stale entries (refresh from a current census when one fails)`
|
||||
|
||||
**Meaning.** Some replay-corpus entries came from an older input series of
|
||||
their case. This is informational; the corpus still replays them.
|
||||
|
||||
**Fix.** When that case goes red in a census, add the new capture to its corpus
|
||||
and check it with `bun test test/detector-corpus-<case>.test.ts`.
|
||||
|
||||
<a id="auq-substance-panel"></a>
|
||||
### `recommendation substance mean <x> < 4 over samples [<a>,<b>,<c>] (boilerplate/weak)`
|
||||
|
||||
**Meaning.** The auq-matrix case scores each captured question's recommendation
|
||||
with a 3-sample judge panel; their mean was below 4.
|
||||
|
||||
**Fix.** Read the logged samples and the captured question. A fix goes in the
|
||||
skill text, followed by one diagnostic run of auq-matrix.
|
||||
|
||||
---
|
||||
|
||||
## Push guard (redaction)
|
||||
|
||||
<a id="redact-postgres-default-pair"></a>
|
||||
@@ -685,78 +796,3 @@ Bypass once: `GSTACK_REDACT_PREPUSH=skip git push`.
|
||||
|
||||
**Fix.** Write it as `version: 1.2.3.4`, `"version": "..."` or `v1.2.3.4`.
|
||||
MEDIUM findings do not block pushes.
|
||||
|
||||
---
|
||||
|
||||
## Evals and census reports (contributors)
|
||||
|
||||
<a id="pass-rates-unknown-flag"></a>
|
||||
### `eval:pass-rates: unknown flag --x` / `unknown case <id>`
|
||||
|
||||
**Meaning.** The flag or case id is not recognized; it exits 2 before fetching anything.
|
||||
|
||||
**Fix.** `bun run eval:pass-rates --help`; use a registry case id or a paid test file path.
|
||||
|
||||
<a id="unknown-failure-cause"></a>
|
||||
### `unknown failure_cause "<x>" (written by a newer gstack; update this checkout to read it)`
|
||||
|
||||
**Fix.** `git pull` (or rebase your branch) and rerun.
|
||||
|
||||
<a id="case-registered-by-several"></a>
|
||||
### `--case <id>: registered by a, b; it needs exactly one`
|
||||
|
||||
**Fix.** Run the red shard's file: `EVALS=1 EVALS_TIER=<tier> bun test <file>`
|
||||
(the report's "after a repair:" line prints it).
|
||||
|
||||
<a id="evidence-too-large"></a>
|
||||
### `evidence: <artifact> too large to fetch (N MB)`
|
||||
|
||||
**Fix.** Download it from the run page, or
|
||||
`gh api repos/<owner>/<repo>/actions/artifacts/<id>/zip > slice.zip`.
|
||||
|
||||
<a id="headroom-alarm"></a>
|
||||
### `[headroom] <case> session <key>: max Xs of Ys (Z%) ... above the 85% cap`
|
||||
|
||||
**Meaning.** The case's slowest session used more than 85% of the timeout it armed.
|
||||
|
||||
**Fix.** Cut work in the skill or fixture. Budgets are never raised.
|
||||
|
||||
<a id="cost-unknown"></a>
|
||||
### `cost unknown (no billing captured for any of N trial(s))` / `... N of M trial(s) cost unknown`
|
||||
|
||||
**Meaning.** PTY and Codex harnesses record no billing, so only the known sum is
|
||||
shown. Informational.
|
||||
|
||||
<a id="provider-stall"></a>
|
||||
### Red line `· cause provider_stall: no stream event for Ns ...`
|
||||
|
||||
**Meaning.** The model request went silent for at least 120 s with no tool,
|
||||
permission prompt, hook or subagent outstanding. Under EVAL_POLICY v1 this is
|
||||
still a failure.
|
||||
|
||||
**Fix.** Inspect it with `bun run eval:pass-rates --run <id> --case <id>`. A
|
||||
paid rerun only after a repair.
|
||||
|
||||
<a id="overlay-contract-v4"></a>
|
||||
### `overlay ... contract-v4 ... taskCorrect: false` with `answerError` set
|
||||
|
||||
**Meaning.** The trial completed with a wrong answer; that is a valid
|
||||
measurement in the comparison. Only an overlay-on wrong answer fails the case.
|
||||
|
||||
**Fix.** Read `answerError` in the trial JSON. No rerun needed.
|
||||
|
||||
<a id="detector-corpora-stale"></a>
|
||||
### `[detector-corpora] N stale entries (refresh from a current census when one fails)`
|
||||
|
||||
**Meaning.** Corpus entries came from an older input series of their case. Informational.
|
||||
|
||||
**Fix.** When that case goes red in a census, add the new capture and check it
|
||||
with `bun test test/detector-corpus-<case>.test.ts`.
|
||||
|
||||
<a id="auq-substance-mean"></a>
|
||||
### `recommendation substance mean X < 4 over samples [a,b,c]` (auq-matrix)
|
||||
|
||||
**Meaning.** The 3-sample judge panel's mean was below the threshold.
|
||||
|
||||
**Fix.** Read the logged samples and the captured question; fix the skill
|
||||
text, then run one diagnostic.
|
||||
@@ -263,6 +263,7 @@ export const CODEX_CI_FILES: readonly string[] = [
|
||||
'test/skill-e2e-outside-voice.test.ts',
|
||||
'test/skill-e2e-outside-plan-disabled.test.ts',
|
||||
'test/skill-e2e-safety-codex-boundary.test.ts',
|
||||
'test/codex-e2e-multiblock-live.test.ts',
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
@@ -122,6 +122,7 @@
|
||||
"test/carve-section-loading-review.test.ts": 132284,
|
||||
"test/carve-section-loading-setup-gbrain.test.ts": 73575,
|
||||
"test/carve-section-loading-spec.test.ts": 137613,
|
||||
"test/codex-e2e-multiblock-live.test.ts": 180000,
|
||||
"test/codex-e2e-recommendation-substance.test.ts": 14617,
|
||||
"test/codex-e2e-shared-libs.test.ts": 49563,
|
||||
"test/codex-e2e-sol-scope.test.ts": 32228,
|
||||
@@ -142,6 +143,7 @@
|
||||
"test/skill-e2e-coverage-audit.test.ts": 1000,
|
||||
"test/skill-e2e-cso.test.ts": 303949,
|
||||
"test/skill-e2e-deploy.test.ts": 1000,
|
||||
"test/skill-e2e-design-model-smoke.test.ts": 90000,
|
||||
"test/skill-e2e-design.test.ts": 817000,
|
||||
"test/skill-e2e-design.test.ts#design-consultation-core": 173162,
|
||||
"test/skill-e2e-design.test.ts#design-consultation-existing": 225306,
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* The red-census docs stay true to the code they describe: the failure-cause
|
||||
* glossary in docs/TESTING_INTERNALS.md lists exactly FAILURE_CAUSES, the
|
||||
* guide's commands use flags eval:pass-rates and test-paid-shards accept, and
|
||||
* the guide is linked from CONTRIBUTING.md and both census reports.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { FAILURE_CAUSES } from './helpers/eval-store';
|
||||
import { parsePassRatesArgs } from '../scripts/lib/eval-history';
|
||||
|
||||
const ROOT = path.join(import.meta.dir, '..');
|
||||
const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf8');
|
||||
const internals = read('docs/TESTING_INTERNALS.md');
|
||||
const guide = read('docs/evals/census-red.md');
|
||||
|
||||
describe('red-census docs', () => {
|
||||
test('the failure-cause glossary has one row per FAILURE_CAUSES value, in precedence order', () => {
|
||||
const section = internals.slice(internals.indexOf('<a id="failure-causes"></a>'));
|
||||
const rows = [...section.slice(0, section.indexOf('\n\n', section.indexOf('| `failure_cause` |'))).matchAll(/^\| `([a-z_]+)` \|/gm)]
|
||||
.map(m => m[1]).filter(cause => cause !== 'failure_cause');
|
||||
expect(rows).toEqual([...FAILURE_CAUSES]);
|
||||
});
|
||||
|
||||
test('every eval:pass-rates command in the guide parses', () => {
|
||||
const commands = [...guide.matchAll(/^bun run eval:pass-rates ([^\n#]*)/gm)].map(m => m[1]!.trim());
|
||||
expect(commands.length).toBeGreaterThanOrEqual(6);
|
||||
for (const command of commands) {
|
||||
const args = command.replace(/<[^>]+>/g, 'x').split(/\s+/).filter(Boolean)
|
||||
.map((arg, i, all) => (all[i - 1] === '--run' || all[i - 1] === '--runs' ? '10' : arg));
|
||||
const parsed = parsePassRatesArgs(args, () => true);
|
||||
expect('error' in parsed ? parsed.error : null, command).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
test('the guide is linked from CONTRIBUTING, TESTING_INTERNALS and both census reports', () => {
|
||||
expect(read('CONTRIBUTING.md')).toContain('docs/evals/census-red.md');
|
||||
expect(internals).toContain('(evals/census-red.md)');
|
||||
for (const workflow of ['evals-periodic.yml', 'evals-marathon.yml']) {
|
||||
expect(read(`.github/workflows/${workflow}`)).toContain('/docs/evals/census-red.md');
|
||||
}
|
||||
for (const anchor of ['failure-causes', 'eval-verdict-policy']) expect(internals).toContain(`<a id="${anchor}"></a>`);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* Live Codex multi-block check (E6, periodic, host-run Codex job).
|
||||
*
|
||||
* Codex runs every shell block of a skill in a fresh shell. This case runs
|
||||
* `./setup --host codex` once from the job's own checkout into a fresh HOME
|
||||
* and CODEX_HOME, then checks what a Codex user gets:
|
||||
* 1. `codex debug prompt-input` lists the `gstack` router, and the router
|
||||
* it names is a real file (Codex skips a symlinked SKILL.md);
|
||||
* 2. `codex exec` running /learn's "Show recent" block, a later block that
|
||||
* only its own prelude can root, runs the installed helper and prints the
|
||||
* learning seeded before the run; no block reports a missing install or
|
||||
* an unresolved gstack helper path.
|
||||
* Auth is the CI login home's auth.json only (scopeCodexAccess hands this
|
||||
* file the CLI on PATH and that home as CODEX_HOME).
|
||||
*/
|
||||
import { describe, test, afterAll, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { spawn, spawnSync } from 'child_process';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { e2eTierEnabled } from './helpers/e2e-gate';
|
||||
import { hermeticChildEnv } from './helpers/hermetic-env';
|
||||
import { parseCodexJSONL, type CodexResult } from './helpers/codex-session-runner';
|
||||
import { CODEX_EVAL_FINALIZE_MS, createCodexEvalCollector, runRecordedCodexEval } from './helpers/codex-eval';
|
||||
import { MULTIBLOCK_SENTINEL_KEY, multiblockProblems, routerPathFromPromptInput } from './helpers/codex-multiblock';
|
||||
import { CODEX_FRONTIER_MODEL } from '../scripts/resolvers/constants';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const CASE = 'codex-multiblock-live';
|
||||
const SETUP_MS = 300_000;
|
||||
const CODEX_AVAILABLE = spawnSync('which', ['codex'], { timeout: 30_000 }).status === 0;
|
||||
const ENABLED = e2eTierEnabled('periodic') && CODEX_AVAILABLE;
|
||||
if (e2eTierEnabled('periodic') && !CODEX_AVAILABLE) process.stderr.write(`\n${CASE}: SKIPPED: codex binary not on PATH\n`);
|
||||
|
||||
const collector = ENABLED ? createCodexEvalCollector('codex-e2e-multiblock-live') : null;
|
||||
afterAll(async () => { await collector?.finalize(); });
|
||||
|
||||
/** `codex exec --json` with stdout/stderr in files (no pipe can outlive the process group), killed on abort. */
|
||||
function codexExec(args: string[], env: NodeJS.ProcessEnv, cwd: string, scratch: string, signal: AbortSignal): Promise<CodexResult> {
|
||||
const started = Date.now();
|
||||
const outFile = path.join(scratch, 'exec.jsonl');
|
||||
const errFile = path.join(scratch, 'exec.stderr');
|
||||
const out = fs.openSync(outFile, 'w');
|
||||
const err = fs.openSync(errFile, 'w');
|
||||
const proc = spawn('codex', args, { cwd, env, stdio: ['ignore', out, err], detached: true });
|
||||
fs.closeSync(out);
|
||||
fs.closeSync(err);
|
||||
const kill = () => { try { process.kill(-proc.pid!, 'SIGKILL'); } catch { /* already gone */ } };
|
||||
signal.addEventListener('abort', kill, { once: true });
|
||||
return new Promise(resolve => {
|
||||
const finish = (exitCode: number) => {
|
||||
signal.removeEventListener('abort', kill);
|
||||
kill();
|
||||
const rawLines = fs.readFileSync(outFile, 'utf8').split('\n').filter(line => line.trim());
|
||||
resolve({ ...parseCodexJSONL(rawLines), rawLines, exitCode: signal.aborted ? 124 : exitCode,
|
||||
durationMs: Date.now() - started, stderr: fs.readFileSync(errFile, 'utf8') });
|
||||
};
|
||||
proc.once('error', () => finish(1));
|
||||
proc.once('exit', (code, sig) => finish(code ?? (sig ? 128 + (os.constants.signals[sig] ?? 0) : 1)));
|
||||
});
|
||||
}
|
||||
|
||||
(ENABLED ? describe : describe.skip)('live Codex multi-block install (E6)', () => {
|
||||
test(CASE, async () => {
|
||||
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'e6-'));
|
||||
try {
|
||||
const home = path.join(scratch, 'home');
|
||||
const codexHome = path.join(home, '.codex');
|
||||
const repo = path.join(scratch, 'repo');
|
||||
// Codex will not create its sandbox helper aliases under its temp dir, so TMPDIR sits beside CODEX_HOME.
|
||||
const tmp = path.join(home, 'tmp');
|
||||
for (const dir of [codexHome, repo, tmp]) fs.mkdirSync(dir, { recursive: true });
|
||||
const ciHome = process.env.CODEX_HOME ?? path.join(os.homedir(), '.codex');
|
||||
if (fs.existsSync(path.join(ciHome, 'auth.json'))) fs.copyFileSync(path.join(ciHome, 'auth.json'), path.join(codexHome, 'auth.json'));
|
||||
const env = hermeticChildEnv({ HOME: home, CODEX_HOME: codexHome, TMPDIR: tmp }, { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] });
|
||||
|
||||
const setupStarted = Date.now();
|
||||
const setup = spawnSync('./setup', ['--host', 'codex'], { cwd: ROOT, env, encoding: 'utf8', timeout: SETUP_MS });
|
||||
console.log(`${CASE}: ./setup --host codex exit ${setup.status} in ${Math.round((Date.now() - setupStarted) / 1000)}s`);
|
||||
expect(setup.status, `${setup.stdout.slice(-2000)}\n${setup.stderr.slice(-2000)}`).toBe(0);
|
||||
|
||||
const promptInput = spawnSync('codex', ['debug', 'prompt-input'], { cwd: repo, env, encoding: 'utf8', timeout: 60_000 });
|
||||
expect(promptInput.status, promptInput.stderr).toBe(0);
|
||||
const router = routerPathFromPromptInput(promptInput.stdout);
|
||||
expect(router, 'codex debug prompt-input does not list the gstack router').toBe(path.join(codexHome, 'skills', 'gstack', 'SKILL.md'));
|
||||
expect(fs.lstatSync(router!).isFile()).toBe(true);
|
||||
|
||||
for (const args of [['init', '-q'], ['remote', 'add', 'origin', 'https://github.com/example/e6-fixture.git']]) {
|
||||
expect(spawnSync('git', args, { cwd: repo, env, timeout: 30_000 }).status).toBe(0);
|
||||
}
|
||||
const seed = spawnSync(path.join(codexHome, 'skills', 'gstack', 'bin', 'gstack-learnings-log'), [JSON.stringify({ skill: 'learn',
|
||||
type: 'pitfall', key: MULTIBLOCK_SENTINEL_KEY, insight: 'Seeded by the E6 live Codex check', confidence: 9, source: 'user-stated' })],
|
||||
{ cwd: repo, env, encoding: 'utf8', timeout: 30_000 });
|
||||
expect(seed.status, seed.stderr).toBe(0);
|
||||
|
||||
const result = await runRecordedCodexEval({
|
||||
name: CASE,
|
||||
suite: 'codex-e2e-multiblock-live',
|
||||
budgetMs: CAPTURE_MS,
|
||||
run: signal => codexExec(['exec', '--json', '-s', 'read-only', '--skip-git-repo-check',
|
||||
'--model', process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL,
|
||||
'Use the gstack learn skill to show this project\'s recent learnings. Run its "Show recent" step, then list what it printed.'],
|
||||
env, repo, scratch, signal),
|
||||
validate: run => {
|
||||
const problems = multiblockProblems(run.rawLines);
|
||||
if (problems.length) throw new Error(problems.join('\n'));
|
||||
},
|
||||
record: entry => collector?.addTest(entry),
|
||||
});
|
||||
console.log(`${CASE}: ${result.toolCalls.length} commands, ${result.tokens} tokens, ${Math.round(result.durationMs / 1000)}s`);
|
||||
} finally {
|
||||
fs.rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
}, SETUP_MS + 120_000 + CAPTURE_MS + CODEX_EVAL_FINALIZE_MS);
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Free replays for the live Codex multi-block oracles (E6): synthetic
|
||||
* `codex exec --json` events in the shape codex-cli 0.160 writes (see
|
||||
* test/fixtures/codex-sandbox/exec-json-healthy.jsonl) and a
|
||||
* `codex debug prompt-input` skills block.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { MULTIBLOCK_SENTINEL_KEY, multiblockProblems, routerPathFromPromptInput } from './helpers/codex-multiblock';
|
||||
|
||||
const PRELUDE = '[ -d "${GSTACK_ROOT:-/-}/bin" ]&&[ -d "$GSTACK_ROOT/lib" ]||{ _r=$(git rev-parse --show-toplevel 2>/dev/null)/.agents/skills/gstack;'
|
||||
+ '[ -d "$_r/bin" ]||_r=${CODEX_HOME:-~/.codex}/skills/gstack;[ -d "$_r/bin" ]||{ echo "gstack: no install found (tried $_r). Fix: ./setup --host codex from your gstack checkout; ./setup --status shows it.">&2;exit 1;};GSTACK_ROOT=$_r;}';
|
||||
const SHOW_RECENT = `${PRELUDE}\nSLUG=$($GSTACK_ROOT/bin/gstack-slug --get SLUG 2>/dev/null)\n$GSTACK_ROOT/bin/gstack-learnings-search --limit 20 2>/dev/null || echo "No learnings yet."`;
|
||||
const LISTING = `LEARNINGS: 1 loaded (1 pitfall)\n\n## Pitfalls\n- [${MULTIBLOCK_SENTINEL_KEY}] (confidence: 9/10, user-stated, 2026-10-04)\n Seeded by the E6 live Codex check\n`;
|
||||
|
||||
const command = (cmd: string, output: string, exitCode: number) => JSON.stringify({ type: 'item.completed',
|
||||
item: { id: 'item_2', type: 'command_execution', command: `/bin/bash -lc ${JSON.stringify(cmd)}`, aggregated_output: output, exit_code: exitCode, status: exitCode === 0 ? 'completed' : 'failed' } });
|
||||
const run = (...lines: string[]) => ['{"type":"thread.started","thread_id":"t"}', ...lines, '{"type":"turn.completed","usage":{"input_tokens":1,"output_tokens":1}}'];
|
||||
const preamble = command(`${PRELUDE}\nGSTACK_BIN=$GSTACK_ROOT/bin\n"$GSTACK_BIN/gstack-skill-start" --skill "learn"`, 'BRANCH: main\n', 0);
|
||||
|
||||
describe('multiblockProblems', () => {
|
||||
test('a later block rooted by its own prelude that prints the seeded learning passes', () => {
|
||||
expect(multiblockProblems(run(preamble, command(SHOW_RECENT, LISTING, 0)))).toEqual([]);
|
||||
});
|
||||
|
||||
test('a block that cannot find the install fails, even when another block succeeded', () => {
|
||||
const missing = command(SHOW_RECENT, 'gstack: no install found (tried /root/.codex/skills/gstack). Fix: ./setup --host codex', 1);
|
||||
const problems = multiblockProblems(run(preamble, missing, command(SHOW_RECENT, LISTING, 0)));
|
||||
expect(problems).toHaveLength(1);
|
||||
expect(problems[0]).toStartWith('a block could not find the install');
|
||||
});
|
||||
|
||||
test('the pre-prelude symptom (an empty root, /bin/gstack-slug) fails', () => {
|
||||
const problems = multiblockProblems(run(command('SLUG=$($GSTACK_ROOT/bin/gstack-slug --get SLUG)\n$GSTACK_ROOT/bin/gstack-learnings-search',
|
||||
'bash: line 1: /bin/gstack-slug: No such file or directory\nbash: line 2: /bin/gstack-learnings-search: No such file or directory\n', 127)));
|
||||
expect(problems.some(p => p.startsWith('a gstack helper path did not resolve: /bin/gstack-slug: No such file or directory'))).toBe(true);
|
||||
expect(problems.some(p => p.startsWith('no learnings-search block rooted by its own prelude printed the seeded learning'))).toBe(true);
|
||||
});
|
||||
|
||||
test('a preamble-only run, a run without the seeded learning, or a hand-set root does not count', () => {
|
||||
expect(multiblockProblems(run(preamble))).toEqual(['no block ran gstack-learnings-search (the skill\'s "Show recent" block)']);
|
||||
expect(multiblockProblems(run(preamble, command(SHOW_RECENT, 'No learnings yet.\n', 0)))[0]).toContain('exit 0, sentinel absent');
|
||||
const handSet = command('GSTACK_ROOT=/home/u/.codex/skills/gstack\n$GSTACK_ROOT/bin/gstack-learnings-search --limit 20', LISTING, 0);
|
||||
expect(multiblockProblems(run(preamble, handSet))[0]).toContain('GSTACK_ROOT set by hand');
|
||||
});
|
||||
});
|
||||
|
||||
describe('routerPathFromPromptInput', () => {
|
||||
const promptInput = (entries: string[]) => JSON.stringify([{ type: 'message', role: 'developer', content: [{ type: 'input_text',
|
||||
text: ['<skills_instructions>', '## Skills', '### Skill roots', '- `r0` = `/h/.codex/skills`', '- `r1` = `/h/.codex/skills/.system`',
|
||||
'### Available skills', '- skill-creator: Create or update a Codex skill. (file: r1/skill-creator/SKILL.md)', ...entries].join('\n') }] }], null, 2);
|
||||
|
||||
test('names the router file through the skill roots table', () => {
|
||||
const listed = promptInput(['- gstack: Route a task to the right gstack skill. (gstack) (file: r0/gstack/SKILL.md)',
|
||||
'- learn: Manage project learnings. (gstack) (file: r0/gstack-learn/SKILL.md)']);
|
||||
expect(routerPathFromPromptInput(`WARNING: proceeding without PATH aliases\n${listed}`)).toBe('/h/.codex/skills/gstack/SKILL.md');
|
||||
});
|
||||
|
||||
test('is null when Codex lists gstack skills but not the router', () => {
|
||||
expect(routerPathFromPromptInput(promptInput(['- learn: Manage project learnings. (gstack) (file: r0/gstack-learn/SKILL.md)']))).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,193 @@
|
||||
/**
|
||||
* The weekly tracking issues (evals-periodic.yml, evals-marathon.yml).
|
||||
*
|
||||
* Only main posts to or closes a tracking issue: a branch-dispatched census
|
||||
* writes the same report to its step summary and a report artifact instead.
|
||||
* The periodic report carries the full pass-rate alarm lines, the red ledger,
|
||||
* session headroom and the all-green estimate, read with an explicit
|
||||
* --branch, and every report-derived line passes the published-text
|
||||
* sanitizer. The write steps run here as shell scripts with a fake
|
||||
* `eval:pass-rates`; the sanitizer is the real one.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
const ROOT = path.join(import.meta.dir, '..');
|
||||
type Step = { name?: string; if?: string; run?: string; uses?: string; with?: Record<string, string> };
|
||||
type Workflow = { jobs: Record<string, { steps?: Step[] }> };
|
||||
const WORKFLOWS = ['evals-periodic.yml', 'evals-marathon.yml'] as const;
|
||||
const parse = (name: string) => Bun.YAML.parse(fs.readFileSync(path.join(ROOT, '.github/workflows', name), 'utf8')) as Workflow;
|
||||
const MAIN_ONLY = "github.ref == 'refs/heads/main'";
|
||||
|
||||
describe('tracking issues are main-only', () => {
|
||||
for (const name of WORKFLOWS) {
|
||||
test(`${name}: only the upsert and close steps call gh issue, and both run on main only`, () => {
|
||||
const issueSteps = Object.values(parse(name).jobs).flatMap(job => job.steps ?? []).filter(step => /\bgh issue\b/.test(step.run ?? ''));
|
||||
expect(issueSteps.map(step => step.name).sort()).toEqual(['Close the tracking issue on a green run', 'Upsert tracking issue on failure']);
|
||||
for (const step of issueSteps) expect(step.if, step.name).toContain(`always() && ${MAIN_ONLY} && `);
|
||||
const upsert = issueSteps.find(step => step.name === 'Upsert tracking issue on failure')!;
|
||||
expect(upsert.run).toMatch(/gh issue comment .*--body-file "\$BODY_FILE"/);
|
||||
expect(upsert.run).toMatch(/gh issue create .*--body-file "\$BODY_FILE"/);
|
||||
expect(issueSteps.find(step => step.name === 'Close the tracking issue on a green run')!.run).toContain('gh issue close');
|
||||
});
|
||||
|
||||
test(`${name}: every ref writes the report to the step summary and an artifact`, () => {
|
||||
const steps = parse(name).jobs.report!.steps!;
|
||||
const write = steps.find(step => /^Write the (census|marathon) report$/.test(step.name ?? ''))!;
|
||||
const upload = steps.find(step => /^Upload the (census|marathon) report$/.test(step.name ?? ''))!;
|
||||
const upsert = steps.find(step => step.name === 'Upsert tracking issue on failure')!;
|
||||
expect(write.if).not.toContain('github.ref');
|
||||
expect(upload.if).toBe(write.if);
|
||||
expect(upsert.if).toBe(write.if!.replace('always() && ', `always() && ${MAIN_ONLY} && `));
|
||||
expect(write.run).toContain('>> "$GITHUB_STEP_SUMMARY"');
|
||||
expect(write.run).toContain('docs/evals/census-red.md');
|
||||
expect(write.run).toContain('./scripts/lib/published-text');
|
||||
const reportFile = /> (\/tmp\/[\w-]+\.md)\n/.exec(write.run!)![1]!;
|
||||
expect(upload.uses).toStartWith('actions/upload-artifact@');
|
||||
expect(upload.with!.path).toBe(reportFile);
|
||||
expect(upload.with!.name).toMatch(/^(census|marathon)-report-a\$\{\{ github\.run_attempt \}\}$/);
|
||||
expect(upsert.run).toContain(`BODY_FILE=${reportFile}`);
|
||||
});
|
||||
}
|
||||
|
||||
test('evals-periodic.yml: every pass-rates read names its branch explicitly', () => {
|
||||
const runs = parse('evals-periodic.yml').jobs.report!.steps!.map(step => step.run ?? '').join('\n');
|
||||
const calls = [...runs.matchAll(/^\s*bun run eval:pass-rates [^\n>]*/gm)].map(m => m[0].trim());
|
||||
expect(calls.length).toBe(3);
|
||||
for (const call of calls) expect(call).toContain('--runs 10 --branch "$GITHUB_REF_NAME"');
|
||||
expect(calls.some(call => call.includes('--gate'))).toBe(true);
|
||||
expect(calls.some(call => call.includes('--reds'))).toBe(true);
|
||||
expect(calls.some(call => call.includes('--headroom'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
/** Run a workflow step's script with its expressions and /tmp paths bound to a scratch dir. */
|
||||
function runStep(workflow: string, stepName: string, files: Record<string, string>, expressions: Record<string, string>) {
|
||||
const step = parse(workflow).jobs.report!.steps!.find(s => s.name === stepName)!;
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tracking-issue-'));
|
||||
const work = path.join(dir, 'tmp');
|
||||
const bin = path.join(dir, 'bin');
|
||||
fs.mkdirSync(bin);
|
||||
for (const [rel, body] of Object.entries(files)) {
|
||||
fs.mkdirSync(path.dirname(path.join(work, rel)), { recursive: true });
|
||||
fs.writeFileSync(path.join(work, rel), body);
|
||||
}
|
||||
const log = path.join(dir, 'pass-rates-calls.log');
|
||||
fs.writeFileSync(path.join(bin, 'bun'), [
|
||||
'#!/bin/sh',
|
||||
'if [ "$1" = run ] && [ "$2" = eval:pass-rates ]; then',
|
||||
` echo "$*" >> '${log}'`,
|
||||
` case " $* " in *" --reds "*) cat '${work}/fake-reds.txt';; *" --headroom "*) cat '${work}/fake-headroom.txt';; esac`,
|
||||
' exit 0',
|
||||
'fi',
|
||||
`exec '${process.execPath}' "$@"`,
|
||||
].join('\n'), { mode: 0o755 });
|
||||
const script = step.run!
|
||||
.replace(/\$\{\{ ([^}]+) \}\}/g, (_, expr: string) => expressions[expr.trim()] ?? `<${expr.trim()}>`)
|
||||
.replaceAll('/tmp/', `${work}/`);
|
||||
const summary = path.join(dir, 'step-summary.md');
|
||||
const result = spawnSync('bash', ['-c', script], {
|
||||
cwd: ROOT, encoding: 'utf8', timeout: 60_000,
|
||||
env: { PATH: `${bin}:${process.env.PATH}`, HOME: dir, GITHUB_STEP_SUMMARY: summary, GITHUB_REF_NAME: 'feature/x',
|
||||
GITHUB_SHA: 'abc1234', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_SERVER_URL: 'https://github.com',
|
||||
GITHUB_REPOSITORY: 'garrytan/gstack', GITHUB_RUN_ID: '42', REDISPATCH: 'false', REDISPATCH_OF: '' },
|
||||
});
|
||||
const read = (file: string) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : '');
|
||||
const reports = Object.fromEntries(fs.readdirSync(work).filter(name => name.endsWith('.md')).map(name => [name, read(path.join(work, name))]));
|
||||
const out = { status: result.status, stderr: result.stderr, summary: read(summary), calls: read(log), body: (name: string) => reports[name] ?? '' };
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
return out;
|
||||
}
|
||||
|
||||
const PERIODIC_EXPRESSIONS = {
|
||||
'steps.reconcile.outputs.exit': '1', 'steps.gate-reconcile.outputs.exit': '0', 'steps.pass-rates.outputs.exit': '1',
|
||||
'needs.eval-slices.result': 'failure', 'needs.eval-codex-slices.result': 'success', 'needs.gate-census.result': 'success',
|
||||
};
|
||||
|
||||
describe('the periodic census report', () => {
|
||||
const passRates = [
|
||||
'scope: garrytan/gstack evals-periodic.yml on feature/x + main, last 10 completed run(s) per branch: 1, 2',
|
||||
'pass-rates: policy v1, 40 post-policy trial(s), 0 pre-policy (display only)',
|
||||
' label kind tier current series pre-policy manual case',
|
||||
' ACTION rule gate 9/10 [59.6%–98.2%] - 0 noisy-case',
|
||||
'ACTION REQUIRED (3):',
|
||||
' [drift] noisy-case passes 9/10 (below 95% over >= 10 trials): fix it, or propose a CASE_QUARANTINE entry with a written diagnosis',
|
||||
' [headroom] slow-case session sdk:slow-case#1: max 520s of 600s (87%) above the 85% cap; cut work, never raise the budget',
|
||||
' [drift] mention @attacker ``` </details><img src=x onerror=alert(1)> ghp_' + 'a'.repeat(36),
|
||||
'free-suite flaky-passes (/home/x/.gstack/flake-ledger.jsonl):',
|
||||
' 2x test/unrelated.test.ts',
|
||||
].join('\n');
|
||||
const reds = [
|
||||
'scope: garrytan/gstack evals-periodic.yml on feature/x + main, last 10 completed run(s) per branch: 1, 2',
|
||||
'reds: verdict reds per census lane by failure class and machine cause',
|
||||
' 1 periodic/full census: 1 red of 146 verdicts — class: assertion 1; cause: provider_stall 1',
|
||||
' ✗ noisy-case FAIL t1 assertion/provider_stall — Error: expect(received).toBe(expected)',
|
||||
'pooled: 1 red of 146 verdicts = 0.7% per verdict [95% 0.1%–3.8%]',
|
||||
'all-green probability (approximation, assumes independent verdicts): 81.2% for the latest census\'s 146 verdicts [55%–95% with the pooled rate\'s 95% interval as prior]; formula Π(1 - p_i) over its cases; rerun: bun run eval:pass-rates --reds',
|
||||
].join('\n');
|
||||
const headroom = [
|
||||
'scope: garrytan/gstack evals-periodic.yml on feature/x + main',
|
||||
'headroom: slowest armed session per case (warn > 75%, alarm > 85%; insufficient below 3 samples; censored = timed out)',
|
||||
' ALARM 520s/600s (87%) 3 540s slow-case',
|
||||
].join('\n');
|
||||
const files = {
|
||||
'pass-rates.txt': passRates, 'fake-reds.txt': reds, 'fake-headroom.txt': headroom,
|
||||
'paid-report/report-summary.md': '## Periodic summary\n✗ noisy-case', 'report.txt': 'reconcile tail',
|
||||
};
|
||||
|
||||
test('carries the full alarm lines, red ledger, headroom and all-green estimate, sanitized, for an explicit branch', () => {
|
||||
const run = runStep('evals-periodic.yml', 'Write the census report', files, PERIODIC_EXPRESSIONS);
|
||||
expect(run.stderr).toBe('');
|
||||
expect(run.status).toBe(0);
|
||||
const body = run.body('census-report.md');
|
||||
expect(run.summary).toBe(body);
|
||||
expect(body).toStartWith('Periodic census report: branch feature/x @ abc1234, event workflow_dispatch, run https://github.com/garrytan/gstack/actions/runs/42');
|
||||
expect(body).toContain('triage with https://github.com/garrytan/gstack/blob/abc1234/docs/evals/census-red.md');
|
||||
expect(body).toContain('- periodic reconciliation exit: 1 (slices job: failure, Codex slices job: success)');
|
||||
expect(body).toContain('- all-green probability (approximation, assumes independent verdicts): 81.2% for the latest census');
|
||||
expect(body).toContain('## Periodic summary');
|
||||
// Every alarm line, not just the header; the flaky-pass block is not an alarm.
|
||||
expect(body).toContain('ACTION REQUIRED (3):');
|
||||
expect(body).toContain('[drift] noisy-case passes 9/10 (below 95% over >= 10 trials): fix it');
|
||||
expect(body).toContain('[headroom] slow-case session sdk:slow-case#1: max 520s of 600s (87%) above the 85% cap');
|
||||
expect(body).not.toContain('test/unrelated.test.ts');
|
||||
expect(body).toContain('✗ noisy-case FAIL t1 assertion/provider_stall — Error: expect(received).toBe(expected)');
|
||||
expect(body).toContain('ALARM 520s/600s (87%)');
|
||||
// Hostile alarm text stays literal inside a fence it cannot close; mentions and tokens are neutralized.
|
||||
const hostile = body.split('\n').findIndex(line => line.includes('mention @'));
|
||||
const fence = body.split('\n').slice(0, hostile).reverse().find(line => /^`{3,}$/.test(line))!;
|
||||
expect(fence.length).toBeGreaterThan(3);
|
||||
expect(body).toContain('@\u200battacker');
|
||||
expect(body).not.toContain('ghp_' + 'a'.repeat(36));
|
||||
expect(run.calls.trim().split('\n')).toEqual([
|
||||
'run eval:pass-rates --reds --runs 10 --branch feature/x',
|
||||
'run eval:pass-rates --headroom --runs 10 --branch feature/x',
|
||||
]);
|
||||
});
|
||||
|
||||
test('missing history becomes a visible line, never a failed step', () => {
|
||||
const run = runStep('evals-periodic.yml', 'Write the census report', { 'fake-reds.txt': '', 'fake-headroom.txt': '' }, PERIODIC_EXPRESSIONS);
|
||||
expect(run.status).toBe(0);
|
||||
const body = run.body('census-report.md');
|
||||
expect(body).toContain('all-green probability: unavailable (see the red ledger)');
|
||||
expect(body).toContain('(no pass-rate alarms)');
|
||||
expect(body).toContain('(no periodic report summary)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('the marathon report', () => {
|
||||
test('fences the sanitized reconciliation output and links the guide', () => {
|
||||
const run = runStep('evals-marathon.yml', 'Write the marathon report', { 'report.txt': 'red slice\nping @someone ```' },
|
||||
{ 'steps.reconcile.outputs.exit': '1', 'needs.eval-slices.result': 'success' });
|
||||
expect(run.status).toBe(0);
|
||||
const body = run.body('marathon-report.md');
|
||||
expect(run.summary).toBe(body);
|
||||
expect(body).toStartWith('Marathon report (non-blocking lane): branch feature/x @ abc1234, event workflow_dispatch');
|
||||
expect(body).toContain('docs/evals/census-red.md');
|
||||
expect(body).toContain('- reconciliation exit: 1');
|
||||
expect(body).toContain('````\nred slice\nping @\u200bsomeone ```\n````');
|
||||
});
|
||||
});
|
||||
@@ -377,11 +377,11 @@ describe('panel verdict surfaces (eval reliability policy)', () => {
|
||||
const gate = report.steps.find(step => step.id === 'pass-rates')!;
|
||||
expect(gate.run).toContain('bun run eval:pass-rates --gate --runs 10');
|
||||
expect(gate.if).toBe('always()');
|
||||
for (const name of ['Upsert tracking issue on failure', 'Fail the workflow when reconciliation failed']) {
|
||||
for (const name of ['Write the census report', 'Upsert tracking issue on failure', 'Fail the workflow when reconciliation failed']) {
|
||||
expect(report.steps.find(step => step.name === name)!.if).toContain("steps.pass-rates.outputs.exit != '0'");
|
||||
}
|
||||
const upsert = report.steps.find(step => step.name === 'Upsert tracking issue on failure')!;
|
||||
expect(upsert.run).toContain('report-summary.md');
|
||||
// The report body is written once on every ref; the main-only upsert posts it (test/evals-tracking-issue.test.ts).
|
||||
expect(report.steps.find(step => step.name === 'Write the census report')!.run).toContain('report-summary.md');
|
||||
expect(report.steps.find(step => step.name === 'Close the tracking issue on a green run')!.run).toContain('gh issue close');
|
||||
expect(report.steps.filter(step => step.with?.name?.startsWith('trial-outcomes-')).length).toBe(2);
|
||||
const redispatch = jobs.redispatch!;
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Oracles for the live Codex multi-block case (E6,
|
||||
* test/codex-e2e-multiblock-live.test.ts). Codex runs every shell block in a
|
||||
* fresh shell, so a gstack skill works there only when each later block
|
||||
* resolves the install on its own (the env-var host prelude). The case
|
||||
* installs gstack with `./setup --host codex` into a fresh CODEX_HOME, seeds
|
||||
* one learning, and runs /learn's "Show recent" block through `codex exec`.
|
||||
* Pure functions: the free test replays synthetic `exec --json` events.
|
||||
*/
|
||||
|
||||
export const MULTIBLOCK_SENTINEL_KEY = 'e6-multiblock-sentinel';
|
||||
|
||||
interface CommandEvent { command: string; output: string; exitCode: number | null }
|
||||
|
||||
function completedCommands(rawLines: readonly string[]): CommandEvent[] {
|
||||
const commands: CommandEvent[] = [];
|
||||
for (const line of rawLines) {
|
||||
try {
|
||||
const event = JSON.parse(line);
|
||||
if (event?.type !== 'item.completed' || event.item?.type !== 'command_execution') continue;
|
||||
commands.push({ command: String(event.item.command ?? ''), output: String(event.item.aggregated_output ?? ''),
|
||||
exitCode: typeof event.item.exit_code === 'number' ? event.item.exit_code : null });
|
||||
} catch { /* not an event line */ }
|
||||
}
|
||||
return commands;
|
||||
}
|
||||
|
||||
/**
|
||||
* Problems with a live run: any block that could not find the install, a
|
||||
* gstack helper path that did not resolve, or no later block (the skill's
|
||||
* learnings-search block, which only the prelude can root) that ran the
|
||||
* installed helper and printed the seeded learning.
|
||||
*/
|
||||
export function multiblockProblems(rawLines: readonly string[], sentinel = MULTIBLOCK_SENTINEL_KEY): string[] {
|
||||
const commands = completedCommands(rawLines);
|
||||
const problems: string[] = [];
|
||||
for (const { command, output } of commands) {
|
||||
if (output.includes('gstack: no install found')) problems.push(`a block could not find the install: ${output.trim().slice(0, 200)}`);
|
||||
const unresolved = /\S*\/bin\/gstack-[\w-]+: (?:No such file or directory|command not found|not found)/.exec(output);
|
||||
if (unresolved) problems.push(`a gstack helper path did not resolve: ${unresolved[0]} (in: ${command.slice(0, 120)})`);
|
||||
}
|
||||
const search = commands.filter(c => c.command.includes('gstack-learnings-search'));
|
||||
if (search.length === 0) problems.push('no block ran gstack-learnings-search (the skill\'s "Show recent" block)');
|
||||
else if (!search.some(c => c.exitCode === 0 && c.output.includes(sentinel) && !/GSTACK_ROOT=["']?\//.test(c.command))) {
|
||||
problems.push(`no learnings-search block rooted by its own prelude printed the seeded learning ${sentinel}: `
|
||||
+ search.map(c => `exit ${c.exitCode}, ${c.output.includes(sentinel) ? 'sentinel present' : 'sentinel absent'}`
|
||||
+ `${/GSTACK_ROOT=["']?\//.test(c.command) ? ', GSTACK_ROOT set by hand' : ''}`).join('; '));
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
/**
|
||||
* The `gstack` router entry from `codex debug prompt-input` output (a JSON
|
||||
* array of input items): the skill path it names, resolved through the skill
|
||||
* roots table, or null when Codex does not list the router.
|
||||
*/
|
||||
export function routerPathFromPromptInput(promptInput: string): string | null {
|
||||
let text = promptInput;
|
||||
try {
|
||||
const items = JSON.parse(promptInput.slice(promptInput.indexOf('[')));
|
||||
text = (Array.isArray(items) ? items : []).flatMap((item: { content?: Array<{ text?: string }> }) => item.content ?? [])
|
||||
.map(part => part.text ?? '').join('\n');
|
||||
} catch { /* scan the raw text */ }
|
||||
const entry = /^- gstack: .*\(file: (r\d+)\/gstack\/SKILL\.md\)$/m.exec(text);
|
||||
if (!entry) return null;
|
||||
const root = new RegExp(`^- \`${entry[1]}\` = \`([^\`]+)\`$`, 'm').exec(text);
|
||||
return root ? `${root[1]}/gstack/SKILL.md` : null;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* Design model smoke (E5, periodic): runs design/scripts/live-model-check.ts,
|
||||
* which sends the design binary's real request bodies with the DEFAULT
|
||||
* models (overrides are ignored), so a retired or unentitled default fails
|
||||
* here before users hit it. Without OPENAI_API_KEY the case is a Bun skip:
|
||||
* the report lists it as SKIPPED with zero coverage credit, never a pass.
|
||||
* With a key, exit 0 passes; exit 2 (the script saw no key) and every other
|
||||
* exit fail with the script's output attached.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { e2eTierEnabled } from './helpers/e2e-gate';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const CASE = 'design-model-smoke';
|
||||
const KEY_PRESENT = !!process.env.OPENAI_API_KEY?.trim();
|
||||
const CHECK_MS = 400_000;
|
||||
if (e2eTierEnabled('periodic') && !KEY_PRESENT) {
|
||||
process.stderr.write(`\n${CASE}: SKIPPED: OPENAI_API_KEY is not set; the design model defaults were not checked (no coverage)\n`);
|
||||
}
|
||||
|
||||
describe.skipIf(!e2eTierEnabled('periodic') || !KEY_PRESENT)('design live model defaults (E5)', () => {
|
||||
test(CASE, () => {
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'design-model-smoke-'));
|
||||
try {
|
||||
const run = spawnSync(process.execPath, ['run', 'design/scripts/live-model-check.ts'], {
|
||||
cwd: ROOT, encoding: 'utf8', timeout: CHECK_MS,
|
||||
env: { ...process.env, HOME: home, GSTACK_HOME: path.join(home, '.gstack') },
|
||||
});
|
||||
const output = `${run.stdout ?? ''}${run.stderr ?? ''}`.trim();
|
||||
console.log(output);
|
||||
if (run.status === 2) throw new Error(`live-model-check saw no OPENAI_API_KEY although the test had one; nothing was checked:\n${output}`);
|
||||
expect(run.status, output).toBe(0);
|
||||
} finally {
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
}, CHECK_MS + 10_000);
|
||||
});
|
||||
Reference in new issue
Block a user