From 934221b50856bd3ba046fa3048bcc32159c7896c Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 24 Oct 2026 22:30:11 -0700 Subject: [PATCH] v1.91.195.0 fix(skills): /health GBrain score in code, office-hours diff base, ios-qa demo rule, plan-design model name (#3347 #3350 #3351 #3352) (#3373) * fix(plan-design-review): say the design check uses OpenAI vision, not GPT-4o (#3350) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(health): compute the GBrain score in code so the rubric and formula can't disagree (#3351) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ios-qa): demo mode states its UI-only rule without overriding the skill's other rules (#3352) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(office-hours): diff against origin/HEAD, not a hard-coded origin/main with stderr hidden (#3347) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): #3347 #3350 #3351 #3352 skill template fixes under [Unreleased] Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 18 ++++++++++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- health/SKILL.md | 38 ++++++++++++++++--------- health/SKILL.md.tmpl | 38 ++++++++++++++++--------- ios-qa/SKILL.md | 8 +++--- ios-qa/SKILL.md.tmpl | 8 +++--- office-hours/SKILL.md | 2 +- office-hours/SKILL.md.tmpl | 2 +- package.json | 2 +- plan-design-review/SKILL.md | 2 +- plan-design-review/SKILL.md.tmpl | 2 +- test/helpers/safety-rules.ts | 4 +-- test/qa-health-rubric.test.ts | 48 ++++++++++++++++++++++++++++++++ 14 files changed, 133 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06ad4a089..440611a9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## [1.91.195.0] - 2026-10-25 + +**Four skill prompts stop contradicting themselves or their code.** dgrant's issues #3347, #3350, #3351 and #3352 each found a skill telling the model something wrong or ambiguous: /office-hours read a master-based repo as "no changes", /plan-design-review named a vision model the design binary no longer uses, /health gave two different GBrain scores for the same readings, and /ios-qa's demo mode claimed to override every other rule in the skill. + +### Fixed +- **`office-hours/SKILL.md.tmpl`** (#3347): Phase 1 runs `git diff origin/HEAD --stat`, the remote's default branch, instead of `git diff origin/main --stat 2>/dev/null`. On a clone whose default branch is `master`, the old command exited 128 with its error hidden, so the model read the empty output as "no changes"; the new one shows the diff, and a repo without `origin/HEAD` now shows its error instead of passing for "no diff". The line is the same length, so office-hours stays at its pinned size on every host (the shared `{{BASE_BRANCH_DETECT}}` block would have added 1,727 chars, past the size-guard pins and the parity skeleton cap). +- **`plan-design-review/SKILL.md.tmpl`** (#3350): the design binary's `check` command is described as "OpenAI vision" instead of "GPT-4o vision", so the prose no longer pins a model. The binary defaults to gpt-5.5 (`design/src/models.ts`), `GSTACK_DESIGN_MODEL` overrides it, and gpt-4o cannot drive gpt-image-2 at all. The swap is the same length, so the pinned host sizes don't move. +- **`health/SKILL.md.tmpl`** (#3351): the GBrain score is computed by a `bun -e` snippet from the doctor component (10 for `ok`, 7 for `warnings`, else 0), the queue length and the hours since the last push, like the composite already was. The snippet takes only numeric `` inputs, the placeholder form the generated-bash lint already allows. The rubric row's 10/7/4 bands, which disagreed with the weighted formula (doctor ok, queue 50, pushed 2h: 7 by the rubric, 9.1 by the formula) and overlapped (a queue of 3 matched both 10 and 7), now point at that computation. The bands are half-open (<10, <100 lines; <24, <72 hours), and with sync off the doctor component carries the whole score. History records `gbrain` with one decimal. +- **`ios-qa/SKILL.md.tmpl`** (#3352): "DEMO MODE OVERRIDES ALL OTHER RULES" becomes the narrow rule it meant: in demo mode drive every action through visible UI rather than `POST /state/*` writes, and every other rule in the skill (the device lock, no replay of a lost tap, never hand-editing DebugBridge) still applies. + +### Tests +- `test/qa-health-rubric.test.ts` checks that the GBrain rubric row defers to the computation, then runs the template's own snippet: the issue's two inputs score 9.1 and 8.5, the band edges (9/10/99/100 lines, 23.9/24/71.9/72 hours) land on the right side, a missing doctor reading counts as 0, and sync off gives 10, 7 and 0. All four new tests fail on the old template. + +- `test/helpers/safety-rules.ts`: the `ios-qa-demo-override` row's anchor moves from `/demo mode overrides/i` to `/drive every action through visible UI/i`, and its hash to the reworded paragraph, in the #3352 commit as the registry requires. `test/safety-rule-registry.test.ts` passes, including its unique-anchor and rule-removed-arm checks. + +### Gate integrity +- One gate edit to review: the safety-rule registry row for `safety-ios-demo-ui-only` now anchors on the reworded paragraph. Verdict: not weaker. The paragraph still requires visible UI (`/tap`, `/swipe`, `/type`) over `POST /state/*` writes, and the only text removed is the "overrides all other rules" claim. The case's rule-removed arm deletes the same paragraph as before. No threshold, budget, timeout, trial count or case kind changed. + ## [1.91.194.0] - 2026-10-25 **A read-only repo can now opt in to Memorable recall, one remote at a time, and a corrupt policy store no longer turns the recall veto off.** #3359 (exGeni) asked for a way to keep a repo out of the brain while still letting its prompts reach Memorable: today the recall hook skips every `deny` and `read-only` repo, and `read-only` is also the tier people pick just to stop code ingest. Garry approved a per-remote flag on 2026-10-24: off by default, and a `deny` always wins. While building the negative controls we found that the hook's policy lookup went through `get`, which quarantines a corrupt store and answers "unset", so a truncated `gbrain-repo-policy.json` let recall run in every repo, read-only and deny included, and reset the store on the way. diff --git a/VERSION b/VERSION index 49986b594..419d704da 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.194.0 +1.91.195.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 7cc162640..e692c5f6d 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.194.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.195.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/health/SKILL.md b/health/SKILL.md index 8eb500741..1078ec6d5 100644 --- a/health/SKILL.md +++ b/health/SKILL.md @@ -500,7 +500,7 @@ Score each category on a 0-10 scale using this rubric: | Tests | 28% | All pass (exit 0) | >95% pass | >80% pass | <=80% pass | | Dead code | 13% | Clean (exit 0) | <5 unused exports | <20 unused | >=20 unused | | Shell lint | 9% | Clean (exit 0) | <5 issues | >=5 issues | N/A (skip) | -| GBrain | 10% | doctor=ok, queue<10, pushed <24h | doctor=warnings OR queue<100 OR pushed <72h | doctor broken OR queue>=100 OR pushed >=72h | N/A (gbrain not installed) | +| GBrain | 10% | Computed in code: see **GBrain sub-score computation** below | — | — | N/A (gbrain not installed) | **Parsing tool output for counts:** Use the complete captured output, not the displayed tail. A zero match count cannot @@ -534,17 +534,28 @@ a numeric composite, and skip numeric history persistence and trend calculation. **GBrain sub-score computation:** -``` -doctor_component: 10 if `gbrain doctor --json | jq -r .status` == "ok"; - 7 if "warnings"; 0 otherwise (or command times out after 5s). -queue_component: 10 if ~/.gstack/.brain-queue.jsonl has <10 lines; - 7 if 10-100; 0 if >=100 (suggests secret-scan rejections - piling up). N/A if artifacts_sync_mode == off. -push_component: 10 if (now - mtime of ~/.gstack/.brain-last-push) < 24h; - 7 if <72h; 0 if >=72h. N/A if artifacts_sync_mode == off. -gbrain_score = 0.5 * doctor_component + 0.3 * queue_component + 0.2 * push_component - (redistribute 0.3 + 0.2 into doctor when sync_mode is off: - gbrain_score = doctor_component in that case) +Compute the GBrain score in code, not by hand, and pass the printed +`GBRAIN_SCORE` as `gbrain` to the composite above. Gather three readings: + +- **doctor:** 10 when `gbrain doctor --json` reports `status` `ok`, 7 for + `warnings`, 0 for anything else or when the call times out after 5s. +- **queue:** the line count of `~/.gstack/.brain-queue.jsonl` (0 when absent). + A long queue suggests secret-scan rejections piling up. +- **pushHours:** hours since the mtime of `~/.gstack/.brain-last-push`. + +Pass `null` for both queue and pushHours when `artifacts_sync_mode` is `off`; +the doctor component then carries the whole score: + +```bash +bun --no-env-file --config=/dev/null -e ' +const doctor = ; +const queue = ; +const pushHours = ; +const d = doctor ?? 0; +const band = (v, ten, seven) => (v < ten ? 10 : v < seven ? 7 : 0); +const score = queue === null || pushHours === null ? d : 0.5 * d + 0.3 * band(queue, 10, 100) + 0.2 * band(pushHours, 24, 72); +console.log("GBRAIN_SCORE: " + score.toFixed(1)); +' ``` The `gbrain doctor --json` call MUST be wrapped in `timeout 5s` so a hung @@ -625,7 +636,8 @@ Fields: - `ts` -- ISO 8601 timestamp - `branch` -- current git branch - `score` -- composite score (one decimal) -- `typecheck`, `lint`, `test`, `deadcode`, `shell`, `gbrain` -- individual category scores (integer 0-10) +- `typecheck`, `lint`, `test`, `deadcode`, `shell` -- individual category scores (integer 0-10) +- `gbrain` -- the printed `GBRAIN_SCORE` (0-10, one decimal) - `duration_s` -- total time for all tools in seconds If a category was skipped, set its value to `null`. Treat a history entry diff --git a/health/SKILL.md.tmpl b/health/SKILL.md.tmpl index c16434508..47f83551e 100644 --- a/health/SKILL.md.tmpl +++ b/health/SKILL.md.tmpl @@ -177,7 +177,7 @@ Score each category on a 0-10 scale using this rubric: | Tests | 28% | All pass (exit 0) | >95% pass | >80% pass | <=80% pass | | Dead code | 13% | Clean (exit 0) | <5 unused exports | <20 unused | >=20 unused | | Shell lint | 9% | Clean (exit 0) | <5 issues | >=5 issues | N/A (skip) | -| GBrain | 10% | doctor=ok, queue<10, pushed <24h | doctor=warnings OR queue<100 OR pushed <72h | doctor broken OR queue>=100 OR pushed >=72h | N/A (gbrain not installed) | +| GBrain | 10% | Computed in code: see **GBrain sub-score computation** below | — | — | N/A (gbrain not installed) | **Parsing tool output for counts:** Use the complete captured output, not the displayed tail. A zero match count cannot @@ -211,17 +211,28 @@ a numeric composite, and skip numeric history persistence and trend calculation. **GBrain sub-score computation:** -``` -doctor_component: 10 if `gbrain doctor --json | jq -r .status` == "ok"; - 7 if "warnings"; 0 otherwise (or command times out after 5s). -queue_component: 10 if ~/.gstack/.brain-queue.jsonl has <10 lines; - 7 if 10-100; 0 if >=100 (suggests secret-scan rejections - piling up). N/A if artifacts_sync_mode == off. -push_component: 10 if (now - mtime of ~/.gstack/.brain-last-push) < 24h; - 7 if <72h; 0 if >=72h. N/A if artifacts_sync_mode == off. -gbrain_score = 0.5 * doctor_component + 0.3 * queue_component + 0.2 * push_component - (redistribute 0.3 + 0.2 into doctor when sync_mode is off: - gbrain_score = doctor_component in that case) +Compute the GBrain score in code, not by hand, and pass the printed +`GBRAIN_SCORE` as `gbrain` to the composite above. Gather three readings: + +- **doctor:** 10 when `gbrain doctor --json` reports `status` `ok`, 7 for + `warnings`, 0 for anything else or when the call times out after 5s. +- **queue:** the line count of `~/.gstack/.brain-queue.jsonl` (0 when absent). + A long queue suggests secret-scan rejections piling up. +- **pushHours:** hours since the mtime of `~/.gstack/.brain-last-push`. + +Pass `null` for both queue and pushHours when `artifacts_sync_mode` is `off`; +the doctor component then carries the whole score: + +```bash +bun --no-env-file --config=/dev/null -e ' +const doctor = ; +const queue = ; +const pushHours = ; +const d = doctor ?? 0; +const band = (v, ten, seven) => (v < ten ? 10 : v < seven ? 7 : 0); +const score = queue === null || pushHours === null ? d : 0.5 * d + 0.3 * band(queue, 10, 100) + 0.2 * band(pushHours, 24, 72); +console.log("GBRAIN_SCORE: " + score.toFixed(1)); +' ``` The `gbrain doctor --json` call MUST be wrapped in `timeout 5s` so a hung @@ -301,7 +312,8 @@ Fields: - `ts` -- ISO 8601 timestamp - `branch` -- current git branch - `score` -- composite score (one decimal) -- `typecheck`, `lint`, `test`, `deadcode`, `shell`, `gbrain` -- individual category scores (integer 0-10) +- `typecheck`, `lint`, `test`, `deadcode`, `shell` -- individual category scores (integer 0-10) +- `gbrain` -- the printed `GBRAIN_SCORE` (0-10, one decimal) - `duration_s` -- total time for all tools in seconds If a category was skipped, set its value to `null`. Treat a history entry diff --git a/ios-qa/SKILL.md b/ios-qa/SKILL.md index dfd43a37a..c985f5975 100644 --- a/ios-qa/SKILL.md +++ b/ios-qa/SKILL.md @@ -593,10 +593,10 @@ If the user says "demo", "demo mode", "show me", or "I want to see it working", run in **DEMO MODE**. This changes how the agent interacts with the app: -**DEMO MODE OVERRIDES ALL OTHER RULES.** When demo mode is active, the -agent MUST drive every action through visible UI (`/tap`, `/swipe`, `/type`) -and NEVER use `POST /state/*` writes to skip steps. Viewers see the agent -type every key, tap every button. The on-device DebugOverlay attribution +In demo mode, drive every action through visible UI (`/tap`, `/swipe`, +`/type`) rather than `POST /state/*` writes, because viewers watch each +step: they see the agent type every key and tap every button. All other +rules in this skill still apply. The on-device DebugOverlay attribution chip shows "Driven by Claude Code (demo)" or the remote agent identity. In demo mode, the screencap rate is bumped to 4fps so the recording feels diff --git a/ios-qa/SKILL.md.tmpl b/ios-qa/SKILL.md.tmpl index 4c7b4a2c5..3c957a051 100644 --- a/ios-qa/SKILL.md.tmpl +++ b/ios-qa/SKILL.md.tmpl @@ -249,10 +249,10 @@ If the user says "demo", "demo mode", "show me", or "I want to see it working", run in **DEMO MODE**. This changes how the agent interacts with the app: -**DEMO MODE OVERRIDES ALL OTHER RULES.** When demo mode is active, the -agent MUST drive every action through visible UI (`/tap`, `/swipe`, `/type`) -and NEVER use `POST /state/*` writes to skip steps. Viewers see the agent -type every key, tap every button. The on-device DebugOverlay attribution +In demo mode, drive every action through visible UI (`/tap`, `/swipe`, +`/type`) rather than `POST /state/*` writes, because viewers watch each +step: they see the agent type every key and tap every button. All other +rules in this skill still apply. The on-device DebugOverlay attribution chip shows "Driven by Claude Code (demo)" or the remote agent identity. In demo mode, the screencap rate is bumped to 4fps so the recording feels diff --git a/office-hours/SKILL.md b/office-hours/SKILL.md index c22b3db49..7e925ebf7 100644 --- a/office-hours/SKILL.md +++ b/office-hours/SKILL.md @@ -509,7 +509,7 @@ SLUG=$(~/.claude/skills/gstack/bin/gstack-slug --get SLUG 2>/dev/null) ``` 1. Read `CLAUDE.md`, `TODOS.md` (if they exist). -2. Run `git log --oneline -30` and `git diff origin/main --stat 2>/dev/null` to understand recent context. +2. Run `git log --oneline -30` and `git diff origin/HEAD --stat` (default branch) to grasp recent context. 3. Use Grep/Glob to map the codebase areas most relevant to the user's request. 4. **List existing design docs for this project:** ```bash diff --git a/office-hours/SKILL.md.tmpl b/office-hours/SKILL.md.tmpl index 868a7fe73..881daa3f9 100644 --- a/office-hours/SKILL.md.tmpl +++ b/office-hours/SKILL.md.tmpl @@ -81,7 +81,7 @@ Understand the project and the area the user wants to change. ``` 1. Read `CLAUDE.md`, `TODOS.md` (if they exist). -2. Run `git log --oneline -30` and `git diff origin/main --stat 2>/dev/null` to understand recent context. +2. Run `git log --oneline -30` and `git diff origin/HEAD --stat` (default branch) to grasp recent context. 3. Use Grep/Glob to map the codebase areas most relevant to the user's request. 4. **List existing design docs for this project:** ```bash diff --git a/package.json b/package.json index 1b04d4a8e..4dc3fb059 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.194", + "version": "1.91.195", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/plan-design-review/SKILL.md b/plan-design-review/SKILL.md index 8ab0c3fd8..049e21ddc 100644 --- a/plan-design-review/SKILL.md +++ b/plan-design-review/SKILL.md @@ -467,7 +467,7 @@ only under the Step 0.5 conditions (no UI scope, designer unavailable, or the us Commands: `generate` (single mockup), `variants` (multiple directions), `compare` (side-by-side review board), `iterate` (refine with feedback), `check` (cross-model -quality gate via GPT-4o vision), `evolve` (improve from screenshot). +quality gate via OpenAI vision), `evolve` (improve from screenshot). Setup is handled by the DESIGN SETUP section below. If `DESIGN_READY` is printed, the designer is available and you should use it. diff --git a/plan-design-review/SKILL.md.tmpl b/plan-design-review/SKILL.md.tmpl index c30f8d0a5..923bbeeed 100644 --- a/plan-design-review/SKILL.md.tmpl +++ b/plan-design-review/SKILL.md.tmpl @@ -85,7 +85,7 @@ only under the Step 0.5 conditions (no UI scope, designer unavailable, or the us Commands: `generate` (single mockup), `variants` (multiple directions), `compare` (side-by-side review board), `iterate` (refine with feedback), `check` (cross-model -quality gate via GPT-4o vision), `evolve` (improve from screenshot). +quality gate via OpenAI vision), `evolve` (improve from screenshot). Setup is handled by the DESIGN SETUP section below. If `DESIGN_READY` is printed, the designer is available and you should use it. diff --git a/test/helpers/safety-rules.ts b/test/helpers/safety-rules.ts index 08a3e1b44..4972d0090 100644 --- a/test/helpers/safety-rules.ts +++ b/test/helpers/safety-rules.ts @@ -68,8 +68,8 @@ export const SAFETY_RULES: readonly SafetyRule[] = [ evalId: 'safety-ios-demo-ui-only', source: 'ios-qa/SKILL.md.tmpl', section: '## Demo mode', - match: /demo mode overrides/i, - sha256: '8d42208c888d76af49ebe6e4d58e85537c979f53ecb32fff7e85e95e6859182a', + match: /drive every action through visible UI/i, + sha256: '0ba3230bff9de32f03719daec35c6cef762e15c550264fcffd342dde52291e21', }, ]; diff --git a/test/qa-health-rubric.test.ts b/test/qa-health-rubric.test.ts index 8f8d9dee0..842f97128 100644 --- a/test/qa-health-rubric.test.ts +++ b/test/qa-health-rubric.test.ts @@ -1,4 +1,8 @@ import { describe, expect, test } from 'bun:test'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { bunIsolationArgs } from '../lib/bun-isolation'; import { generateQAMethodology } from '../scripts/resolvers/utility'; import { HOST_PATHS } from '../scripts/resolvers/types'; @@ -31,3 +35,47 @@ describe('QA health rubric scoring contract', () => { expect(rubric).toContain('Round only the final score'); }); }); + +const healthTmpl = fs.readFileSync(path.join(import.meta.dir, '..', 'health', 'SKILL.md.tmpl'), 'utf-8'); +const gbrainSection = healthTmpl.split('**GBrain sub-score computation:**')[1] ?? ''; +const gbrainScript = gbrainSection.match(/```bash\nbun --no-env-file --config=\/dev\/null -e '\n([\s\S]*?)\n'\n```/)?.[1] ?? ''; + +function gbrainScore(doctor: number | null, queue: number | null, pushHours: number | null): string { + const script = gbrainScript + .replace('', String(doctor)) + .replace('', String(queue)) + .replace('', String(pushHours)); + const run = spawnSync(process.execPath, [...bunIsolationArgs(), '-e', script], { encoding: 'utf-8', timeout: 30_000 }); + expect(run.status).toBe(0); + return run.stdout.trim(); +} + +describe('/health GBrain score has one rule, computed in code (#3351)', () => { + test('the rubric row defers to the computation instead of scoring bands', () => { + const row = healthTmpl.split('\n').find(line => line.startsWith('| GBrain |')) ?? ''; + expect(row).toContain('GBrain sub-score computation'); + expect(row).not.toMatch(/queue\s* { + expect(gbrainScript).not.toBe(''); + expect(gbrainScore(10, 50, 2)).toBe('GBRAIN_SCORE: 9.1'); + expect(gbrainScore(7, 3, 2)).toBe('GBRAIN_SCORE: 8.5'); + }); + + test('bands are half-open at 10/100 queue lines and 24/72 hours', () => { + expect(gbrainScore(10, 9, 23.9)).toBe('GBRAIN_SCORE: 10.0'); + expect(gbrainScore(10, 10, 24)).toBe('GBRAIN_SCORE: 8.5'); + expect(gbrainScore(10, 99, 71.9)).toBe('GBRAIN_SCORE: 8.5'); + expect(gbrainScore(10, 100, 72)).toBe('GBRAIN_SCORE: 5.0'); + expect(gbrainScore(0, 100, 72)).toBe('GBRAIN_SCORE: 0.0'); + expect(gbrainScore(null, 0, 1)).toBe('GBRAIN_SCORE: 5.0'); + }); + + test('sync off gives the doctor component the whole weight', () => { + expect(gbrainScore(10, null, null)).toBe('GBRAIN_SCORE: 10.0'); + expect(gbrainScore(7, null, null)).toBe('GBRAIN_SCORE: 7.0'); + expect(gbrainScore(0, null, null)).toBe('GBRAIN_SCORE: 0.0'); + }); +});