From fa898b78c6c05a03c1e548fa421d19faba71d491 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 24 Oct 2026 20:08:32 +0000 Subject: [PATCH] fix(health): compute the GBrain score in code so the rubric and formula can't disagree (#3351) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- health/SKILL.md | 38 +++++++++++++++++---------- health/SKILL.md.tmpl | 38 +++++++++++++++++---------- test/qa-health-rubric.test.ts | 48 +++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 26 deletions(-) diff --git a/health/SKILL.md b/health/SKILL.md index 8eb500741..1078ec6d5 100644 --- a/health/SKILL.md +++ b/health/SKILL.md @@ -500,7 +500,7 @@ Score each category on a 0-10 scale using this rubric: | Tests | 28% | All pass (exit 0) | >95% pass | >80% pass | <=80% pass | | Dead code | 13% | Clean (exit 0) | <5 unused exports | <20 unused | >=20 unused | | Shell lint | 9% | Clean (exit 0) | <5 issues | >=5 issues | N/A (skip) | -| GBrain | 10% | doctor=ok, queue<10, pushed <24h | doctor=warnings OR queue<100 OR pushed <72h | doctor broken OR queue>=100 OR pushed >=72h | N/A (gbrain not installed) | +| GBrain | 10% | Computed in code: see **GBrain sub-score computation** below | — | — | N/A (gbrain not installed) | **Parsing tool output for counts:** Use the complete captured output, not the displayed tail. A zero match count cannot @@ -534,17 +534,28 @@ a numeric composite, and skip numeric history persistence and trend calculation. **GBrain sub-score computation:** -``` -doctor_component: 10 if `gbrain doctor --json | jq -r .status` == "ok"; - 7 if "warnings"; 0 otherwise (or command times out after 5s). -queue_component: 10 if ~/.gstack/.brain-queue.jsonl has <10 lines; - 7 if 10-100; 0 if >=100 (suggests secret-scan rejections - piling up). N/A if artifacts_sync_mode == off. -push_component: 10 if (now - mtime of ~/.gstack/.brain-last-push) < 24h; - 7 if <72h; 0 if >=72h. N/A if artifacts_sync_mode == off. -gbrain_score = 0.5 * doctor_component + 0.3 * queue_component + 0.2 * push_component - (redistribute 0.3 + 0.2 into doctor when sync_mode is off: - gbrain_score = doctor_component in that case) +Compute the GBrain score in code, not by hand, and pass the printed +`GBRAIN_SCORE` as `gbrain` to the composite above. Gather three readings: + +- **doctor:** 10 when `gbrain doctor --json` reports `status` `ok`, 7 for + `warnings`, 0 for anything else or when the call times out after 5s. +- **queue:** the line count of `~/.gstack/.brain-queue.jsonl` (0 when absent). + A long queue suggests secret-scan rejections piling up. +- **pushHours:** hours since the mtime of `~/.gstack/.brain-last-push`. + +Pass `null` for both queue and pushHours when `artifacts_sync_mode` is `off`; +the doctor component then carries the whole score: + +```bash +bun --no-env-file --config=/dev/null -e ' +const doctor = ; +const queue = ; +const pushHours = ; +const d = doctor ?? 0; +const band = (v, ten, seven) => (v < ten ? 10 : v < seven ? 7 : 0); +const score = queue === null || pushHours === null ? d : 0.5 * d + 0.3 * band(queue, 10, 100) + 0.2 * band(pushHours, 24, 72); +console.log("GBRAIN_SCORE: " + score.toFixed(1)); +' ``` The `gbrain doctor --json` call MUST be wrapped in `timeout 5s` so a hung @@ -625,7 +636,8 @@ Fields: - `ts` -- ISO 8601 timestamp - `branch` -- current git branch - `score` -- composite score (one decimal) -- `typecheck`, `lint`, `test`, `deadcode`, `shell`, `gbrain` -- individual category scores (integer 0-10) +- `typecheck`, `lint`, `test`, `deadcode`, `shell` -- individual category scores (integer 0-10) +- `gbrain` -- the printed `GBRAIN_SCORE` (0-10, one decimal) - `duration_s` -- total time for all tools in seconds If a category was skipped, set its value to `null`. Treat a history entry diff --git a/health/SKILL.md.tmpl b/health/SKILL.md.tmpl index c16434508..47f83551e 100644 --- a/health/SKILL.md.tmpl +++ b/health/SKILL.md.tmpl @@ -177,7 +177,7 @@ Score each category on a 0-10 scale using this rubric: | Tests | 28% | All pass (exit 0) | >95% pass | >80% pass | <=80% pass | | Dead code | 13% | Clean (exit 0) | <5 unused exports | <20 unused | >=20 unused | | Shell lint | 9% | Clean (exit 0) | <5 issues | >=5 issues | N/A (skip) | -| GBrain | 10% | doctor=ok, queue<10, pushed <24h | doctor=warnings OR queue<100 OR pushed <72h | doctor broken OR queue>=100 OR pushed >=72h | N/A (gbrain not installed) | +| GBrain | 10% | Computed in code: see **GBrain sub-score computation** below | — | — | N/A (gbrain not installed) | **Parsing tool output for counts:** Use the complete captured output, not the displayed tail. A zero match count cannot @@ -211,17 +211,28 @@ a numeric composite, and skip numeric history persistence and trend calculation. **GBrain sub-score computation:** -``` -doctor_component: 10 if `gbrain doctor --json | jq -r .status` == "ok"; - 7 if "warnings"; 0 otherwise (or command times out after 5s). -queue_component: 10 if ~/.gstack/.brain-queue.jsonl has <10 lines; - 7 if 10-100; 0 if >=100 (suggests secret-scan rejections - piling up). N/A if artifacts_sync_mode == off. -push_component: 10 if (now - mtime of ~/.gstack/.brain-last-push) < 24h; - 7 if <72h; 0 if >=72h. N/A if artifacts_sync_mode == off. -gbrain_score = 0.5 * doctor_component + 0.3 * queue_component + 0.2 * push_component - (redistribute 0.3 + 0.2 into doctor when sync_mode is off: - gbrain_score = doctor_component in that case) +Compute the GBrain score in code, not by hand, and pass the printed +`GBRAIN_SCORE` as `gbrain` to the composite above. Gather three readings: + +- **doctor:** 10 when `gbrain doctor --json` reports `status` `ok`, 7 for + `warnings`, 0 for anything else or when the call times out after 5s. +- **queue:** the line count of `~/.gstack/.brain-queue.jsonl` (0 when absent). + A long queue suggests secret-scan rejections piling up. +- **pushHours:** hours since the mtime of `~/.gstack/.brain-last-push`. + +Pass `null` for both queue and pushHours when `artifacts_sync_mode` is `off`; +the doctor component then carries the whole score: + +```bash +bun --no-env-file --config=/dev/null -e ' +const doctor = ; +const queue = ; +const pushHours = ; +const d = doctor ?? 0; +const band = (v, ten, seven) => (v < ten ? 10 : v < seven ? 7 : 0); +const score = queue === null || pushHours === null ? d : 0.5 * d + 0.3 * band(queue, 10, 100) + 0.2 * band(pushHours, 24, 72); +console.log("GBRAIN_SCORE: " + score.toFixed(1)); +' ``` The `gbrain doctor --json` call MUST be wrapped in `timeout 5s` so a hung @@ -301,7 +312,8 @@ Fields: - `ts` -- ISO 8601 timestamp - `branch` -- current git branch - `score` -- composite score (one decimal) -- `typecheck`, `lint`, `test`, `deadcode`, `shell`, `gbrain` -- individual category scores (integer 0-10) +- `typecheck`, `lint`, `test`, `deadcode`, `shell` -- individual category scores (integer 0-10) +- `gbrain` -- the printed `GBRAIN_SCORE` (0-10, one decimal) - `duration_s` -- total time for all tools in seconds If a category was skipped, set its value to `null`. Treat a history entry diff --git a/test/qa-health-rubric.test.ts b/test/qa-health-rubric.test.ts index 8f8d9dee0..842f97128 100644 --- a/test/qa-health-rubric.test.ts +++ b/test/qa-health-rubric.test.ts @@ -1,4 +1,8 @@ import { describe, expect, test } from 'bun:test'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { bunIsolationArgs } from '../lib/bun-isolation'; import { generateQAMethodology } from '../scripts/resolvers/utility'; import { HOST_PATHS } from '../scripts/resolvers/types'; @@ -31,3 +35,47 @@ describe('QA health rubric scoring contract', () => { expect(rubric).toContain('Round only the final score'); }); }); + +const healthTmpl = fs.readFileSync(path.join(import.meta.dir, '..', 'health', 'SKILL.md.tmpl'), 'utf-8'); +const gbrainSection = healthTmpl.split('**GBrain sub-score computation:**')[1] ?? ''; +const gbrainScript = gbrainSection.match(/```bash\nbun --no-env-file --config=\/dev\/null -e '\n([\s\S]*?)\n'\n```/)?.[1] ?? ''; + +function gbrainScore(doctor: number | null, queue: number | null, pushHours: number | null): string { + const script = gbrainScript + .replace('', String(doctor)) + .replace('', String(queue)) + .replace('', String(pushHours)); + const run = spawnSync(process.execPath, [...bunIsolationArgs(), '-e', script], { encoding: 'utf-8', timeout: 30_000 }); + expect(run.status).toBe(0); + return run.stdout.trim(); +} + +describe('/health GBrain score has one rule, computed in code (#3351)', () => { + test('the rubric row defers to the computation instead of scoring bands', () => { + const row = healthTmpl.split('\n').find(line => line.startsWith('| GBrain |')) ?? ''; + expect(row).toContain('GBrain sub-score computation'); + expect(row).not.toMatch(/queue\s* { + expect(gbrainScript).not.toBe(''); + expect(gbrainScore(10, 50, 2)).toBe('GBRAIN_SCORE: 9.1'); + expect(gbrainScore(7, 3, 2)).toBe('GBRAIN_SCORE: 8.5'); + }); + + test('bands are half-open at 10/100 queue lines and 24/72 hours', () => { + expect(gbrainScore(10, 9, 23.9)).toBe('GBRAIN_SCORE: 10.0'); + expect(gbrainScore(10, 10, 24)).toBe('GBRAIN_SCORE: 8.5'); + expect(gbrainScore(10, 99, 71.9)).toBe('GBRAIN_SCORE: 8.5'); + expect(gbrainScore(10, 100, 72)).toBe('GBRAIN_SCORE: 5.0'); + expect(gbrainScore(0, 100, 72)).toBe('GBRAIN_SCORE: 0.0'); + expect(gbrainScore(null, 0, 1)).toBe('GBRAIN_SCORE: 5.0'); + }); + + test('sync off gives the doctor component the whole weight', () => { + expect(gbrainScore(10, null, null)).toBe('GBRAIN_SCORE: 10.0'); + expect(gbrainScore(7, null, null)).toBe('GBRAIN_SCORE: 7.0'); + expect(gbrainScore(0, null, null)).toBe('GBRAIN_SCORE: 0.0'); + }); +});