mirror of
https://github.com/garrytan/gstack.git
synced 2026-08-21 13:37:14 +02:00
test: rebase parity baseline to v1.64.0.0; fix capture-vs-check drift
The parity ratchet had quietly failed for 7 skills — v1.58-v1.64 growth landed past the v1.57.7.0 anchors and nothing caught it because this test had no CI lane (verified pre-existing: SKILL.md content is byte-identical to origin/main). Same rebase protocol as v1.53->v1.57.7.0; old baseline retained for the audit trail. Root-caused a second latent bug while rebasing: captureBaseline recorded SKELETON-ONLY bytes while the checker compares UNION bytes (skeleton + carved sections/*.md), so a fresh capture read carved skills at ~2x ratio (ship: 82KB captured vs 183KB checked). captureBaseline now takes sectionedSkills and records unions for carved skills — capture and check measure the same thing, so the NEXT rebase can't hit this. Four CARVE_GUARDS skeleton caps re-ratcheted to current +headroom (plan-ceo 92K, plan-eng 70K, office-hours 100K, design-consultation 70K), annotated inline. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Fable 5
Claude Fable 5
parent
39715addff
commit
0d1544c2db
@@ -14,6 +14,7 @@
|
|||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import { captureBaseline } from '../test/helpers/capture-parity-baseline';
|
import { captureBaseline } from '../test/helpers/capture-parity-baseline';
|
||||||
|
import { PARITY_INVARIANTS } from '../test/helpers/parity-harness';
|
||||||
|
|
||||||
const ROOT = path.resolve(import.meta.dir, '..');
|
const ROOT = path.resolve(import.meta.dir, '..');
|
||||||
|
|
||||||
@@ -33,7 +34,13 @@ const defaultOut = path.join(
|
|||||||
);
|
);
|
||||||
const outPath = outOverride ? path.resolve(outOverride) : defaultOut;
|
const outPath = outOverride ? path.resolve(outOverride) : defaultOut;
|
||||||
|
|
||||||
const baseline = captureBaseline({ repoRoot: ROOT, tag });
|
const baseline = captureBaseline({
|
||||||
|
repoRoot: ROOT,
|
||||||
|
tag,
|
||||||
|
// Carved skills record UNION bytes (skeleton + sections/*.md) so the
|
||||||
|
// baseline measures the same thing parity-harness checks against.
|
||||||
|
sectionedSkills: PARITY_INVARIANTS.filter(i => i.sectioned).map(i => i.skill),
|
||||||
|
});
|
||||||
|
|
||||||
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
||||||
fs.writeFileSync(outPath, JSON.stringify(baseline, null, 2) + '\n');
|
fs.writeFileSync(outPath, JSON.stringify(baseline, null, 2) + '\n');
|
||||||
|
|||||||
+643
@@ -0,0 +1,643 @@
|
|||||||
|
{
|
||||||
|
"tag": "v1.64.0.0",
|
||||||
|
"capturedAt": "2026-08-15T15:41:52.440Z",
|
||||||
|
"capturedFromCommit": "39715add",
|
||||||
|
"capturedFromBranch": "garrytan/test-evals-ci-speedup",
|
||||||
|
"totalSkills": 53,
|
||||||
|
"totalCorpusBytes": 3732362,
|
||||||
|
"estTotalCatalogTokens": 4177,
|
||||||
|
"topHeaviest": [
|
||||||
|
{
|
||||||
|
"skill": "ship",
|
||||||
|
"skillMdBytes": 187706,
|
||||||
|
"skillMdLines": 1435,
|
||||||
|
"estTokens": 46927,
|
||||||
|
"tmplBytes": 26135,
|
||||||
|
"descriptionLen": 293,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"skill": "plan-ceo-review",
|
||||||
|
"skillMdBytes": 151747,
|
||||||
|
"skillMdLines": 1485,
|
||||||
|
"estTokens": 37937,
|
||||||
|
"tmplBytes": 29268,
|
||||||
|
"descriptionLen": 794,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"skill": "office-hours",
|
||||||
|
"skillMdBytes": 131110,
|
||||||
|
"skillMdLines": 1706,
|
||||||
|
"estTokens": 32778,
|
||||||
|
"tmplBytes": 30541,
|
||||||
|
"descriptionLen": 860,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"skill": "spec",
|
||||||
|
"skillMdBytes": 128768,
|
||||||
|
"skillMdLines": 2376,
|
||||||
|
"estTokens": 32192,
|
||||||
|
"tmplBytes": 31226,
|
||||||
|
"descriptionLen": 282,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"skill": "plan-design-review",
|
||||||
|
"skillMdBytes": 127366,
|
||||||
|
"skillMdLines": 1531,
|
||||||
|
"estTokens": 31842,
|
||||||
|
"tmplBytes": 18463,
|
||||||
|
"descriptionLen": 218,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"skill": "plan-devex-review",
|
||||||
|
"skillMdBytes": 125243,
|
||||||
|
"skillMdLines": 1469,
|
||||||
|
"estTokens": 31311,
|
||||||
|
"tmplBytes": 18838,
|
||||||
|
"descriptionLen": 250,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"skill": "plan-eng-review",
|
||||||
|
"skillMdBytes": 124597,
|
||||||
|
"skillMdLines": 1067,
|
||||||
|
"estTokens": 31149,
|
||||||
|
"tmplBytes": 14195,
|
||||||
|
"descriptionLen": 231,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"skill": "review",
|
||||||
|
"skillMdBytes": 108523,
|
||||||
|
"skillMdLines": 1871,
|
||||||
|
"estTokens": 27131,
|
||||||
|
"tmplBytes": 14123,
|
||||||
|
"descriptionLen": 205,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"skill": "design-review",
|
||||||
|
"skillMdBytes": 105842,
|
||||||
|
"skillMdLines": 2003,
|
||||||
|
"estTokens": 26461,
|
||||||
|
"tmplBytes": 11674,
|
||||||
|
"descriptionLen": 306,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"skill": "land-and-deploy",
|
||||||
|
"skillMdBytes": 104578,
|
||||||
|
"skillMdLines": 1957,
|
||||||
|
"estTokens": 26145,
|
||||||
|
"tmplBytes": 50736,
|
||||||
|
"descriptionLen": 160,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"skills": {
|
||||||
|
"autoplan": {
|
||||||
|
"skill": "autoplan",
|
||||||
|
"skillMdBytes": 101979,
|
||||||
|
"skillMdLines": 1867,
|
||||||
|
"estTokens": 25495,
|
||||||
|
"tmplBytes": 45857,
|
||||||
|
"descriptionLen": 366,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": true
|
||||||
|
},
|
||||||
|
"benchmark": {
|
||||||
|
"skill": "benchmark",
|
||||||
|
"skillMdBytes": 37206,
|
||||||
|
"skillMdLines": 794,
|
||||||
|
"estTokens": 9302,
|
||||||
|
"tmplBytes": 9378,
|
||||||
|
"descriptionLen": 213,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"benchmark-models": {
|
||||||
|
"skill": "benchmark-models",
|
||||||
|
"skillMdBytes": 33273,
|
||||||
|
"skillMdLines": 669,
|
||||||
|
"estTokens": 8318,
|
||||||
|
"tmplBytes": 6631,
|
||||||
|
"descriptionLen": 217,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"browse": {
|
||||||
|
"skill": "browse",
|
||||||
|
"skillMdBytes": 55782,
|
||||||
|
"skillMdLines": 1031,
|
||||||
|
"estTokens": 13946,
|
||||||
|
"tmplBytes": 13944,
|
||||||
|
"descriptionLen": 181,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"canary": {
|
||||||
|
"skill": "canary",
|
||||||
|
"skillMdBytes": 57291,
|
||||||
|
"skillMdLines": 1061,
|
||||||
|
"estTokens": 14323,
|
||||||
|
"tmplBytes": 8033,
|
||||||
|
"descriptionLen": 180,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"careful": {
|
||||||
|
"skill": "careful",
|
||||||
|
"skillMdBytes": 2715,
|
||||||
|
"skillMdLines": 70,
|
||||||
|
"estTokens": 679,
|
||||||
|
"tmplBytes": 2599,
|
||||||
|
"descriptionLen": 331,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"codex": {
|
||||||
|
"skill": "codex",
|
||||||
|
"skillMdBytes": 98229,
|
||||||
|
"skillMdLines": 1717,
|
||||||
|
"estTokens": 24557,
|
||||||
|
"tmplBytes": 41467,
|
||||||
|
"descriptionLen": 187,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"context-restore": {
|
||||||
|
"skill": "context-restore",
|
||||||
|
"skillMdBytes": 53516,
|
||||||
|
"skillMdLines": 952,
|
||||||
|
"estTokens": 13379,
|
||||||
|
"tmplBytes": 7092,
|
||||||
|
"descriptionLen": 238,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"context-save": {
|
||||||
|
"skill": "context-save",
|
||||||
|
"skillMdBytes": 55876,
|
||||||
|
"skillMdLines": 1037,
|
||||||
|
"estTokens": 13969,
|
||||||
|
"tmplBytes": 9293,
|
||||||
|
"descriptionLen": 168,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"cso": {
|
||||||
|
"skill": "cso",
|
||||||
|
"skillMdBytes": 89500,
|
||||||
|
"skillMdLines": 1294,
|
||||||
|
"estTokens": 22375,
|
||||||
|
"tmplBytes": 21724,
|
||||||
|
"descriptionLen": 196,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"design-consultation": {
|
||||||
|
"skill": "design-consultation",
|
||||||
|
"skillMdBytes": 90375,
|
||||||
|
"skillMdLines": 1239,
|
||||||
|
"estTokens": 22594,
|
||||||
|
"tmplBytes": 9554,
|
||||||
|
"descriptionLen": 890,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"design-html": {
|
||||||
|
"skill": "design-html",
|
||||||
|
"skillMdBytes": 76735,
|
||||||
|
"skillMdLines": 1520,
|
||||||
|
"estTokens": 19184,
|
||||||
|
"tmplBytes": 22567,
|
||||||
|
"descriptionLen": 235,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"design-review": {
|
||||||
|
"skill": "design-review",
|
||||||
|
"skillMdBytes": 105842,
|
||||||
|
"skillMdLines": 2003,
|
||||||
|
"estTokens": 26461,
|
||||||
|
"tmplBytes": 11674,
|
||||||
|
"descriptionLen": 306,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"design-shotgun": {
|
||||||
|
"skill": "design-shotgun",
|
||||||
|
"skillMdBytes": 73024,
|
||||||
|
"skillMdLines": 1382,
|
||||||
|
"estTokens": 18256,
|
||||||
|
"tmplBytes": 13331,
|
||||||
|
"descriptionLen": 788,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"devex-review": {
|
||||||
|
"skill": "devex-review",
|
||||||
|
"skillMdBytes": 75374,
|
||||||
|
"skillMdLines": 1314,
|
||||||
|
"estTokens": 18844,
|
||||||
|
"tmplBytes": 7984,
|
||||||
|
"descriptionLen": 201,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"diagram": {
|
||||||
|
"skill": "diagram",
|
||||||
|
"skillMdBytes": 54076,
|
||||||
|
"skillMdLines": 931,
|
||||||
|
"estTokens": 13519,
|
||||||
|
"tmplBytes": 6715,
|
||||||
|
"descriptionLen": 359,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"document-generate": {
|
||||||
|
"skill": "document-generate",
|
||||||
|
"skillMdBytes": 64020,
|
||||||
|
"skillMdLines": 1261,
|
||||||
|
"estTokens": 16005,
|
||||||
|
"tmplBytes": 15940,
|
||||||
|
"descriptionLen": 334,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"document-release": {
|
||||||
|
"skill": "document-release",
|
||||||
|
"skillMdBytes": 76636,
|
||||||
|
"skillMdLines": 970,
|
||||||
|
"estTokens": 19159,
|
||||||
|
"tmplBytes": 6688,
|
||||||
|
"descriptionLen": 192,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"freeze": {
|
||||||
|
"skill": "freeze",
|
||||||
|
"skillMdBytes": 3313,
|
||||||
|
"skillMdLines": 94,
|
||||||
|
"estTokens": 828,
|
||||||
|
"tmplBytes": 3197,
|
||||||
|
"descriptionLen": 533,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"gstack-upgrade": {
|
||||||
|
"skill": "gstack-upgrade",
|
||||||
|
"skillMdBytes": 10817,
|
||||||
|
"skillMdLines": 285,
|
||||||
|
"estTokens": 2704,
|
||||||
|
"tmplBytes": 10667,
|
||||||
|
"descriptionLen": 163,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"guard": {
|
||||||
|
"skill": "guard",
|
||||||
|
"skillMdBytes": 3314,
|
||||||
|
"skillMdLines": 91,
|
||||||
|
"estTokens": 829,
|
||||||
|
"tmplBytes": 3196,
|
||||||
|
"descriptionLen": 703,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"health": {
|
||||||
|
"skill": "health",
|
||||||
|
"skillMdBytes": 58102,
|
||||||
|
"skillMdLines": 1085,
|
||||||
|
"estTokens": 14526,
|
||||||
|
"tmplBytes": 11617,
|
||||||
|
"descriptionLen": 184,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"investigate": {
|
||||||
|
"skill": "investigate",
|
||||||
|
"skillMdBytes": 60600,
|
||||||
|
"skillMdLines": 1085,
|
||||||
|
"estTokens": 15150,
|
||||||
|
"tmplBytes": 11566,
|
||||||
|
"descriptionLen": 1241,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"ios-clean": {
|
||||||
|
"skill": "ios-clean",
|
||||||
|
"skillMdBytes": 51125,
|
||||||
|
"skillMdLines": 883,
|
||||||
|
"estTokens": 12781,
|
||||||
|
"tmplBytes": 3743,
|
||||||
|
"descriptionLen": 254,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"ios-design-review": {
|
||||||
|
"skill": "ios-design-review",
|
||||||
|
"skillMdBytes": 51817,
|
||||||
|
"skillMdLines": 886,
|
||||||
|
"estTokens": 12954,
|
||||||
|
"tmplBytes": 4417,
|
||||||
|
"descriptionLen": 209,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"ios-fix": {
|
||||||
|
"skill": "ios-fix",
|
||||||
|
"skillMdBytes": 50946,
|
||||||
|
"skillMdLines": 882,
|
||||||
|
"estTokens": 12737,
|
||||||
|
"tmplBytes": 3574,
|
||||||
|
"descriptionLen": 187,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"ios-qa": {
|
||||||
|
"skill": "ios-qa",
|
||||||
|
"skillMdBytes": 58828,
|
||||||
|
"skillMdLines": 1026,
|
||||||
|
"estTokens": 14707,
|
||||||
|
"tmplBytes": 11461,
|
||||||
|
"descriptionLen": 223,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"ios-sync": {
|
||||||
|
"skill": "ios-sync",
|
||||||
|
"skillMdBytes": 51599,
|
||||||
|
"skillMdLines": 880,
|
||||||
|
"estTokens": 12900,
|
||||||
|
"tmplBytes": 4220,
|
||||||
|
"descriptionLen": 269,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"land-and-deploy": {
|
||||||
|
"skill": "land-and-deploy",
|
||||||
|
"skillMdBytes": 104578,
|
||||||
|
"skillMdLines": 1957,
|
||||||
|
"estTokens": 26145,
|
||||||
|
"tmplBytes": 50736,
|
||||||
|
"descriptionLen": 160,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"landing-report": {
|
||||||
|
"skill": "landing-report",
|
||||||
|
"skillMdBytes": 54195,
|
||||||
|
"skillMdLines": 945,
|
||||||
|
"estTokens": 13549,
|
||||||
|
"tmplBytes": 6830,
|
||||||
|
"descriptionLen": 195,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"learn": {
|
||||||
|
"skill": "learn",
|
||||||
|
"skillMdBytes": 51908,
|
||||||
|
"skillMdLines": 962,
|
||||||
|
"estTokens": 12977,
|
||||||
|
"tmplBytes": 5594,
|
||||||
|
"descriptionLen": 178,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"make-pdf": {
|
||||||
|
"skill": "make-pdf",
|
||||||
|
"skillMdBytes": 37590,
|
||||||
|
"skillMdLines": 796,
|
||||||
|
"estTokens": 9398,
|
||||||
|
"tmplBytes": 9306,
|
||||||
|
"descriptionLen": 177,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"office-hours": {
|
||||||
|
"skill": "office-hours",
|
||||||
|
"skillMdBytes": 131110,
|
||||||
|
"skillMdLines": 1706,
|
||||||
|
"estTokens": 32778,
|
||||||
|
"tmplBytes": 30541,
|
||||||
|
"descriptionLen": 860,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"open-gstack-browser": {
|
||||||
|
"skill": "open-gstack-browser",
|
||||||
|
"skillMdBytes": 56317,
|
||||||
|
"skillMdLines": 1025,
|
||||||
|
"estTokens": 14079,
|
||||||
|
"tmplBytes": 7702,
|
||||||
|
"descriptionLen": 204,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"pair-agent": {
|
||||||
|
"skill": "pair-agent",
|
||||||
|
"skillMdBytes": 57869,
|
||||||
|
"skillMdLines": 1092,
|
||||||
|
"estTokens": 14467,
|
||||||
|
"tmplBytes": 9292,
|
||||||
|
"descriptionLen": 167,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"plan-ceo-review": {
|
||||||
|
"skill": "plan-ceo-review",
|
||||||
|
"skillMdBytes": 151747,
|
||||||
|
"skillMdLines": 1485,
|
||||||
|
"estTokens": 37937,
|
||||||
|
"tmplBytes": 29268,
|
||||||
|
"descriptionLen": 794,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": true
|
||||||
|
},
|
||||||
|
"plan-design-review": {
|
||||||
|
"skill": "plan-design-review",
|
||||||
|
"skillMdBytes": 127366,
|
||||||
|
"skillMdLines": 1531,
|
||||||
|
"estTokens": 31842,
|
||||||
|
"tmplBytes": 18463,
|
||||||
|
"descriptionLen": 218,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": true
|
||||||
|
},
|
||||||
|
"plan-devex-review": {
|
||||||
|
"skill": "plan-devex-review",
|
||||||
|
"skillMdBytes": 125243,
|
||||||
|
"skillMdLines": 1469,
|
||||||
|
"estTokens": 31311,
|
||||||
|
"tmplBytes": 18838,
|
||||||
|
"descriptionLen": 250,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": true
|
||||||
|
},
|
||||||
|
"plan-eng-review": {
|
||||||
|
"skill": "plan-eng-review",
|
||||||
|
"skillMdBytes": 124597,
|
||||||
|
"skillMdLines": 1067,
|
||||||
|
"estTokens": 31149,
|
||||||
|
"tmplBytes": 14195,
|
||||||
|
"descriptionLen": 231,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": true
|
||||||
|
},
|
||||||
|
"plan-tune": {
|
||||||
|
"skill": "plan-tune",
|
||||||
|
"skillMdBytes": 73241,
|
||||||
|
"skillMdLines": 1422,
|
||||||
|
"estTokens": 18310,
|
||||||
|
"tmplBytes": 26922,
|
||||||
|
"descriptionLen": 327,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"qa": {
|
||||||
|
"skill": "qa",
|
||||||
|
"skillMdBytes": 84049,
|
||||||
|
"skillMdLines": 1693,
|
||||||
|
"estTokens": 21012,
|
||||||
|
"tmplBytes": 12701,
|
||||||
|
"descriptionLen": 218,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"qa-only": {
|
||||||
|
"skill": "qa-only",
|
||||||
|
"skillMdBytes": 66607,
|
||||||
|
"skillMdLines": 1265,
|
||||||
|
"estTokens": 16652,
|
||||||
|
"tmplBytes": 3851,
|
||||||
|
"descriptionLen": 165,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"retro": {
|
||||||
|
"skill": "retro",
|
||||||
|
"skillMdBytes": 93029,
|
||||||
|
"skillMdLines": 1821,
|
||||||
|
"estTokens": 23257,
|
||||||
|
"tmplBytes": 42381,
|
||||||
|
"descriptionLen": 648,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"review": {
|
||||||
|
"skill": "review",
|
||||||
|
"skillMdBytes": 108523,
|
||||||
|
"skillMdLines": 1871,
|
||||||
|
"estTokens": 27131,
|
||||||
|
"tmplBytes": 14123,
|
||||||
|
"descriptionLen": 205,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"scrape": {
|
||||||
|
"skill": "scrape",
|
||||||
|
"skillMdBytes": 53827,
|
||||||
|
"skillMdLines": 958,
|
||||||
|
"estTokens": 13457,
|
||||||
|
"tmplBytes": 5220,
|
||||||
|
"descriptionLen": 167,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"setup-browser-cookies": {
|
||||||
|
"skill": "setup-browser-cookies",
|
||||||
|
"skillMdBytes": 30558,
|
||||||
|
"skillMdLines": 641,
|
||||||
|
"estTokens": 7640,
|
||||||
|
"tmplBytes": 2724,
|
||||||
|
"descriptionLen": 222,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"setup-deploy": {
|
||||||
|
"skill": "setup-deploy",
|
||||||
|
"skillMdBytes": 54113,
|
||||||
|
"skillMdLines": 990,
|
||||||
|
"estTokens": 13528,
|
||||||
|
"tmplBytes": 7780,
|
||||||
|
"descriptionLen": 197,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"setup-gbrain": {
|
||||||
|
"skill": "setup-gbrain",
|
||||||
|
"skillMdBytes": 92312,
|
||||||
|
"skillMdLines": 1860,
|
||||||
|
"estTokens": 23078,
|
||||||
|
"tmplBytes": 45975,
|
||||||
|
"descriptionLen": 325,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"ship": {
|
||||||
|
"skill": "ship",
|
||||||
|
"skillMdBytes": 187706,
|
||||||
|
"skillMdLines": 1435,
|
||||||
|
"estTokens": 46927,
|
||||||
|
"tmplBytes": 26135,
|
||||||
|
"descriptionLen": 293,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": true
|
||||||
|
},
|
||||||
|
"skillify": {
|
||||||
|
"skill": "skillify",
|
||||||
|
"skillMdBytes": 63720,
|
||||||
|
"skillMdLines": 1239,
|
||||||
|
"estTokens": 15930,
|
||||||
|
"tmplBytes": 15107,
|
||||||
|
"descriptionLen": 233,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"spec": {
|
||||||
|
"skill": "spec",
|
||||||
|
"skillMdBytes": 128768,
|
||||||
|
"skillMdLines": 2376,
|
||||||
|
"estTokens": 32192,
|
||||||
|
"tmplBytes": 31226,
|
||||||
|
"descriptionLen": 282,
|
||||||
|
"hasGateEval": true,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"sync-gbrain": {
|
||||||
|
"skill": "sync-gbrain",
|
||||||
|
"skillMdBytes": 69975,
|
||||||
|
"skillMdLines": 1259,
|
||||||
|
"estTokens": 17494,
|
||||||
|
"tmplBytes": 23629,
|
||||||
|
"descriptionLen": 246,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
},
|
||||||
|
"unfreeze": {
|
||||||
|
"skill": "unfreeze",
|
||||||
|
"skillMdBytes": 1504,
|
||||||
|
"skillMdLines": 49,
|
||||||
|
"estTokens": 376,
|
||||||
|
"tmplBytes": 1386,
|
||||||
|
"descriptionLen": 199,
|
||||||
|
"hasGateEval": false,
|
||||||
|
"hasPeriodicEval": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -46,6 +46,14 @@ export interface ParityBaseline {
|
|||||||
export interface CaptureOptions {
|
export interface CaptureOptions {
|
||||||
repoRoot: string;
|
repoRoot: string;
|
||||||
tag?: string;
|
tag?: string;
|
||||||
|
/**
|
||||||
|
* Skills whose baseline bytes must be the UNION of skeleton + sections/*.md
|
||||||
|
* (mirroring parity-harness readSkillForParity, which is what the checker
|
||||||
|
* compares against). Omitting a carved skill here records skeleton-only
|
||||||
|
* bytes and the ratio check then reads ~2x on the next parity run — the
|
||||||
|
* exact capture-vs-check drift that broke the v1.64 rebase.
|
||||||
|
*/
|
||||||
|
sectionedSkills?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Extract the frontmatter description from a SKILL.md file. Empty string if none. */
|
/** Extract the frontmatter description from a SKILL.md file. Empty string if none. */
|
||||||
@@ -134,7 +142,7 @@ function getGitInfo(repoRoot: string): { commit: string; branch: string } {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function captureBaseline(opts: CaptureOptions): ParityBaseline {
|
export function captureBaseline(opts: CaptureOptions): ParityBaseline {
|
||||||
const { repoRoot, tag } = opts;
|
const { repoRoot, tag, sectionedSkills } = opts;
|
||||||
const skillDirs = discoverSkillDirs(repoRoot);
|
const skillDirs = discoverSkillDirs(repoRoot);
|
||||||
const evalCoverage = discoverEvalCoverage(repoRoot, skillDirs);
|
const evalCoverage = discoverEvalCoverage(repoRoot, skillDirs);
|
||||||
const skills: Record<string, SkillBaselineEntry> = {};
|
const skills: Record<string, SkillBaselineEntry> = {};
|
||||||
@@ -144,7 +152,18 @@ export function captureBaseline(opts: CaptureOptions): ParityBaseline {
|
|||||||
const skillMdPath = path.join(repoRoot, dir, 'SKILL.md');
|
const skillMdPath = path.join(repoRoot, dir, 'SKILL.md');
|
||||||
const tmplPath = path.join(repoRoot, dir, 'SKILL.md.tmpl');
|
const tmplPath = path.join(repoRoot, dir, 'SKILL.md.tmpl');
|
||||||
const content = fs.readFileSync(skillMdPath, 'utf-8');
|
const content = fs.readFileSync(skillMdPath, 'utf-8');
|
||||||
const bytes = Buffer.byteLength(content, 'utf-8');
|
let bytes = Buffer.byteLength(content, 'utf-8');
|
||||||
|
// Union in the carved sections for sectioned skills — semantic twin of
|
||||||
|
// parity-harness readSkillForParity (which the checker uses). Kept inline
|
||||||
|
// because parity-harness imports this module as a value (cycle).
|
||||||
|
if (sectionedSkills?.includes(dir)) {
|
||||||
|
const sectionsDir = path.join(repoRoot, dir, 'sections');
|
||||||
|
if (fs.existsSync(sectionsDir)) {
|
||||||
|
for (const f of fs.readdirSync(sectionsDir).filter(f => f.endsWith('.md')).sort()) {
|
||||||
|
bytes += Buffer.byteLength(fs.readFileSync(path.join(sectionsDir, f), 'utf-8'), 'utf-8');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
const lines = content.split('\n').length;
|
const lines = content.split('\n').length;
|
||||||
const description = extractDescription(content);
|
const description = extractDescription(content);
|
||||||
const descriptionLen = Buffer.byteLength(description, 'utf-8');
|
const descriptionLen = Buffer.byteLength(description, 'utf-8');
|
||||||
|
|||||||
@@ -144,7 +144,9 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
|||||||
},
|
},
|
||||||
behavioral: 'external',
|
behavioral: 'external',
|
||||||
externalTest: 'test/skill-e2e-plan-ceo-review-section-loading.test.ts',
|
externalTest: 'test/skill-e2e-plan-ceo-review-section-loading.test.ts',
|
||||||
maxSkeletonBytes: 90_000,
|
// Re-ratcheted 2026-08 (v1.64 baseline rebase): v1.58-v1.64 growth landed
|
||||||
|
// while no CI lane ran the parity check; free-tests lane now enforces it.
|
||||||
|
maxSkeletonBytes: 92_000,
|
||||||
minUnionBytes: 80_000,
|
minUnionBytes: 80_000,
|
||||||
mustContain: ['SCOPE EXPANSION', 'SELECTIVE EXPANSION', 'HOLD SCOPE', 'SCOPE REDUCTION'],
|
mustContain: ['SCOPE EXPANSION', 'SELECTIVE EXPANSION', 'HOLD SCOPE', 'SCOPE REDUCTION'],
|
||||||
// Default-on Codex outside-voice (codexPreflight block + CODEX_MODE branch
|
// Default-on Codex outside-voice (codexPreflight block + CODEX_MODE branch
|
||||||
@@ -165,7 +167,9 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
|||||||
behavioral: 'plan',
|
behavioral: 'plan',
|
||||||
// v1.2.0 activation lift (shared first-run-guidance preamble) + #2077 ask-first scope gate.
|
// v1.2.0 activation lift (shared first-run-guidance preamble) + #2077 ask-first scope gate.
|
||||||
// +~1 KB: plan-mode auto-select-B scope-gate exceptions (2026-08).
|
// +~1 KB: plan-mode auto-select-B scope-gate exceptions (2026-08).
|
||||||
maxSkeletonBytes: 68_000,
|
// Re-ratcheted 2026-08 (v1.64 baseline rebase): v1.58-v1.64 growth landed
|
||||||
|
// while no CI lane ran the parity check; free-tests lane now enforces it.
|
||||||
|
maxSkeletonBytes: 70_000,
|
||||||
minUnionBytes: 70_000,
|
minUnionBytes: 70_000,
|
||||||
mustContain: ['Architecture', 'Code Quality', 'Test', 'Performance'],
|
mustContain: ['Architecture', 'Code Quality', 'Test', 'Performance'],
|
||||||
// Cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback + the
|
// Cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback + the
|
||||||
@@ -240,7 +244,9 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
|||||||
behavioral: 'prompt',
|
behavioral: 'prompt',
|
||||||
// v1.2.0 activation lift: first-run-guidance section in the shared preamble,
|
// v1.2.0 activation lift: first-run-guidance section in the shared preamble,
|
||||||
// plus the P1 office-hours closing handoff (AUQ that launches the next skill).
|
// plus the P1 office-hours closing handoff (AUQ that launches the next skill).
|
||||||
maxSkeletonBytes: 98_000,
|
// Re-ratcheted 2026-08 (v1.64 baseline rebase): v1.58-v1.64 growth landed
|
||||||
|
// while no CI lane ran the parity check; free-tests lane now enforces it.
|
||||||
|
maxSkeletonBytes: 100_000,
|
||||||
minUnionBytes: 70_000,
|
minUnionBytes: 70_000,
|
||||||
mustContain: ['design doc', 'problem statement'],
|
mustContain: ['design doc', 'problem statement'],
|
||||||
maxSizeRatio: 1.07,
|
maxSizeRatio: 1.07,
|
||||||
@@ -289,7 +295,9 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
|||||||
// +Conductor AUQ-default-prose rule + one-way/continuation safety in the
|
// +Conductor AUQ-default-prose rule + one-way/continuation safety in the
|
||||||
// always-loaded AskUserQuestion Format section.
|
// always-loaded AskUserQuestion Format section.
|
||||||
// v1.2.0 activation lift: first-run-guidance section in the shared preamble.
|
// v1.2.0 activation lift: first-run-guidance section in the shared preamble.
|
||||||
maxSkeletonBytes: 69_000,
|
// Re-ratcheted 2026-08 (v1.64 baseline rebase): v1.58-v1.64 growth landed
|
||||||
|
// while no CI lane ran the parity check; free-tests lane now enforces it.
|
||||||
|
maxSkeletonBytes: 70_000,
|
||||||
minUnionBytes: 72_000,
|
minUnionBytes: 72_000,
|
||||||
mustContain: ['Typography', 'Color', 'Aesthetic Direction'],
|
mustContain: ['Typography', 'Color', 'Aesthetic Direction'],
|
||||||
// Cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback ~2KB +
|
// Cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback ~2KB +
|
||||||
|
|||||||
@@ -2,10 +2,17 @@
|
|||||||
* Cathedral parity suite — gate-tier (free, structural + content checks).
|
* Cathedral parity suite — gate-tier (free, structural + content checks).
|
||||||
*
|
*
|
||||||
* Runs every PARITY_INVARIANTS check against the current SKILL.md output
|
* Runs every PARITY_INVARIANTS check against the current SKILL.md output
|
||||||
* vs the v1.57.7.0 baseline. Failures get an actionable, per-skill report
|
* vs the v1.64.0.0 baseline. Failures get an actionable, per-skill report
|
||||||
* showing missing phrases, missing headings, and size ratios.
|
* showing missing phrases, missing headings, and size ratios.
|
||||||
*
|
*
|
||||||
* Baseline rebased v1.53.0.0 → v1.57.7.0: the v1.54–v1.57 releases (ship/plan
|
* Baseline rebased v1.57.7.0 → v1.64.0.0: the v1.58–v1.64 waves grew 7 skills
|
||||||
|
* past their ratchets (review/qa/investigate size ratios; plan-ceo, plan-eng,
|
||||||
|
* office-hours, design-consultation skeleton caps) and nothing caught it —
|
||||||
|
* this test had NO Linux CI lane, so the drift accumulated silently across
|
||||||
|
* six releases and only surfaced when the free-tests CI lane landed. The
|
||||||
|
* v1.64.0.0 baseline captures current sizes so the ratchet catches FUTURE
|
||||||
|
* bloat again.
|
||||||
|
* Earlier rebase v1.53.0.0 → v1.57.7.0: the v1.54–v1.57 releases (ship/plan
|
||||||
* carving, carve-guards, AUQ prose fallback, the cross-session decision-log
|
* carving, carve-guards, AUQ prose fallback, the cross-session decision-log
|
||||||
* preamble) plus the mandatory unresolved-decisions status added to every
|
* preamble) plus the mandatory unresolved-decisions status added to every
|
||||||
* GSTACK REVIEW REPORT pushed the three plan-review skills past the 5% ratchet
|
* GSTACK REVIEW REPORT pushed the three plan-review skills past the 5% ratchet
|
||||||
@@ -14,7 +21,7 @@
|
|||||||
* harness measures) so the per-skill 1.05 ratio still catches future bloat.
|
* harness measures) so the per-skill 1.05 ratio still catches future bloat.
|
||||||
* Earlier rebase v1.44.1 → v1.53.0.0: brain-aware-planning (v1.49–v1.52) + the
|
* Earlier rebase v1.44.1 → v1.53.0.0: brain-aware-planning (v1.49–v1.52) + the
|
||||||
* v1.53 redaction guard. Historical v1.44.1 / v1.46.0.0 / v1.47.0.0 / v1.53.0.0
|
* v1.53 redaction guard. Historical v1.44.1 / v1.46.0.0 / v1.47.0.0 / v1.53.0.0
|
||||||
* baselines are retained in test/fixtures/ for the audit trail.
|
* / v1.57.7.0 baselines are retained in test/fixtures/ for the audit trail.
|
||||||
*
|
*
|
||||||
* Periodic-tier LLM-judge parity (paid) lands in Phase B (v2.0.0.0)
|
* Periodic-tier LLM-judge parity (paid) lands in Phase B (v2.0.0.0)
|
||||||
* alongside the sections/ extraction. Plumbing is in parity-harness.ts.
|
* alongside the sections/ extraction. Plumbing is in parity-harness.ts.
|
||||||
@@ -27,9 +34,9 @@ import { runParityChecks, PARITY_INVARIANTS } from './helpers/parity-harness';
|
|||||||
import type { ParityBaseline } from './helpers/capture-parity-baseline';
|
import type { ParityBaseline } from './helpers/capture-parity-baseline';
|
||||||
|
|
||||||
const REPO_ROOT = path.resolve(import.meta.dir, '..');
|
const REPO_ROOT = path.resolve(import.meta.dir, '..');
|
||||||
const BASELINE_PATH = path.join(REPO_ROOT, 'test', 'fixtures', 'parity-baseline-v1.57.7.0.json');
|
const BASELINE_PATH = path.join(REPO_ROOT, 'test', 'fixtures', 'parity-baseline-v1.64.0.0.json');
|
||||||
|
|
||||||
describe('parity suite vs v1.57.7.0 baseline (gate, free)', () => {
|
describe('parity suite vs v1.64.0.0 baseline (gate, free)', () => {
|
||||||
test('baseline exists', () => {
|
test('baseline exists', () => {
|
||||||
expect(fs.existsSync(BASELINE_PATH)).toBe(true);
|
expect(fs.existsSync(BASELINE_PATH)).toBe(true);
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user