From 322906ab2b0975c6b009ae4a2804259be9e8ed70 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 2 Nov 2026 05:17:17 -0800 Subject: [PATCH] v1.92.79.0 fix: /plan-tune stops reporting mismatches with no observed answers behind them (#3436) (#3856) * fix(plan-tune): compare declared and inferred only with 3+ observed answers (#3436) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: derive keeps the gap at 3 events; changelog fragment for #3436 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(plan-tune): profile view and design doc honor the 3-answer floor (#3436) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(plan-tune): keep the 10-answer floor in --check-mismatch beside the per-dimension floor Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 11 ++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- bin/gstack-developer-profile | 23 +++++-- docs/designs/PLAN_TUNING_V0.md | 2 +- package.json | 2 +- plan-tune/SKILL.md | 5 ++ plan-tune/SKILL.md.tmpl | 5 ++ test/gstack-developer-profile.test.ts | 92 +++++++++++++++++++++++++++ 9 files changed, 135 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 517e1d270..1fc172340 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [1.92.79.0] - 2026-11-02 + +**/plan-tune no longer reports a mismatch between your stated and observed preferences when it has no observed answers behind it (#3436).** `--check-mismatch` compared a declared value against an inferred value that had never moved from its 0.5 starting point, so a strong declaration with no matching answers printed `scope_appetite: declared 0.9 vs inferred 0.5 (gap 0.4)` even though no logged answer touched that dimension. + +### Fixed +- **`bin/gstack-developer-profile`**: `--check-mismatch`, `--gap` and the gap that `--derive` stores compare a dimension only when at least 3 logged answers moved it. `--check-mismatch` still needs 10 logged answers overall before it reports anything. Below the 3-answer floor, `--check-mismatch` prints `no observed evidence` or `too little evidence (n of 3 events)` instead of a gap, and `--gap` lists the dimension under a new `low_evidence` field. A profile derived before `contributing_events` existed keeps the old behavior until its next derive. +- **/plan-tune "Show gap"**: says "not enough observed answers yet" for those dimensions instead of calling them drift. + +### Tests +- `test/gstack-developer-profile.test.ts`: a declared dimension with no evidence is not flagged by `--check-mismatch` or `--gap` (both failed on main), 1 or 2 events report too little evidence, 3 events are compared, three same-day answers that agree with the declaration still report not enough data, and `--derive` stores no gap without evidence. + ## [1.92.78.0] - 2026-11-02 **Seven flaky tests from the 2026-11-01 flaky pass stop failing correct runs, and six more now name their cause when they fail.** One product lock broke a fresh lock it had misjudged as abandoned, a /ship lane tripped the free runner's home check with its own log, and the rest were fixtures that raced their own setup, leaned on wall-clock sleeps or read the caller's state root. diff --git a/VERSION b/VERSION index 66231b754..dfb847afe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.92.78.0 +1.92.79.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 9f27e6a29..192027e8e 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.92.78.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.92.79.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/bin/gstack-developer-profile b/bin/gstack-developer-profile index bc489c5c1..409c4f56e 100755 --- a/bin/gstack-developer-profile +++ b/bin/gstack-developer-profile @@ -54,6 +54,7 @@ case "$(uname -s)" in esac PROFILE_FILE="$GSTACK_HOME/developer-profile.json" LEGACY_FILE="$GSTACK_HOME/builder-profile.jsonl" +MIN_EVIDENCE=3 eval "$("$SCRIPT_DIR/gstack-slug" 2>/dev/null || true)" SLUG="${SLUG:-unknown}" @@ -512,14 +513,19 @@ do_gap() { const p = JSON.parse(await Bun.stdin.text()); const declared = p.declared || {}; const inferred = (p.inferred && p.inferred.values) || {}; + const contributing = p.inferred && p.inferred.contributing_events; const dims = ['scope_appetite','risk_tolerance','detail_preference','autonomy','architecture_care']; const gap = {}; + const low_evidence = {}; for (const d of dims) { - if (declared[d] !== undefined && inferred[d] !== undefined) { - gap[d] = +(Math.abs(declared[d] - inferred[d])).toFixed(3); + if (declared[d] === undefined || inferred[d] === undefined) continue; + if (contributing && (contributing[d] || 0) < $MIN_EVIDENCE) { + low_evidence[d] = contributing[d] || 0; + continue; } + gap[d] = +(Math.abs(declared[d] - inferred[d])).toFixed(3); } - console.log(JSON.stringify({ declared, inferred, gap }, null, 2)); + console.log(JSON.stringify({ declared, inferred, gap, low_evidence }, null, 2)); " } @@ -657,6 +663,7 @@ do_derive() { // Recompute gap. const gap = {}; for (const d of Object.keys(values)) { + if ((contributing[d] || 0) < $MIN_EVIDENCE) continue; if (profile.declared && profile.declared[d] !== undefined) { gap[d] = +(Math.abs(profile.declared[d] - values[d])).toFixed(3); } @@ -727,9 +734,8 @@ do_check_mismatch() { const declared = p.declared || {}; const inferred = (p.inferred && p.inferred.values) || {}; const sampleSize = (p.inferred && p.inferred.sample_size) || 0; - const diversity = (p.inferred && p.inferred.diversity) || {}; + const contributing = p.inferred && p.inferred.contributing_events; - // Require enough data before reporting mismatch. if (sampleSize < 10) { console.log('MISMATCH: not enough data (' + sampleSize + ' events; need 10+)'); process.exit(0); @@ -737,8 +743,14 @@ do_check_mismatch() { const THRESHOLD = 0.3; const flagged = []; + const thin = []; for (const d of Object.keys(declared)) { if (inferred[d] === undefined) continue; + const n = contributing ? (contributing[d] || 0) : undefined; + if (n !== undefined && n < $MIN_EVIDENCE) { + thin.push(' ' + d + ': ' + (n === 0 ? 'no observed evidence' : 'too little evidence (' + n + ' of $MIN_EVIDENCE events)')); + continue; + } const gap = Math.abs(declared[d] - inferred[d]); if (gap > THRESHOLD) { flagged.push({ dim: d, declared: declared[d], inferred: inferred[d], gap: +gap.toFixed(3) }); @@ -753,6 +765,7 @@ do_check_mismatch() { console.log(' ' + f.dim + ': declared ' + f.declared + ' vs inferred ' + f.inferred + ' (gap ' + f.gap + ')'); } } + for (const line of thin) console.log(line); " } diff --git a/docs/designs/PLAN_TUNING_V0.md b/docs/designs/PLAN_TUNING_V0.md index e9bf2b440..77bf95e8d 100644 --- a/docs/designs/PLAN_TUNING_V0.md +++ b/docs/designs/PLAN_TUNING_V0.md @@ -105,7 +105,7 @@ After weighing Codex's argument, we chose to roll back CEO EXPANSION and ship an } ``` -`signal_events` counts the logged answers that moved any dimension, and `contributing_events` counts, per dimension, the logged answers that moved it (an answer that writes two deltas to one dimension counts once), so a dimension with no evidence (0) is visible beside one whose evidence balances out; both print 0.5 (#2623). Each value is the sigmoid of its accumulated deltas, and each UTC day's sum per dimension (by the row's `ts`) is clamped to the largest single delta in the signal map (currently ±0.06), so one burst of answers can't pin a dimension; `--trace ` lists every contributing answer, each clamped day and the total. The bound keys on the day, not on `session_id`: the PostToolUse hook and the agent log different ids for the same review (the host's conversation id and the preamble's run id), and an agent can leave the `SESSION_ID` placeholder unsubstituted, so no logged id reliably marks one review. The cost is that several reviews on one day count as one. `contributing_events` still counts an answer logged by both writers twice. Only an ISO 8601 date or date-time `ts` counts as dated (a time without a zone is read as UTC), every other row shares one undated bucket, and `diversity.days_span` counts the same UTC days. Nothing displays `contributing_events` yet (#3436). +`signal_events` counts the logged answers that moved any dimension, and `contributing_events` counts, per dimension, the logged answers that moved it (an answer that writes two deltas to one dimension counts once), so a dimension with no evidence (0) is visible beside one whose evidence balances out; both print 0.5 (#2623). Each value is the sigmoid of its accumulated deltas, and each UTC day's sum per dimension (by the row's `ts`) is clamped to the largest single delta in the signal map (currently ±0.06), so one burst of answers can't pin a dimension; `--trace ` lists every contributing answer, each clamped day and the total. The bound keys on the day, not on `session_id`: the PostToolUse hook and the agent log different ids for the same review (the host's conversation id and the preamble's run id), and an agent can leave the `SESSION_ID` placeholder unsubstituted, so no logged id reliably marks one review. The cost is that several reviews on one day count as one. `contributing_events` still counts an answer logged by both writers twice. Only an ISO 8601 date or date-time `ts` counts as dated (a time without a zone is read as UTC), every other row shares one undated bucket, and `diversity.days_span` counts the same UTC days. `--check-mismatch`, `--gap`, the stored `gap` and /plan-tune's profile view compare a dimension only when its `contributing_events` is at least 3; `--gap` lists the others under `low_evidence` and the stored `gap` omits them (#3436). **Diversity check** (Codex #13): `inferred` is considered "enough data" only when `sample_size >= 20 AND skills_covered >= 3 AND question_ids_covered >= 8 AND days_span >= 7`. Below this, `/plan-tune profile` shows "not enough observed data yet" instead of a potentially-misleading inferred value. diff --git a/package.json b/package.json index bce0fb11b..b8a73016c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.92.78", + "version": "1.92.79", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/plan-tune/SKILL.md b/plan-tune/SKILL.md index 86b2cbdb7..481d7764e 100644 --- a/plan-tune/SKILL.md +++ b/plan-tune/SKILL.md @@ -341,6 +341,8 @@ Parse the JSON. Present in **plain English**, not raw floats: the inferred column next to declared: "**scope_appetite:** declared 0.8 (boil the ocean) ↔ observed 0.72 (close)" Use words for the gap: 0.0-0.1 "close", 0.1-0.3 "drift", 0.3+ "mismatch". + When `inferred.contributing_events[dim]` is below 3, show "not enough + observed answers yet" for that dimension instead of an observed value or gap word. This display gate is intentionally lower than the E1 **promotion gate** (90+ days stable across 3+ skills, per `docs/designs/PLAN_TUNING_V0.md`). @@ -491,6 +493,9 @@ Parse the JSON. For each dimension where both declared and inferred exist: Consider updating your declared value, or reflect on whether your behavior is actually what you want." +For each dimension in `low_evidence` (fewer than 3 observed answers moved it), +say "not enough observed answers yet" instead of a gap. + Never auto-update declared based on the gap. In v1 the gap is reporting only — the user decides whether declared is wrong or behavior is wrong. diff --git a/plan-tune/SKILL.md.tmpl b/plan-tune/SKILL.md.tmpl index 748dfad62..88b7a8eb3 100644 --- a/plan-tune/SKILL.md.tmpl +++ b/plan-tune/SKILL.md.tmpl @@ -272,6 +272,8 @@ Parse the JSON. Present in **plain English**, not raw floats: the inferred column next to declared: "**scope_appetite:** declared 0.8 (boil the ocean) ↔ observed 0.72 (close)" Use words for the gap: 0.0-0.1 "close", 0.1-0.3 "drift", 0.3+ "mismatch". + When `inferred.contributing_events[dim]` is below 3, show "not enough + observed answers yet" for that dimension instead of an observed value or gap word. This display gate is intentionally lower than the E1 **promotion gate** (90+ days stable across 3+ skills, per `docs/designs/PLAN_TUNING_V0.md`). @@ -422,6 +424,9 @@ Parse the JSON. For each dimension where both declared and inferred exist: Consider updating your declared value, or reflect on whether your behavior is actually what you want." +For each dimension in `low_evidence` (fewer than 3 observed answers moved it), +say "not enough observed answers yet" instead of a gap. + Never auto-update declared based on the gap. In v1 the gap is reporting only — the user decides whether declared is wrong or behavior is wrong. diff --git a/test/gstack-developer-profile.test.ts b/test/gstack-developer-profile.test.ts index df23a9c3b..cffaadebc 100644 --- a/test/gstack-developer-profile.test.ts +++ b/test/gstack-developer-profile.test.ts @@ -622,6 +622,98 @@ describe('gstack-developer-profile --check-mismatch', () => { expect(r.stdout).toContain('scope_appetite'); expect(r.stdout).toContain('autonomy'); }); + + const deriveWithOneSignal = () => { + logQuestion({ skill: 'plan-eng-review', question_id: 'plan-eng-review-arch-finding', question_summary: 'arch?', user_choice: 'fix-now', session_id: 's0' }); + for (let i = 1; i < 13; i++) { + logQuestion({ skill: 'plan-eng-review', question_id: `adhoc-unregistered-${i}`, question_summary: 'x', user_choice: 'anything', session_id: `s${i}` }); + } + expect(runDev('--derive').status).toBe(0); + const file = path.join(tmpHome, 'developer-profile.json'); + const p = readProfile() as any; + expect(p.inferred.contributing_events.scope_appetite).toBe(0); + p.declared = { scope_appetite: 0.9 }; + fs.writeFileSync(file, JSON.stringify(p)); + }; + + test('a declared dimension with no observed evidence is not flagged (#3436)', () => { + deriveWithOneSignal(); + const r = runDev('--check-mismatch'); + expect(r.status).toBe(0); + expect(r.stdout).not.toContain('scope_appetite: declared'); + expect(r.stdout).toContain('scope_appetite: no observed evidence'); + }); + + test('--gap leaves out a dimension with no observed evidence (#3436)', () => { + deriveWithOneSignal(); + const out = JSON.parse(runDev('--gap').stdout); + expect(out.gap.scope_appetite).toBeUndefined(); + expect(out.low_evidence).toEqual({ scope_appetite: 0 }); + }); + + test('a dimension with 1 or 2 contributing events reports too little evidence', () => { + runDev('--read'); + const file = path.join(tmpHome, 'developer-profile.json'); + const p = readProfile() as any; + p.declared = { scope_appetite: 0.9 }; + p.inferred.values.scope_appetite = 0.45; + p.inferred.sample_size = 10; + p.inferred.contributing_events = { scope_appetite: 2 }; + fs.writeFileSync(file, JSON.stringify(p)); + const r = runDev('--check-mismatch'); + expect(r.stdout).toContain('MISMATCH: none'); + expect(r.stdout).toContain('scope_appetite: too little evidence (2 of 3 events)'); + expect(JSON.parse(runDev('--gap').stdout).low_evidence).toEqual({ scope_appetite: 2 }); + }); + + test('a dimension with 3 contributing events is compared', () => { + runDev('--read'); + const file = path.join(tmpHome, 'developer-profile.json'); + const p = readProfile() as any; + p.declared = { scope_appetite: 0.9 }; + p.inferred.values.scope_appetite = 0.45; + p.inferred.sample_size = 10; + p.inferred.contributing_events = { scope_appetite: 3 }; + fs.writeFileSync(file, JSON.stringify(p)); + expect(runDev('--check-mismatch').stdout).toContain('scope_appetite: declared 0.9 vs inferred 0.45 (gap 0.45)'); + expect(JSON.parse(runDev('--gap').stdout).gap.scope_appetite).toBeCloseTo(0.45, 2); + }); + + test('three same-day answers that agree with the declaration are not a mismatch', () => { + for (let i = 0; i < 3; i++) { + logQuestion({ skill: 'plan-ceo-review', question_id: 'plan-ceo-review-mode', question_summary: 'mode?', user_choice: 'expand', session_id: `s${i}`, ts: '2026-10-20T12:00:00Z' }); + } + expect(runDev('--derive').status).toBe(0); + const file = path.join(tmpHome, 'developer-profile.json'); + const p = readProfile() as any; + expect(p.inferred.contributing_events.scope_appetite).toBe(3); + p.declared = { scope_appetite: 0.9 }; + fs.writeFileSync(file, JSON.stringify(p)); + const r = runDev('--check-mismatch'); + expect(r.stdout).toContain('not enough data'); + expect(r.stdout).not.toContain('scope_appetite: declared'); + }); + + test('derive stores the gap for a dimension with 3 contributing events', () => { + for (let i = 0; i < 3; i++) { + logQuestion({ skill: 'plan-eng-review', question_id: 'plan-eng-review-arch-finding', question_summary: 'arch?', user_choice: 'fix-now', session_id: `s${i}`, ts: `2026-10-2${i}T12:00:00Z` }); + } + expect(runDev('--derive').status).toBe(0); + const file = path.join(tmpHome, 'developer-profile.json'); + const p = readProfile() as any; + expect(p.inferred.contributing_events.architecture_care).toBe(3); + p.declared = { architecture_care: 0.1 }; + fs.writeFileSync(file, JSON.stringify(p)); + expect(runDev('--derive').status).toBe(0); + expect((readProfile() as any).gap.architecture_care).toBeGreaterThan(0.3); + }); + + test('derive stores no gap for a dimension without evidence', () => { + deriveWithOneSignal(); + expect(runDev('--derive').status).toBe(0); + const p = readProfile() as any; + expect(p.gap.scope_appetite).toBeUndefined(); + }); }); // -----------------------------------------------------------------------