From 11b1a966772e2f12ec6555e32b432087ae9d3de5 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 21:08:15 +0000 Subject: [PATCH] B10: plan-tune never reports CALIBRATED with zero signal events Stats printed CALIBRATED: true from logged-answer counts alone, while every answer could miss SIGNAL_MAP and leave each dimension at its 0.5 seed (#2623). --derive now records inferred.signal_events, the answers that actually moved a dimension; calibration requires at least one, and the summary prints 'not calibrated: no recorded signals' verbatim when there are none (or says the count is unknown for a profile derived before this change). The psychographic redesign stays out of scope. --- bin/gstack-developer-profile | 8 ++++-- plan-tune/SKILL.md.tmpl | 8 +++++- test/plan-tune.test.ts | 52 ++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/bin/gstack-developer-profile b/bin/gstack-developer-profile index e5f4197f8..cb556d487 100755 --- a/bin/gstack-developer-profile +++ b/bin/gstack-developer-profile @@ -353,6 +353,7 @@ do_derive() { const qids = new Set(); const days = new Set(); let count = 0; + let signalEvents = 0; for (const line of lines) { let e; try { e = JSON.parse(line); } catch { continue; } @@ -362,8 +363,8 @@ do_derive() { qids.add(e.question_id); if (e.ts) days.add(String(e.ts).slice(0,10)); const def = QUESTIONS[e.question_id]; - if (def && def.signal_key) { - applySignal(totals, def.signal_key, e.user_choice); + if (def && def.signal_key && applySignal(totals, def.signal_key, e.user_choice).length > 0) { + signalEvents++; } } @@ -375,6 +376,9 @@ do_derive() { profile.inferred = { values, sample_size: count, + // Events that actually moved a dimension (B10): sample_size counts every + // logged answer, most of which match no signal and leave the 0.5 seed. + signal_events: signalEvents, diversity: { skills_covered: skills.size, question_ids_covered: qids.size, diff --git a/plan-tune/SKILL.md.tmpl b/plan-tune/SKILL.md.tmpl index a8790fd18..236b61ee2 100644 --- a/plan-tune/SKILL.md.tmpl +++ b/plan-tune/SKILL.md.tmpl @@ -465,7 +465,11 @@ fi console.log('SKILLS_COVERED: ' + (d.skills_covered ?? 0)); console.log('QUESTIONS_COVERED: ' + (d.question_ids_covered ?? 0)); console.log('DAYS_SPAN: ' + (d.days_span ?? 0)); - console.log('CALIBRATED: ' + (p.inferred?.sample_size >= 20 && d.skills_covered >= 3 && d.question_ids_covered >= 8 && d.days_span >= 7)); + const signals = p.inferred?.signal_events; + console.log('SIGNAL_EVENTS: ' + (signals ?? 'unknown')); + console.log('CALIBRATED: ' + (signals > 0 && p.inferred?.sample_size >= 20 && d.skills_covered >= 3 && d.question_ids_covered >= 8 && d.days_span >= 7)); + if (signals === 0) console.log('CALIBRATION: not calibrated: no recorded signals'); + if (signals === undefined) console.log('CALIBRATION: not calibrated: signal count unknown (run gstack-developer-profile --derive)'); " echo '---DISTILL---' ~/.claude/skills/gstack/bin/gstack-distill-free-text --status @@ -473,6 +477,8 @@ echo '---DISTILL---' Present as a compact summary with plain-English calibration status ("5 more events across 2 more skills and you'll be calibrated" or "you're calibrated"). +When `CALIBRATION:` is printed, report that line verbatim: logged answers that +moved no dimension are not calibration, whatever the counts say. Surface the source breakdown so the user can see which capture paths are actually logging. diff --git a/test/plan-tune.test.ts b/test/plan-tune.test.ts index 4eb9483ba..9de19a6c2 100644 --- a/test/plan-tune.test.ts +++ b/test/plan-tune.test.ts @@ -671,3 +671,55 @@ function findAllTemplates(): string[] { walk(ROOT); return results; } + +// B10 (#2623, honest-status half): Stats printed CALIBRATED: true from event +// counts alone, while every logged answer could miss SIGNAL_MAP and leave each +// dimension at its 0.5 seed. Calibration now needs at least one signal event. +describe('B10: no CALIBRATED with zero signals', () => { + const tmpl = fs.readFileSync(path.join(ROOT, 'plan-tune', 'SKILL.md.tmpl'), 'utf-8'); + const start = tmpl.indexOf('gstack-developer-profile --profile | bun -e "') + 'gstack-developer-profile --profile | bun -e "'.length; + const script = tmpl.slice(start, tmpl.indexOf('\n"\necho \'---DISTILL---\'', start)); + const calibration = (inferred: object) => { + const { spawnSync } = require('child_process'); + const r = spawnSync('bun', ['-e', script], { input: JSON.stringify({ inferred }), encoding: 'utf-8', timeout: 30_000 }); + expect(r.status).toBe(0); + return r.stdout as string; + }; + const enough = { sample_size: 40, diversity: { skills_covered: 5, question_ids_covered: 12, days_span: 9 } }; + + test('every count gate passing but zero signal events is not calibrated', () => { + const out = calibration({ ...enough, signal_events: 0 }); + expect(out).toContain('CALIBRATED: false'); + expect(out).toContain('CALIBRATION: not calibrated: no recorded signals'); + }); + + test('a profile derived before signal counting is not reported calibrated', () => { + expect(calibration(enough)).toContain('CALIBRATED: false'); + }); + + test('signal events plus the count gates still calibrate', () => { + const out = calibration({ ...enough, signal_events: 7 }); + expect(out).toContain('CALIBRATED: true'); + expect(out).not.toContain('CALIBRATION:'); + }); + + test('--derive counts only answers that moved a dimension', () => { + const tmpHome = fs.mkdtempSync(path.join(require('os').tmpdir(), 'gstack-b10-')); + try { + const { spawnSync } = require('child_process'); + const env = { ...process.env, GSTACK_HOME: tmpHome, GSTACK_QUESTION_LOG_NO_DERIVE: '1' }; + const logBin = path.join(ROOT, 'bin', 'gstack-question-log'); + const devBin = path.join(ROOT, 'bin', 'gstack-developer-profile'); + for (const [i, choice] of ['expand', 'A) Expand scope (Recommended)', 'something-unmapped'].entries()) { + const r = spawnSync(logBin, [JSON.stringify({ skill: 'plan-ceo-review', question_id: 'plan-ceo-review-mode', question_summary: 'mode?', user_choice: choice, session_id: `s${i}`, ts: `2026-04-0${i + 1}T10:00:00Z` })], { env, cwd: ROOT, encoding: 'utf-8', timeout: 30_000 }); + expect(r.status).toBe(0); + } + expect(spawnSync(devBin, ['--derive'], { env, cwd: ROOT, encoding: 'utf-8', timeout: 30_000 }).status).toBe(0); + const p = JSON.parse(spawnSync(devBin, ['--profile'], { env, cwd: ROOT, encoding: 'utf-8', timeout: 30_000 }).stdout); + expect(p.inferred.sample_size).toBe(3); + expect(p.inferred.signal_events).toBe(1); + } finally { + fs.rmSync(tmpHome, { recursive: true, force: true }); + } + }); +});