From 8e2d8893311c8328a8b07e165a85ae79858babdc Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 20:57:54 +0000 Subject: [PATCH] fix(plan-tune): compare declared and inferred only with 3+ observed answers (#3436) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-developer-profile | 25 ++++++++--- plan-tune/SKILL.md | 3 ++ plan-tune/SKILL.md.tmpl | 3 ++ test/gstack-developer-profile.test.ts | 63 +++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 6 deletions(-) diff --git a/bin/gstack-developer-profile b/bin/gstack-developer-profile index bc489c5c1..f2846b0cf 100755 --- a/bin/gstack-developer-profile +++ b/bin/gstack-developer-profile @@ -54,6 +54,7 @@ case "$(uname -s)" in esac PROFILE_FILE="$GSTACK_HOME/developer-profile.json" LEGACY_FILE="$GSTACK_HOME/builder-profile.jsonl" +MIN_EVIDENCE=3 eval "$("$SCRIPT_DIR/gstack-slug" 2>/dev/null || true)" SLUG="${SLUG:-unknown}" @@ -512,14 +513,19 @@ do_gap() { const p = JSON.parse(await Bun.stdin.text()); const declared = p.declared || {}; const inferred = (p.inferred && p.inferred.values) || {}; + const contributing = p.inferred && p.inferred.contributing_events; const dims = ['scope_appetite','risk_tolerance','detail_preference','autonomy','architecture_care']; const gap = {}; + const low_evidence = {}; for (const d of dims) { - if (declared[d] !== undefined && inferred[d] !== undefined) { - gap[d] = +(Math.abs(declared[d] - inferred[d])).toFixed(3); + if (declared[d] === undefined || inferred[d] === undefined) continue; + if (contributing && (contributing[d] || 0) < $MIN_EVIDENCE) { + low_evidence[d] = contributing[d] || 0; + continue; } + gap[d] = +(Math.abs(declared[d] - inferred[d])).toFixed(3); } - console.log(JSON.stringify({ declared, inferred, gap }, null, 2)); + console.log(JSON.stringify({ declared, inferred, gap, low_evidence }, null, 2)); " } @@ -657,6 +663,7 @@ do_derive() { // Recompute gap. const gap = {}; for (const d of Object.keys(values)) { + if ((contributing[d] || 0) < $MIN_EVIDENCE) continue; if (profile.declared && profile.declared[d] !== undefined) { gap[d] = +(Math.abs(profile.declared[d] - values[d])).toFixed(3); } @@ -727,18 +734,23 @@ do_check_mismatch() { const declared = p.declared || {}; const inferred = (p.inferred && p.inferred.values) || {}; const sampleSize = (p.inferred && p.inferred.sample_size) || 0; - const diversity = (p.inferred && p.inferred.diversity) || {}; + const contributing = p.inferred && p.inferred.contributing_events; - // Require enough data before reporting mismatch. - if (sampleSize < 10) { + if (!contributing && sampleSize < 10) { console.log('MISMATCH: not enough data (' + sampleSize + ' events; need 10+)'); process.exit(0); } const THRESHOLD = 0.3; const flagged = []; + const thin = []; for (const d of Object.keys(declared)) { if (inferred[d] === undefined) continue; + const n = contributing ? (contributing[d] || 0) : undefined; + if (n !== undefined && n < $MIN_EVIDENCE) { + thin.push(' ' + d + ': ' + (n === 0 ? 'no observed evidence' : 'too little evidence (' + n + ' of $MIN_EVIDENCE events)')); + continue; + } const gap = Math.abs(declared[d] - inferred[d]); if (gap > THRESHOLD) { flagged.push({ dim: d, declared: declared[d], inferred: inferred[d], gap: +gap.toFixed(3) }); @@ -753,6 +765,7 @@ do_check_mismatch() { console.log(' ' + f.dim + ': declared ' + f.declared + ' vs inferred ' + f.inferred + ' (gap ' + f.gap + ')'); } } + for (const line of thin) console.log(line); " } diff --git a/plan-tune/SKILL.md b/plan-tune/SKILL.md index 86b2cbdb7..eb13308fe 100644 --- a/plan-tune/SKILL.md +++ b/plan-tune/SKILL.md @@ -491,6 +491,9 @@ Parse the JSON. For each dimension where both declared and inferred exist: Consider updating your declared value, or reflect on whether your behavior is actually what you want." +For each dimension in `low_evidence` (fewer than 3 observed answers moved it), +say "not enough observed answers yet" instead of a gap. + Never auto-update declared based on the gap. In v1 the gap is reporting only — the user decides whether declared is wrong or behavior is wrong. diff --git a/plan-tune/SKILL.md.tmpl b/plan-tune/SKILL.md.tmpl index 748dfad62..319ed6faa 100644 --- a/plan-tune/SKILL.md.tmpl +++ b/plan-tune/SKILL.md.tmpl @@ -422,6 +422,9 @@ Parse the JSON. For each dimension where both declared and inferred exist: Consider updating your declared value, or reflect on whether your behavior is actually what you want." +For each dimension in `low_evidence` (fewer than 3 observed answers moved it), +say "not enough observed answers yet" instead of a gap. + Never auto-update declared based on the gap. In v1 the gap is reporting only — the user decides whether declared is wrong or behavior is wrong. diff --git a/test/gstack-developer-profile.test.ts b/test/gstack-developer-profile.test.ts index df23a9c3b..ebdf1587c 100644 --- a/test/gstack-developer-profile.test.ts +++ b/test/gstack-developer-profile.test.ts @@ -622,6 +622,69 @@ describe('gstack-developer-profile --check-mismatch', () => { expect(r.stdout).toContain('scope_appetite'); expect(r.stdout).toContain('autonomy'); }); + + const deriveWithOneSignal = () => { + logQuestion({ skill: 'plan-eng-review', question_id: 'plan-eng-review-arch-finding', question_summary: 'arch?', user_choice: 'fix-now', session_id: 's0' }); + for (let i = 1; i < 13; i++) { + logQuestion({ skill: 'plan-eng-review', question_id: `adhoc-unregistered-${i}`, question_summary: 'x', user_choice: 'anything', session_id: `s${i}` }); + } + expect(runDev('--derive').status).toBe(0); + const file = path.join(tmpHome, 'developer-profile.json'); + const p = readProfile() as any; + expect(p.inferred.contributing_events.scope_appetite).toBe(0); + p.declared = { scope_appetite: 0.9 }; + fs.writeFileSync(file, JSON.stringify(p)); + }; + + test('a declared dimension with no observed evidence is not flagged (#3436)', () => { + deriveWithOneSignal(); + const r = runDev('--check-mismatch'); + expect(r.status).toBe(0); + expect(r.stdout).not.toContain('scope_appetite: declared'); + expect(r.stdout).toContain('scope_appetite: no observed evidence'); + }); + + test('--gap leaves out a dimension with no observed evidence (#3436)', () => { + deriveWithOneSignal(); + const out = JSON.parse(runDev('--gap').stdout); + expect(out.gap.scope_appetite).toBeUndefined(); + expect(out.low_evidence).toEqual({ scope_appetite: 0 }); + }); + + test('a dimension with 1 or 2 contributing events reports too little evidence', () => { + runDev('--read'); + const file = path.join(tmpHome, 'developer-profile.json'); + const p = readProfile() as any; + p.declared = { scope_appetite: 0.9 }; + p.inferred.values.scope_appetite = 0.45; + p.inferred.sample_size = 2; + p.inferred.contributing_events = { scope_appetite: 2 }; + fs.writeFileSync(file, JSON.stringify(p)); + const r = runDev('--check-mismatch'); + expect(r.stdout).toContain('MISMATCH: none'); + expect(r.stdout).toContain('scope_appetite: too little evidence (2 of 3 events)'); + expect(JSON.parse(runDev('--gap').stdout).low_evidence).toEqual({ scope_appetite: 2 }); + }); + + test('a dimension with 3 contributing events is compared', () => { + runDev('--read'); + const file = path.join(tmpHome, 'developer-profile.json'); + const p = readProfile() as any; + p.declared = { scope_appetite: 0.9 }; + p.inferred.values.scope_appetite = 0.45; + p.inferred.sample_size = 3; + p.inferred.contributing_events = { scope_appetite: 3 }; + fs.writeFileSync(file, JSON.stringify(p)); + expect(runDev('--check-mismatch').stdout).toContain('scope_appetite: declared 0.9 vs inferred 0.45 (gap 0.45)'); + expect(JSON.parse(runDev('--gap').stdout).gap.scope_appetite).toBeCloseTo(0.45, 2); + }); + + test('derive stores no gap for a dimension without evidence', () => { + deriveWithOneSignal(); + expect(runDev('--derive').status).toBe(0); + const p = readProfile() as any; + expect(p.gap.scope_appetite).toBeUndefined(); + }); }); // -----------------------------------------------------------------------