fix(plan-tune): compare declared and inferred only with 3+ observed answers (#3436)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

bin/gstack-developer-profile

@@ -54,6 +54,7 @@ case "$(uname -s)" in
 esac
 PROFILE_FILE="$GSTACK_HOME/developer-profile.json"
 LEGACY_FILE="$GSTACK_HOME/builder-profile.jsonl"
+MIN_EVIDENCE=3
 eval "$("$SCRIPT_DIR/gstack-slug" 2>/dev/null || true)"
 SLUG="${SLUG:-unknown}"
 
@@ -512,14 +513,19 @@ do_gap() {
     const p = JSON.parse(await Bun.stdin.text());
     const declared = p.declared || {};
     const inferred = (p.inferred && p.inferred.values) || {};
+    const contributing = p.inferred && p.inferred.contributing_events;
     const dims = ['scope_appetite','risk_tolerance','detail_preference','autonomy','architecture_care'];
     const gap = {};
+    const low_evidence = {};
     for (const d of dims) {
-      if (declared[d] !== undefined && inferred[d] !== undefined) {
-        gap[d] = +(Math.abs(declared[d] - inferred[d])).toFixed(3);
+      if (declared[d] === undefined || inferred[d] === undefined) continue;
+      if (contributing && (contributing[d] || 0) < $MIN_EVIDENCE) {
+        low_evidence[d] = contributing[d] || 0;
+        continue;
       }
+      gap[d] = +(Math.abs(declared[d] - inferred[d])).toFixed(3);
     }
-    console.log(JSON.stringify({ declared, inferred, gap }, null, 2));
+    console.log(JSON.stringify({ declared, inferred, gap, low_evidence }, null, 2));
   "
 }
 
@@ -657,6 +663,7 @@ do_derive() {
       // Recompute gap.
       const gap = {};
       for (const d of Object.keys(values)) {
+        if ((contributing[d] || 0) < $MIN_EVIDENCE) continue;
         if (profile.declared && profile.declared[d] !== undefined) {
           gap[d] = +(Math.abs(profile.declared[d] - values[d])).toFixed(3);
         }
@@ -727,18 +734,23 @@ do_check_mismatch() {
     const declared = p.declared || {};
     const inferred = (p.inferred && p.inferred.values) || {};
     const sampleSize = (p.inferred && p.inferred.sample_size) || 0;
-    const diversity = (p.inferred && p.inferred.diversity) || {};
+    const contributing = p.inferred && p.inferred.contributing_events;
 
-    // Require enough data before reporting mismatch.
-    if (sampleSize < 10) {
+    if (!contributing && sampleSize < 10) {
       console.log('MISMATCH: not enough data (' + sampleSize + ' events; need 10+)');
       process.exit(0);
     }
 
     const THRESHOLD = 0.3;
     const flagged = [];
+    const thin = [];
     for (const d of Object.keys(declared)) {
       if (inferred[d] === undefined) continue;
+      const n = contributing ? (contributing[d] || 0) : undefined;
+      if (n !== undefined && n < $MIN_EVIDENCE) {
+        thin.push('  ' + d + ': ' + (n === 0 ? 'no observed evidence' : 'too little evidence (' + n + ' of $MIN_EVIDENCE events)'));
+        continue;
+      }
       const gap = Math.abs(declared[d] - inferred[d]);
       if (gap > THRESHOLD) {
         flagged.push({ dim: d, declared: declared[d], inferred: inferred[d], gap: +gap.toFixed(3) });
@@ -753,6 +765,7 @@ do_check_mismatch() {
         console.log('  ' + f.dim + ': declared ' + f.declared + ' vs inferred ' + f.inferred + ' (gap ' + f.gap + ')');
       }
     }
+    for (const line of thin) console.log(line);
   "
 }
 

plan-tune/SKILL.md

@@ -491,6 +491,9 @@ Parse the JSON. For each dimension where both declared and inferred exist:
   Consider updating your declared value, or reflect on whether your behavior
   is actually what you want."
 
+For each dimension in `low_evidence` (fewer than 3 observed answers moved it),
+say "not enough observed answers yet" instead of a gap.
+
 Never auto-update declared based on the gap. In v1 the gap is reporting only —
 the user decides whether declared is wrong or behavior is wrong.
 

plan-tune/SKILL.md.tmpl

@@ -422,6 +422,9 @@ Parse the JSON. For each dimension where both declared and inferred exist:
   Consider updating your declared value, or reflect on whether your behavior
   is actually what you want."
 
+For each dimension in `low_evidence` (fewer than 3 observed answers moved it),
+say "not enough observed answers yet" instead of a gap.
+
 Never auto-update declared based on the gap. In v1 the gap is reporting only —
 the user decides whether declared is wrong or behavior is wrong.
 

test/gstack-developer-profile.test.ts

@@ -622,6 +622,69 @@ describe('gstack-developer-profile --check-mismatch', () => {
     expect(r.stdout).toContain('scope_appetite');
     expect(r.stdout).toContain('autonomy');
   });
+
+  const deriveWithOneSignal = () => {
+    logQuestion({ skill: 'plan-eng-review', question_id: 'plan-eng-review-arch-finding', question_summary: 'arch?', user_choice: 'fix-now', session_id: 's0' });
+    for (let i = 1; i < 13; i++) {
+      logQuestion({ skill: 'plan-eng-review', question_id: `adhoc-unregistered-${i}`, question_summary: 'x', user_choice: 'anything', session_id: `s${i}` });
+    }
+    expect(runDev('--derive').status).toBe(0);
+    const file = path.join(tmpHome, 'developer-profile.json');
+    const p = readProfile() as any;
+    expect(p.inferred.contributing_events.scope_appetite).toBe(0);
+    p.declared = { scope_appetite: 0.9 };
+    fs.writeFileSync(file, JSON.stringify(p));
+  };
+
+  test('a declared dimension with no observed evidence is not flagged (#3436)', () => {
+    deriveWithOneSignal();
+    const r = runDev('--check-mismatch');
+    expect(r.status).toBe(0);
+    expect(r.stdout).not.toContain('scope_appetite: declared');
+    expect(r.stdout).toContain('scope_appetite: no observed evidence');
+  });
+
+  test('--gap leaves out a dimension with no observed evidence (#3436)', () => {
+    deriveWithOneSignal();
+    const out = JSON.parse(runDev('--gap').stdout);
+    expect(out.gap.scope_appetite).toBeUndefined();
+    expect(out.low_evidence).toEqual({ scope_appetite: 0 });
+  });
+
+  test('a dimension with 1 or 2 contributing events reports too little evidence', () => {
+    runDev('--read');
+    const file = path.join(tmpHome, 'developer-profile.json');
+    const p = readProfile() as any;
+    p.declared = { scope_appetite: 0.9 };
+    p.inferred.values.scope_appetite = 0.45;
+    p.inferred.sample_size = 2;
+    p.inferred.contributing_events = { scope_appetite: 2 };
+    fs.writeFileSync(file, JSON.stringify(p));
+    const r = runDev('--check-mismatch');
+    expect(r.stdout).toContain('MISMATCH: none');
+    expect(r.stdout).toContain('scope_appetite: too little evidence (2 of 3 events)');
+    expect(JSON.parse(runDev('--gap').stdout).low_evidence).toEqual({ scope_appetite: 2 });
+  });
+
+  test('a dimension with 3 contributing events is compared', () => {
+    runDev('--read');
+    const file = path.join(tmpHome, 'developer-profile.json');
+    const p = readProfile() as any;
+    p.declared = { scope_appetite: 0.9 };
+    p.inferred.values.scope_appetite = 0.45;
+    p.inferred.sample_size = 3;
+    p.inferred.contributing_events = { scope_appetite: 3 };
+    fs.writeFileSync(file, JSON.stringify(p));
+    expect(runDev('--check-mismatch').stdout).toContain('scope_appetite: declared 0.9 vs inferred 0.45 (gap 0.45)');
+    expect(JSON.parse(runDev('--gap').stdout).gap.scope_appetite).toBeCloseTo(0.45, 2);
+  });
+
+  test('derive stores no gap for a dimension without evidence', () => {
+    deriveWithOneSignal();
+    expect(runDev('--derive').status).toBe(0);
+    const p = readProfile() as any;
+    expect(p.gap.scope_appetite).toBeUndefined();
+  });
 });
 
 // -----------------------------------------------------------------------