B10: plan-tune never reports CALIBRATED with zero signal events

Stats printed CALIBRATED: true from logged-answer counts alone, while
every answer could miss SIGNAL_MAP and leave each dimension at its 0.5
seed (#2623). --derive now records inferred.signal_events, the answers
that actually moved a dimension; calibration requires at least one, and
the summary prints 'not calibrated: no recorded signals' verbatim when
there are none (or says the count is unknown for a profile derived
before this change). The psychographic redesign stays out of scope.
This commit is contained in:
garrytan committed 2026-10-03 21:08:15 +00:00
1 parent aa90e52af8
commit 11b1a96677
3 files changed
+65 -3

No files matched your search

+6 -2
View File
@@ -353,6 +353,7 @@ do_derive() {
const qids = new Set();
const days = new Set();
let count = 0;
let signalEvents = 0;
for (const line of lines) {
let e;
try { e = JSON.parse(line); } catch { continue; }
@@ -362,8 +363,8 @@ do_derive() {
qids.add(e.question_id);
if (e.ts) days.add(String(e.ts).slice(0,10));
const def = QUESTIONS[e.question_id];
if (def && def.signal_key) {
applySignal(totals, def.signal_key, e.user_choice);
if (def && def.signal_key && applySignal(totals, def.signal_key, e.user_choice).length > 0) {
signalEvents++;
}
}
@@ -375,6 +376,9 @@ do_derive() {
profile.inferred = {
values,
sample_size: count,
// Events that actually moved a dimension (B10): sample_size counts every
// logged answer, most of which match no signal and leave the 0.5 seed.
signal_events: signalEvents,
diversity: {
skills_covered: skills.size,
question_ids_covered: qids.size,
+7 -1
View File
@@ -465,7 +465,11 @@ fi
console.log('SKILLS_COVERED: ' + (d.skills_covered ?? 0));
console.log('QUESTIONS_COVERED: ' + (d.question_ids_covered ?? 0));
console.log('DAYS_SPAN: ' + (d.days_span ?? 0));
console.log('CALIBRATED: ' + (p.inferred?.sample_size >= 20 && d.skills_covered >= 3 && d.question_ids_covered >= 8 && d.days_span >= 7));
const signals = p.inferred?.signal_events;
console.log('SIGNAL_EVENTS: ' + (signals ?? 'unknown'));
console.log('CALIBRATED: ' + (signals > 0 && p.inferred?.sample_size >= 20 && d.skills_covered >= 3 && d.question_ids_covered >= 8 && d.days_span >= 7));
if (signals === 0) console.log('CALIBRATION: not calibrated: no recorded signals');
if (signals === undefined) console.log('CALIBRATION: not calibrated: signal count unknown (run gstack-developer-profile --derive)');
"
echo '---DISTILL---'
~/.claude/skills/gstack/bin/gstack-distill-free-text --status
@@ -473,6 +477,8 @@ echo '---DISTILL---'
Present as a compact summary with plain-English calibration status ("5 more
events across 2 more skills and you'll be calibrated" or "you're calibrated").
When `CALIBRATION:` is printed, report that line verbatim: logged answers that
moved no dimension are not calibration, whatever the counts say.
Surface the source breakdown so the user can see which capture paths are
actually logging.
+52
View File
@@ -671,3 +671,55 @@ function findAllTemplates(): string[] {
walk(ROOT);
return results;
}
// B10 (#2623, honest-status half): Stats printed CALIBRATED: true from event
// counts alone, while every logged answer could miss SIGNAL_MAP and leave each
// dimension at its 0.5 seed. Calibration now needs at least one signal event.
describe('B10: no CALIBRATED with zero signals', () => {
const tmpl = fs.readFileSync(path.join(ROOT, 'plan-tune', 'SKILL.md.tmpl'), 'utf-8');
const start = tmpl.indexOf('gstack-developer-profile --profile | bun -e "') + 'gstack-developer-profile --profile | bun -e "'.length;
const script = tmpl.slice(start, tmpl.indexOf('\n"\necho \'---DISTILL---\'', start));
const calibration = (inferred: object) => {
const { spawnSync } = require('child_process');
const r = spawnSync('bun', ['-e', script], { input: JSON.stringify({ inferred }), encoding: 'utf-8', timeout: 30_000 });
expect(r.status).toBe(0);
return r.stdout as string;
};
const enough = { sample_size: 40, diversity: { skills_covered: 5, question_ids_covered: 12, days_span: 9 } };
test('every count gate passing but zero signal events is not calibrated', () => {
const out = calibration({ ...enough, signal_events: 0 });
expect(out).toContain('CALIBRATED: false');
expect(out).toContain('CALIBRATION: not calibrated: no recorded signals');
});
test('a profile derived before signal counting is not reported calibrated', () => {
expect(calibration(enough)).toContain('CALIBRATED: false');
});
test('signal events plus the count gates still calibrate', () => {
const out = calibration({ ...enough, signal_events: 7 });
expect(out).toContain('CALIBRATED: true');
expect(out).not.toContain('CALIBRATION:');
});
test('--derive counts only answers that moved a dimension', () => {
const tmpHome = fs.mkdtempSync(path.join(require('os').tmpdir(), 'gstack-b10-'));
try {
const { spawnSync } = require('child_process');
const env = { ...process.env, GSTACK_HOME: tmpHome, GSTACK_QUESTION_LOG_NO_DERIVE: '1' };
const logBin = path.join(ROOT, 'bin', 'gstack-question-log');
const devBin = path.join(ROOT, 'bin', 'gstack-developer-profile');
for (const [i, choice] of ['expand', 'A) Expand scope (Recommended)', 'something-unmapped'].entries()) {
const r = spawnSync(logBin, [JSON.stringify({ skill: 'plan-ceo-review', question_id: 'plan-ceo-review-mode', question_summary: 'mode?', user_choice: choice, session_id: `s${i}`, ts: `2026-04-0${i + 1}T10:00:00Z` })], { env, cwd: ROOT, encoding: 'utf-8', timeout: 30_000 });
expect(r.status).toBe(0);
}
expect(spawnSync(devBin, ['--derive'], { env, cwd: ROOT, encoding: 'utf-8', timeout: 30_000 }).status).toBe(0);
const p = JSON.parse(spawnSync(devBin, ['--profile'], { env, cwd: ROOT, encoding: 'utf-8', timeout: 30_000 }).stdout);
expect(p.inferred.sample_size).toBe(3);
expect(p.inferred.signal_events).toBe(1);
} finally {
fs.rmSync(tmpHome, { recursive: true, force: true });
}
});
});