mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 14:06:50 +01:00
B10: plan-tune never reports CALIBRATED with zero signal events
Stats printed CALIBRATED: true from logged-answer counts alone, while every answer could miss SIGNAL_MAP and leave each dimension at its 0.5 seed (#2623). --derive now records inferred.signal_events, the answers that actually moved a dimension; calibration requires at least one, and the summary prints 'not calibrated: no recorded signals' verbatim when there are none (or says the count is unknown for a profile derived before this change). The psychographic redesign stays out of scope.
This commit is contained in:
1 parent
aa90e52af8
commit
11b1a96677
3 files changed
+65
-3
No files matched your search
@@ -353,6 +353,7 @@ do_derive() {
|
||||
const qids = new Set();
|
||||
const days = new Set();
|
||||
let count = 0;
|
||||
let signalEvents = 0;
|
||||
for (const line of lines) {
|
||||
let e;
|
||||
try { e = JSON.parse(line); } catch { continue; }
|
||||
@@ -362,8 +363,8 @@ do_derive() {
|
||||
qids.add(e.question_id);
|
||||
if (e.ts) days.add(String(e.ts).slice(0,10));
|
||||
const def = QUESTIONS[e.question_id];
|
||||
if (def && def.signal_key) {
|
||||
applySignal(totals, def.signal_key, e.user_choice);
|
||||
if (def && def.signal_key && applySignal(totals, def.signal_key, e.user_choice).length > 0) {
|
||||
signalEvents++;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -375,6 +376,9 @@ do_derive() {
|
||||
profile.inferred = {
|
||||
values,
|
||||
sample_size: count,
|
||||
// Events that actually moved a dimension (B10): sample_size counts every
|
||||
// logged answer, most of which match no signal and leave the 0.5 seed.
|
||||
signal_events: signalEvents,
|
||||
diversity: {
|
||||
skills_covered: skills.size,
|
||||
question_ids_covered: qids.size,
|
||||
|
||||
@@ -465,7 +465,11 @@ fi
|
||||
console.log('SKILLS_COVERED: ' + (d.skills_covered ?? 0));
|
||||
console.log('QUESTIONS_COVERED: ' + (d.question_ids_covered ?? 0));
|
||||
console.log('DAYS_SPAN: ' + (d.days_span ?? 0));
|
||||
console.log('CALIBRATED: ' + (p.inferred?.sample_size >= 20 && d.skills_covered >= 3 && d.question_ids_covered >= 8 && d.days_span >= 7));
|
||||
const signals = p.inferred?.signal_events;
|
||||
console.log('SIGNAL_EVENTS: ' + (signals ?? 'unknown'));
|
||||
console.log('CALIBRATED: ' + (signals > 0 && p.inferred?.sample_size >= 20 && d.skills_covered >= 3 && d.question_ids_covered >= 8 && d.days_span >= 7));
|
||||
if (signals === 0) console.log('CALIBRATION: not calibrated: no recorded signals');
|
||||
if (signals === undefined) console.log('CALIBRATION: not calibrated: signal count unknown (run gstack-developer-profile --derive)');
|
||||
"
|
||||
echo '---DISTILL---'
|
||||
~/.claude/skills/gstack/bin/gstack-distill-free-text --status
|
||||
@@ -473,6 +477,8 @@ echo '---DISTILL---'
|
||||
|
||||
Present as a compact summary with plain-English calibration status ("5 more
|
||||
events across 2 more skills and you'll be calibrated" or "you're calibrated").
|
||||
When `CALIBRATION:` is printed, report that line verbatim: logged answers that
|
||||
moved no dimension are not calibration, whatever the counts say.
|
||||
Surface the source breakdown so the user can see which capture paths are
|
||||
actually logging.
|
||||
|
||||
|
||||
@@ -671,3 +671,55 @@ function findAllTemplates(): string[] {
|
||||
walk(ROOT);
|
||||
return results;
|
||||
}
|
||||
|
||||
// B10 (#2623, honest-status half): Stats printed CALIBRATED: true from event
|
||||
// counts alone, while every logged answer could miss SIGNAL_MAP and leave each
|
||||
// dimension at its 0.5 seed. Calibration now needs at least one signal event.
|
||||
describe('B10: no CALIBRATED with zero signals', () => {
|
||||
const tmpl = fs.readFileSync(path.join(ROOT, 'plan-tune', 'SKILL.md.tmpl'), 'utf-8');
|
||||
const start = tmpl.indexOf('gstack-developer-profile --profile | bun -e "') + 'gstack-developer-profile --profile | bun -e "'.length;
|
||||
const script = tmpl.slice(start, tmpl.indexOf('\n"\necho \'---DISTILL---\'', start));
|
||||
const calibration = (inferred: object) => {
|
||||
const { spawnSync } = require('child_process');
|
||||
const r = spawnSync('bun', ['-e', script], { input: JSON.stringify({ inferred }), encoding: 'utf-8', timeout: 30_000 });
|
||||
expect(r.status).toBe(0);
|
||||
return r.stdout as string;
|
||||
};
|
||||
const enough = { sample_size: 40, diversity: { skills_covered: 5, question_ids_covered: 12, days_span: 9 } };
|
||||
|
||||
test('every count gate passing but zero signal events is not calibrated', () => {
|
||||
const out = calibration({ ...enough, signal_events: 0 });
|
||||
expect(out).toContain('CALIBRATED: false');
|
||||
expect(out).toContain('CALIBRATION: not calibrated: no recorded signals');
|
||||
});
|
||||
|
||||
test('a profile derived before signal counting is not reported calibrated', () => {
|
||||
expect(calibration(enough)).toContain('CALIBRATED: false');
|
||||
});
|
||||
|
||||
test('signal events plus the count gates still calibrate', () => {
|
||||
const out = calibration({ ...enough, signal_events: 7 });
|
||||
expect(out).toContain('CALIBRATED: true');
|
||||
expect(out).not.toContain('CALIBRATION:');
|
||||
});
|
||||
|
||||
test('--derive counts only answers that moved a dimension', () => {
|
||||
const tmpHome = fs.mkdtempSync(path.join(require('os').tmpdir(), 'gstack-b10-'));
|
||||
try {
|
||||
const { spawnSync } = require('child_process');
|
||||
const env = { ...process.env, GSTACK_HOME: tmpHome, GSTACK_QUESTION_LOG_NO_DERIVE: '1' };
|
||||
const logBin = path.join(ROOT, 'bin', 'gstack-question-log');
|
||||
const devBin = path.join(ROOT, 'bin', 'gstack-developer-profile');
|
||||
for (const [i, choice] of ['expand', 'A) Expand scope (Recommended)', 'something-unmapped'].entries()) {
|
||||
const r = spawnSync(logBin, [JSON.stringify({ skill: 'plan-ceo-review', question_id: 'plan-ceo-review-mode', question_summary: 'mode?', user_choice: choice, session_id: `s${i}`, ts: `2026-04-0${i + 1}T10:00:00Z` })], { env, cwd: ROOT, encoding: 'utf-8', timeout: 30_000 });
|
||||
expect(r.status).toBe(0);
|
||||
}
|
||||
expect(spawnSync(devBin, ['--derive'], { env, cwd: ROOT, encoding: 'utf-8', timeout: 30_000 }).status).toBe(0);
|
||||
const p = JSON.parse(spawnSync(devBin, ['--profile'], { env, cwd: ROOT, encoding: 'utf-8', timeout: 30_000 }).stdout);
|
||||
expect(p.inferred.sample_size).toBe(3);
|
||||
expect(p.inferred.signal_events).toBe(1);
|
||||
} finally {
|
||||
fs.rmSync(tmpHome, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user