From 4833b1df1455e540d2341e5f1110e3f9086a98c2 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Wed, 18 Mar 2026 23:58:18 -0700 Subject: [PATCH] feat: add telemetry preamble injection + opt-in prompt + epilogue Extends generatePreamble() with telemetry start block (config read, timer, session ID, .pending marker), opt-in prompt (gated by .telemetry-prompted), and epilogue instructions for Claude to log events after skill completion. Adds 5 telemetry tests. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/gen-skill-docs.ts | 60 ++++++++++++++++++++++++++++++++++++- test/gen-skill-docs.test.ts | 47 +++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/scripts/gen-skill-docs.ts b/scripts/gen-skill-docs.ts index 687143c0..6055a890 100644 --- a/scripts/gen-skill-docs.ts +++ b/scripts/gen-skill-docs.ts @@ -105,12 +105,27 @@ touch ~/.gstack/sessions/"$PPID" _SESSIONS=$(find ~/.gstack/sessions -mmin -120 -type f 2>/dev/null | wc -l | tr -d ' ') find ~/.gstack/sessions -mmin +120 -type f -delete 2>/dev/null || true _CONTRIB=$(~/.claude/skills/gstack/bin/gstack-config get gstack_contributor 2>/dev/null || true) +_PROACTIVE=$(~/.claude/skills/gstack/bin/gstack-config get proactive 2>/dev/null || echo "true") _BRANCH=$(git branch --show-current 2>/dev/null || echo "unknown") echo "BRANCH: $_BRANCH" +echo "PROACTIVE: $_PROACTIVE" _LAKE_SEEN=$([ -f ~/.gstack/.completeness-intro-seen ] && echo "yes" || echo "no") echo "LAKE_INTRO: $_LAKE_SEEN" +_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || true) +_TEL_PROMPTED=$([ -f ~/.gstack/.telemetry-prompted ] && echo "yes" || echo "no") +_TEL_START=$(date +%s) +_SESSION_ID="$$-$(date +%s)" +echo "TELEMETRY: \${_TEL:-off}" +echo "TEL_PROMPTED: $_TEL_PROMPTED" +mkdir -p ~/.gstack/analytics +if [ -f ~/.gstack/analytics/.pending ]; then + ~/.claude/skills/gstack/bin/gstack-telemetry-log --event-type skill_run --skill _pending_finalize --outcome unknown 2>/dev/null || true +fi \`\`\` +If \`PROACTIVE\` is \`"false"\`, do not proactively suggest gstack skills — only invoke +them when the user explicitly asks. The user opted out of proactive suggestions. + If output shows \`UPGRADE_AVAILABLE \`: read \`~/.claude/skills/gstack/gstack-upgrade/SKILL.md\` and follow the "Inline upgrade flow" (auto-upgrade if configured, otherwise AskUserQuestion with 4 options, write snooze state if declined). If \`JUST_UPGRADED \`: tell user "Running gstack v{to} (just updated!)" and continue. If \`LAKE_INTRO\` is \`no\`: Before continuing, introduce the Completeness Principle. @@ -125,6 +140,27 @@ touch ~/.gstack/.completeness-intro-seen Only run \`open\` if the user says yes. Always run \`touch\` to mark as seen. This only happens once. +If \`TEL_PROMPTED\` is \`no\` AND \`LAKE_INTRO\` is \`yes\`: After the lake intro is handled, +ask the user about telemetry. Use AskUserQuestion: + +> gstack can share anonymous usage data (which skills you use, how long they take, crash info) +> to help improve the project. No code, file paths, or repo names are ever sent. +> Change anytime with \`gstack-config set telemetry off\`. + +Options: +- A) Yes, share anonymous data (recommended) +- B) No thanks + +If A: run \`~/.claude/skills/gstack/bin/gstack-config set telemetry anonymous\` +If B: run \`~/.claude/skills/gstack/bin/gstack-config set telemetry off\` + +Always run: +\`\`\`bash +touch ~/.gstack/.telemetry-prompted +\`\`\` + +This only happens once. If \`TEL_PROMPTED\` is \`yes\`, skip this entirely. + ## AskUserQuestion Format **ALWAYS follow this structure for every AskUserQuestion call:** @@ -197,7 +233,29 @@ Hey gstack team — ran into this while using /{skill-name}: **Date:** {YYYY-MM-DD} | **Version:** {gstack version} | **Skill:** /{skill} \`\`\` -Slug: lowercase, hyphens, max 60 chars (e.g. \`browse-js-no-await\`). Skip if file already exists. Max 3 reports per session. File inline and continue — don't stop the workflow. Tell user: "Filed gstack field report: {title}"`; +Slug: lowercase, hyphens, max 60 chars (e.g. \`browse-js-no-await\`). Skip if file already exists. Max 3 reports per session. File inline and continue — don't stop the workflow. Tell user: "Filed gstack field report: {title}" + +## Telemetry (run last) + +After the skill workflow completes (success, error, or abort), write the .pending marker +with the actual skill name, then log the telemetry event. Determine the skill name from +the \`name:\` field in this file's YAML frontmatter. Determine the outcome from the +workflow result (success if completed normally, error if it failed, abort if the user +interrupted). Run this bash: + +\`\`\`bash +_TEL_END=$(date +%s) +_TEL_DUR=$(( _TEL_END - _TEL_START )) +rm -f ~/.gstack/analytics/.pending 2>/dev/null || true +~/.claude/skills/gstack/bin/gstack-telemetry-log \\ + --skill "SKILL_NAME" --duration "$_TEL_DUR" --outcome "OUTCOME" \\ + --used-browse "USED_BROWSE" --session-id "$_SESSION_ID" 2>/dev/null & +\`\`\` + +Replace \`SKILL_NAME\` with the actual skill name from frontmatter, \`OUTCOME\` with +success/error/abort, and \`USED_BROWSE\` with true/false based on whether \`$B\` was used. +If you cannot determine the outcome, use "unknown". This runs in the background and +never blocks the user.`; } function generateBrowseSetup(): string { diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index 9dfd1a1c..9b3653f7 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -350,3 +350,50 @@ describe('REVIEW_DASHBOARD resolver', () => { expect(content).toContain('skip_eng_review'); }); }); + +describe('telemetry', () => { + test('generated SKILL.md contains telemetry start block', () => { + const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + expect(content).toContain('_TEL_START'); + expect(content).toContain('_SESSION_ID'); + expect(content).toContain('TELEMETRY:'); + expect(content).toContain('TEL_PROMPTED:'); + expect(content).toContain('gstack-config get telemetry'); + }); + + test('generated SKILL.md contains telemetry opt-in prompt', () => { + const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + expect(content).toContain('.telemetry-prompted'); + expect(content).toContain('anonymous usage data'); + expect(content).toContain('gstack-config set telemetry anonymous'); + expect(content).toContain('gstack-config set telemetry off'); + }); + + test('generated SKILL.md contains telemetry epilogue', () => { + const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + expect(content).toContain('Telemetry (run last)'); + expect(content).toContain('gstack-telemetry-log'); + expect(content).toContain('_TEL_END'); + expect(content).toContain('_TEL_DUR'); + expect(content).toContain('SKILL_NAME'); + expect(content).toContain('OUTCOME'); + }); + + test('generated SKILL.md contains pending marker handling', () => { + const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + expect(content).toContain('.pending'); + expect(content).toContain('_pending_finalize'); + }); + + test('telemetry blocks appear in all skill files that use PREAMBLE', () => { + const skills = ['qa', 'ship', 'review', 'plan-ceo-review', 'plan-eng-review', 'retro']; + for (const skill of skills) { + const skillPath = path.join(ROOT, skill, 'SKILL.md'); + if (fs.existsSync(skillPath)) { + const content = fs.readFileSync(skillPath, 'utf-8'); + expect(content).toContain('_TEL_START'); + expect(content).toContain('Telemetry (run last)'); + } + } + }); +});