From 78840c64a8b36a0208785f4d211dea18745e1974 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 16 Mar 2026 02:44:47 -0500 Subject: [PATCH] feat: add shared team dashboard, regression alerts, weekly digest edge functions Dashboard: Supabase edge function serving self-contained HTML with PKCE OAuth, 6 parallel client-side REST queries, SVG charts, dark theme, auto-refresh, who's-online from heartbeats. Public URL. Regression alert: webhook on eval_runs INSERT, 5-min cooldown dedup via alert_cooldowns, Slack notification on >5% pass rate drop. Weekly digest: pg_cron Monday 9am UTC, aggregates 7-day team data, Slack message with evals/ships/sessions/costs. 15 tests. Co-Authored-By: Claude Opus 4.6 (1M context) --- supabase/functions/dashboard/index.ts | 23 + supabase/functions/dashboard/ui.ts | 909 +++++++++++++++++++ supabase/functions/regression-alert/index.ts | 171 ++++ supabase/functions/weekly-digest/index.ts | 228 +++++ test/lib-dashboard-ui.test.ts | 82 ++ 5 files changed, 1413 insertions(+) create mode 100644 supabase/functions/dashboard/index.ts create mode 100644 supabase/functions/dashboard/ui.ts create mode 100644 supabase/functions/regression-alert/index.ts create mode 100644 supabase/functions/weekly-digest/index.ts create mode 100644 test/lib-dashboard-ui.test.ts diff --git a/supabase/functions/dashboard/index.ts b/supabase/functions/dashboard/index.ts new file mode 100644 index 00000000..1850ebd1 --- /dev/null +++ b/supabase/functions/dashboard/index.ts @@ -0,0 +1,23 @@ +/** + * Dashboard edge function — serves the team dashboard HTML. + * + * Public URL: https://.supabase.co/functions/v1/dashboard + * No auth required (the HTML page handles auth client-side via PKCE). + */ + +import { getDashboardHTML } from './ui.ts'; + +Deno.serve((_req: Request) => { + const supabaseUrl = Deno.env.get('SUPABASE_URL') ?? ''; + const anonKey = Deno.env.get('SUPABASE_ANON_KEY') ?? ''; + + const html = getDashboardHTML(supabaseUrl, anonKey); + + return new Response(html, { + status: 200, + headers: { + 'Content-Type': 'text/html; charset=utf-8', + 'Cache-Control': 'no-cache, no-store, must-revalidate', + }, + }); +}); diff --git a/supabase/functions/dashboard/ui.ts b/supabase/functions/dashboard/ui.ts new file mode 100644 index 00000000..24924424 --- /dev/null +++ b/supabase/functions/dashboard/ui.ts @@ -0,0 +1,909 @@ +/** + * Dashboard UI — self-contained HTML page for gstack's team engineering intelligence platform. + * + * Served by a Supabase edge function. All auth (PKCE), data fetching, and rendering + * happen client-side. The server only injects supabaseUrl and anonKey into the template. + */ + +export function getDashboardHTML(supabaseUrl: string, anonKey: string): string { + return ` + + + + +gstack Dashboard + + + + + + + + + + + + +`; +} diff --git a/supabase/functions/regression-alert/index.ts b/supabase/functions/regression-alert/index.ts new file mode 100644 index 00000000..25abcea4 --- /dev/null +++ b/supabase/functions/regression-alert/index.ts @@ -0,0 +1,171 @@ +/** + * Regression alert edge function. + * + * Trigger: Database webhook on eval_runs INSERT. + * Logic: Compare new run's pass rate against recent baseline. + * If >5% drop, POST to team's Slack webhook. + * Dedup via alert_cooldowns table (5-min window). + * + * Uses service_role key (bypasses RLS) — standard for webhooks. + */ + +import { createClient } from 'https://esm.sh/@supabase/supabase-js@2'; + +interface WebhookPayload { + type: 'INSERT'; + table: string; + record: { + id: string; + team_id: string; + repo_slug: string; + branch: string; + passed: number; + total_tests: number; + timestamp: string; + }; + schema: string; +} + +// --- Pure functions (testable) --- + +export function computePassRate(passed: number, total: number): number | null { + return total > 0 ? (passed / total) * 100 : null; +} + +export function shouldAlert( + currentRate: number | null, + baselineRate: number | null, + thresholdPct: number = 5, +): boolean { + if (currentRate === null || baselineRate === null) return false; + return baselineRate - currentRate > thresholdPct; +} + +export function formatSlackMessage(opts: { + repoSlug: string; + branch: string; + previousRate: number; + currentRate: number; +}): string { + const delta = opts.currentRate - opts.previousRate; + const arrow = delta < 0 ? 'regressed' : 'improved'; + return [ + `:warning: *Eval ${arrow}* on \`${opts.branch}\` (${opts.repoSlug})`, + `Pass rate: ${opts.previousRate.toFixed(0)}% → ${opts.currentRate.toFixed(0)}% (${delta > 0 ? '+' : ''}${delta.toFixed(0)}%)`, + ].join('\n'); +} + +// --- Main handler --- + +Deno.serve(async (req: Request) => { + try { + const payload: WebhookPayload = await req.json(); + const { record } = payload; + + if (!record || !record.team_id || !record.total_tests) { + return new Response('OK (skipped: missing fields)', { status: 200 }); + } + + const currentRate = computePassRate(record.passed, record.total_tests); + if (currentRate === null) { + return new Response('OK (skipped: total_tests=0)', { status: 200 }); + } + + const supabaseUrl = Deno.env.get('SUPABASE_URL')!; + const serviceKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY')!; + const supabase = createClient(supabaseUrl, serviceKey); + + // Check cooldown (5-min dedup) + const { data: cooldown } = await supabase + .from('alert_cooldowns') + .select('last_sent_at') + .eq('team_id', record.team_id) + .eq('repo_slug', record.repo_slug) + .eq('alert_type', 'regression') + .single(); + + if (cooldown?.last_sent_at) { + const cooldownMs = Date.now() - new Date(cooldown.last_sent_at).getTime(); + if (cooldownMs < 5 * 60 * 1000) { + return new Response('OK (cooldown active)', { status: 200 }); + } + } + + // Get previous runs for baseline + const { data: previousRuns } = await supabase + .from('eval_runs') + .select('passed, total_tests') + .eq('team_id', record.team_id) + .eq('repo_slug', record.repo_slug) + .neq('id', record.id) + .order('timestamp', { ascending: false }) + .limit(19); + + if (!previousRuns || previousRuns.length < 2) { + return new Response('OK (not enough history)', { status: 200 }); + } + + // Compute baseline pass rate + const rates = previousRuns + .map(r => computePassRate(r.passed, r.total_tests)) + .filter((r): r is number => r !== null); + + if (rates.length === 0) { + return new Response('OK (no valid baseline)', { status: 200 }); + } + + const baselineRate = rates.reduce((a, b) => a + b, 0) / rates.length; + + if (!shouldAlert(currentRate, baselineRate)) { + return new Response('OK (no regression)', { status: 200 }); + } + + // Get Slack webhook URL + const { data: setting } = await supabase + .from('team_settings') + .select('value') + .eq('team_id', record.team_id) + .eq('key', 'slack-webhook') + .single(); + + if (!setting?.value) { + console.log(`Regression detected but no Slack webhook configured for team ${record.team_id}`); + return new Response('OK (no webhook configured)', { status: 200 }); + } + + // Send Slack alert + const message = formatSlackMessage({ + repoSlug: record.repo_slug, + branch: record.branch, + previousRate: baselineRate, + currentRate, + }); + + const slackRes = await fetch(setting.value, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text: message }), + }); + + if (!slackRes.ok) { + console.error(`Slack webhook failed: ${slackRes.status} ${await slackRes.text()}`); + } + + // Update cooldown + await supabase + .from('alert_cooldowns') + .upsert({ + team_id: record.team_id, + repo_slug: record.repo_slug, + alert_type: 'regression', + last_sent_at: new Date().toISOString(), + }); + + console.log(`Regression alert sent: ${record.repo_slug} ${baselineRate.toFixed(0)}% → ${currentRate.toFixed(0)}%`); + + return new Response('OK (alert sent)', { status: 200 }); + } catch (err) { + console.error(`Regression alert error: ${err}`); + return new Response('OK (error logged)', { status: 200 }); + } +}); diff --git a/supabase/functions/weekly-digest/index.ts b/supabase/functions/weekly-digest/index.ts new file mode 100644 index 00000000..a2838bc1 --- /dev/null +++ b/supabase/functions/weekly-digest/index.ts @@ -0,0 +1,228 @@ +/** + * Weekly digest edge function. + * + * Trigger: pg_cron every Monday 9am UTC. + * Logic: For each team with digest_enabled=true, aggregate 7-day data + * and POST a summary to their Slack webhook. + * + * Uses service_role key (bypasses RLS) — standard for cron functions. + */ + +import { createClient } from 'https://esm.sh/@supabase/supabase-js@2'; + +// --- Pure functions (testable) --- + +interface DigestData { + teamSlug: string; + evalRuns: number; + evalPassRate: number | null; + evalPassRateDelta: number | null; + shipsByPerson: Array<{ email: string; count: number }>; + totalShips: number; + sessionCount: number; + topTools: Array<{ tool: string; count: number }>; + totalCost: number; +} + +export function formatDigestMessage(data: DigestData): string { + const lines: string[] = []; + lines.push(`:bar_chart: *Weekly gstack Digest* — ${data.teamSlug}`); + lines.push(''); + + // Evals + if (data.evalRuns > 0) { + let evalLine = `:white_check_mark: *Evals:* ${data.evalRuns} runs`; + if (data.evalPassRate !== null) { + evalLine += `, ${data.evalPassRate.toFixed(0)}% pass rate`; + if (data.evalPassRateDelta !== null) { + const sign = data.evalPassRateDelta >= 0 ? '+' : ''; + evalLine += ` (${sign}${data.evalPassRateDelta.toFixed(0)}% from last week)`; + } + } + lines.push(evalLine); + } + + // Ships + if (data.totalShips > 0) { + const people = data.shipsByPerson + .sort((a, b) => b.count - a.count) + .slice(0, 5) + .map(p => `${p.email.split('@')[0]}: ${p.count}`) + .join(', '); + lines.push(`:rocket: *Ships:* ${data.totalShips} PRs (${people})`); + } + + // Sessions + if (data.sessionCount > 0) { + let sessionLine = `:robot_face: *AI Sessions:* ${data.sessionCount}`; + if (data.topTools.length > 0) { + const tools = data.topTools.slice(0, 5).map(t => `${t.tool}(${t.count})`).join(', '); + sessionLine += ` — top tools: ${tools}`; + } + lines.push(sessionLine); + } + + // Cost + if (data.totalCost > 0) { + lines.push(`:moneybag: *Eval spend:* $${data.totalCost.toFixed(2)}`); + } + + // Quiet week fallback + if (data.evalRuns === 0 && data.totalShips === 0 && data.sessionCount === 0) { + lines.push('_Quiet week — no evals, ships, or sessions recorded._'); + } + + return lines.join('\n'); +} + +// --- Main handler --- + +Deno.serve(async (_req: Request) => { + try { + const supabaseUrl = Deno.env.get('SUPABASE_URL')!; + const serviceKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY')!; + const supabase = createClient(supabaseUrl, serviceKey); + + const weekAgo = new Date(Date.now() - 7 * 86_400_000).toISOString(); + const twoWeeksAgo = new Date(Date.now() - 14 * 86_400_000).toISOString(); + + // Find all teams with digest enabled + const { data: digestSettings } = await supabase + .from('team_settings') + .select('team_id, value') + .eq('key', 'digest-enabled') + .eq('value', 'true'); + + if (!digestSettings || digestSettings.length === 0) { + console.log('No teams have digest enabled'); + return new Response('OK (no teams)', { status: 200 }); + } + + let sentCount = 0; + + for (const setting of digestSettings) { + const teamId = setting.team_id; + + // Get Slack webhook + const { data: webhookSetting } = await supabase + .from('team_settings') + .select('value') + .eq('team_id', teamId) + .eq('key', 'slack-webhook') + .single(); + + if (!webhookSetting?.value) { + console.log(`Team ${teamId}: digest enabled but no Slack webhook`); + continue; + } + + // Get team slug + const { data: team } = await supabase + .from('teams') + .select('slug') + .eq('id', teamId) + .single(); + + // Fetch this week's data + const [evalRes, shipRes, sessionRes] = await Promise.all([ + supabase.from('eval_runs') + .select('passed, total_tests, total_cost_usd, user_id') + .eq('team_id', teamId) + .gte('timestamp', weekAgo), + supabase.from('ship_logs') + .select('user_id, email') + .eq('team_id', teamId) + .gte('created_at', weekAgo), + supabase.from('session_transcripts') + .select('tools_used') + .eq('team_id', teamId) + .gte('started_at', weekAgo), + ]); + + const evalRuns = evalRes.data || []; + const shipLogs = shipRes.data || []; + const sessions = sessionRes.data || []; + + // Compute pass rate + let passRate: number | null = null; + const validRuns = evalRuns.filter(r => r.total_tests > 0); + if (validRuns.length > 0) { + const totalPassed = validRuns.reduce((s, r) => s + r.passed, 0); + const totalTests = validRuns.reduce((s, r) => s + r.total_tests, 0); + passRate = totalTests > 0 ? (totalPassed / totalTests) * 100 : null; + } + + // Compute previous week's pass rate for delta + let passRateDelta: number | null = null; + const { data: prevWeekRuns } = await supabase + .from('eval_runs') + .select('passed, total_tests') + .eq('team_id', teamId) + .gte('timestamp', twoWeeksAgo) + .lt('timestamp', weekAgo); + + if (prevWeekRuns && prevWeekRuns.length > 0 && passRate !== null) { + const prevValid = prevWeekRuns.filter(r => r.total_tests > 0); + if (prevValid.length > 0) { + const prevPassed = prevValid.reduce((s, r) => s + r.passed, 0); + const prevTotal = prevValid.reduce((s, r) => s + r.total_tests, 0); + const prevRate = prevTotal > 0 ? (prevPassed / prevTotal) * 100 : null; + if (prevRate !== null) passRateDelta = passRate - prevRate; + } + } + + // Ships by person + const shipsByPerson = new Map(); + for (const log of shipLogs) { + const key = String(log.email || log.user_id || 'unknown'); + shipsByPerson.set(key, (shipsByPerson.get(key) || 0) + 1); + } + + // Top tools from sessions + const toolCounts = new Map(); + for (const s of sessions) { + const tools = (s.tools_used as string[]) || []; + for (const t of tools) { + toolCounts.set(t, (toolCounts.get(t) || 0) + 1); + } + } + + const totalCost = evalRuns.reduce((s, r) => s + (Number(r.total_cost_usd) || 0), 0); + + const digest: DigestData = { + teamSlug: team?.slug || 'unknown', + evalRuns: evalRuns.length, + evalPassRate: passRate, + evalPassRateDelta: passRateDelta, + shipsByPerson: [...shipsByPerson.entries()].map(([email, count]) => ({ email, count })), + totalShips: shipLogs.length, + sessionCount: sessions.length, + topTools: [...toolCounts.entries()] + .map(([tool, count]) => ({ tool, count })) + .sort((a, b) => b.count - a.count), + totalCost, + }; + + const message = formatDigestMessage(digest); + + // Send to Slack + const slackRes = await fetch(webhookSetting.value, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text: message }), + }); + + if (slackRes.ok) { + sentCount++; + console.log(`Digest sent for team ${team?.slug || teamId}`); + } else { + console.error(`Slack failed for team ${teamId}: ${slackRes.status}`); + } + } + + return new Response(`OK (${sentCount} digests sent)`, { status: 200 }); + } catch (err) { + console.error(`Weekly digest error: ${err}`); + return new Response('OK (error logged)', { status: 200 }); + } +}); diff --git a/test/lib-dashboard-ui.test.ts b/test/lib-dashboard-ui.test.ts new file mode 100644 index 00000000..6aff1713 --- /dev/null +++ b/test/lib-dashboard-ui.test.ts @@ -0,0 +1,82 @@ +/** + * Tests for dashboard UI HTML generation. + */ + +import { describe, test, expect } from 'bun:test'; +import { getDashboardHTML } from '../supabase/functions/dashboard/ui'; + +const SUPABASE_URL = 'https://test-project.supabase.co'; +const ANON_KEY = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test-anon-key'; + +describe('getDashboardHTML', () => { + const html = getDashboardHTML(SUPABASE_URL, ANON_KEY); + + test('returns valid HTML document', () => { + expect(html).toContain(''); + expect(html).toContain(''); + }); + + test('contains page title', () => { + expect(html).toContain('gstack Dashboard'); + }); + + test('embeds supabase URL', () => { + expect(html).toContain(SUPABASE_URL); + }); + + test('embeds anon key', () => { + expect(html).toContain(ANON_KEY); + }); + + test('contains login UI elements', () => { + expect(html).toContain('Sign in with GitHub'); + }); + + test('contains tab navigation', () => { + expect(html).toContain('Overview'); + expect(html).toContain('Evals'); + expect(html).toContain('Ships'); + expect(html).toContain('Costs'); + expect(html).toContain('Leaderboard'); + expect(html).toContain('QA'); + }); + + test('contains auto-refresh logic', () => { + expect(html).toContain('visibilitychange'); + expect(html).toContain('setInterval'); + }); + + test('contains PKCE auth code', () => { + expect(html).toContain('code_challenge'); + expect(html).toContain('code_verifier'); + }); + + test('uses textContent for XSS prevention', () => { + expect(html).toContain('textContent'); + }); + + test('contains dark theme styling', () => { + expect(html).toContain('#0a0a0a'); + }); + + test('contains SVG chart elements', () => { + expect(html).toContain('svg'); + }); + + test('fetches from eval_runs endpoint', () => { + expect(html).toContain('eval_runs'); + }); + + test('fetches from ship_logs endpoint', () => { + expect(html).toContain('ship_logs'); + }); + + test('fetches from sync_heartbeats for who\'s online', () => { + expect(html).toContain('sync_heartbeats'); + }); + + test('contains sign out functionality', () => { + expect(html).toContain('Sign out'); + }); +});