feat: benchmarks + recommendations edge functions

- community-benchmarks: computes per-skill median/p25/p75 duration,
  total runs, and success rate from last 30 days of telemetry events.
  Upserts into community_benchmarks table, cached 1 hour.
- community-recommendations: co-occurrence-based skill suggestions
  ("used by X% of /qa users"). Cached 24 hours.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-03-19 22:54:37 -07:00
parent 7400d87db2
commit 3330d97b57
2 changed files with 214 additions and 0 deletions
@@ -0,0 +1,108 @@
// gstack community-benchmarks edge function
// Computes per-skill duration stats from telemetry_events (last 30 days).
// Upserts results into community_benchmarks table.
// Cached for 1 hour via Cache-Control header.
import { createClient } from "https://esm.sh/@supabase/supabase-js@2";
Deno.serve(async () => {
const supabase = createClient(
Deno.env.get("SUPABASE_URL") ?? "",
Deno.env.get("SUPABASE_SERVICE_ROLE_KEY") ?? ""
);
try {
const thirtyDaysAgo = new Date(
Date.now() - 30 * 24 * 60 * 60 * 1000
).toISOString();
// Fetch all skill_run events with duration from last 30 days
const { data: events, error } = await supabase
.from("telemetry_events")
.select("skill, duration_s, outcome")
.eq("event_type", "skill_run")
.not("duration_s", "is", null)
.not("skill", "is", null)
.gte("event_timestamp", thirtyDaysAgo)
.order("skill")
.limit(10000);
if (error) throw error;
if (!events || events.length === 0) {
return new Response(JSON.stringify([]), {
status: 200,
headers: {
"Content-Type": "application/json",
"Cache-Control": "public, max-age=3600",
},
});
}
// Group by skill and compute stats
const skillMap: Record<
string,
{ durations: number[]; successes: number; total: number }
> = {};
for (const event of events) {
if (!event.skill || event.duration_s == null) continue;
if (!skillMap[event.skill]) {
skillMap[event.skill] = { durations: [], successes: 0, total: 0 };
}
skillMap[event.skill].durations.push(Number(event.duration_s));
skillMap[event.skill].total++;
if (event.outcome === "success") {
skillMap[event.skill].successes++;
}
}
const benchmarks = Object.entries(skillMap)
.filter(([skill]) => !skill.startsWith("_")) // skip internal skills
.map(([skill, data]) => {
const sorted = data.durations.sort((a, b) => a - b);
const len = sorted.length;
const percentile = (p: number) => {
const idx = Math.floor((p / 100) * (len - 1));
return sorted[idx] ?? 0;
};
return {
skill,
median_duration_s: percentile(50),
p25_duration_s: percentile(25),
p75_duration_s: percentile(75),
total_runs: data.total,
success_rate:
data.total > 0
? Math.round((data.successes / data.total) * 1000) / 10
: 0,
updated_at: new Date().toISOString(),
};
});
// Upsert into community_benchmarks table
if (benchmarks.length > 0) {
const { error: upsertError } = await supabase
.from("community_benchmarks")
.upsert(benchmarks, { onConflict: "skill" });
if (upsertError) {
console.error("Upsert error:", upsertError);
}
}
return new Response(JSON.stringify(benchmarks), {
status: 200,
headers: {
"Content-Type": "application/json",
"Cache-Control": "public, max-age=3600",
},
});
} catch (err) {
console.error("Benchmarks error:", err);
return new Response(JSON.stringify([]), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
});
@@ -0,0 +1,106 @@
// gstack community-recommendations edge function
// Returns skill recommendations based on co-occurrence patterns.
// Input: ?skills=qa,ship (user's top skills as comma-separated query param)
// Output: top 3 recommended skills the user hasn't tried yet.
// Cached for 24 hours via Cache-Control header.
import { createClient } from "https://esm.sh/@supabase/supabase-js@2";
Deno.serve(async (req) => {
const supabase = createClient(
Deno.env.get("SUPABASE_URL") ?? "",
Deno.env.get("SUPABASE_SERVICE_ROLE_KEY") ?? ""
);
try {
const url = new URL(req.url);
const userSkills = (url.searchParams.get("skills") ?? "")
.split(",")
.map((s) => s.trim())
.filter(Boolean);
if (userSkills.length === 0) {
return new Response(JSON.stringify({ recommendations: [] }), {
status: 200,
headers: {
"Content-Type": "application/json",
"Cache-Control": "public, max-age=86400",
},
});
}
// Query skill_sequences for co-occurring skills
const { data: sequences, error } = await supabase
.from("skill_sequences")
.select("skill_a, skill_b, co_occurrences")
.in("skill_a", userSkills)
.order("co_occurrences", { ascending: false })
.limit(50);
if (error) throw error;
// Find skills the user hasn't used yet, ranked by co-occurrence
const userSkillSet = new Set(userSkills);
const recommendations: Record<
string,
{ co_occurrences: number; paired_with: string[] }
> = {};
for (const seq of sequences ?? []) {
if (userSkillSet.has(seq.skill_b)) continue; // already used
if (seq.skill_b.startsWith("_")) continue; // skip internal
if (!recommendations[seq.skill_b]) {
recommendations[seq.skill_b] = {
co_occurrences: 0,
paired_with: [],
};
}
recommendations[seq.skill_b].co_occurrences += seq.co_occurrences;
recommendations[seq.skill_b].paired_with.push(seq.skill_a);
}
// Also get total run counts for percentage calculation
const { data: benchmarks } = await supabase
.from("community_benchmarks")
.select("skill, total_runs");
const totalBySkill: Record<string, number> = {};
for (const b of benchmarks ?? []) {
totalBySkill[b.skill] = b.total_runs;
}
// Build top 3 recommendations
const sorted = Object.entries(recommendations)
.sort(([, a], [, b]) => b.co_occurrences - a.co_occurrences)
.slice(0, 3)
.map(([skill, data]) => {
const pairedSkill = data.paired_with[0];
const pairedTotal = totalBySkill[pairedSkill] ?? 0;
const pct =
pairedTotal > 0
? Math.round((data.co_occurrences / pairedTotal) * 100)
: 0;
return {
skill,
reason: `used by ${pct}% of /${pairedSkill} users`,
co_occurrences: data.co_occurrences,
};
});
return new Response(JSON.stringify({ recommendations: sorted }), {
status: 200,
headers: {
"Content-Type": "application/json",
"Cache-Control": "public, max-age=86400",
},
});
} catch (err) {
console.error("Recommendations error:", err);
return new Response(JSON.stringify({ recommendations: [] }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
});