From 12cc21872d8614eafcbb9c28ac9ce79c5eae235c Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Thu, 19 Nov 2026 12:31:31 -0800 Subject: [PATCH] Telemetry backend only accepts writes through its own endpoint, and tests stop sending to it (GST-3234) (#4537) The telemetry backend now takes writes only through its validated endpoint, signed-in Supabase users can read or write nothing, and gstack's tests can no longer send telemetry to the live project. Why: a signed-in prober read two views on 2026-11-19, probes wrote about 2,000 junk rows through the open insert policy, and 41% of all skill-run rows were test-suite traffic. Nothing changes for users until the migrations and edge functions are deployed; the deploy order is under Deploy. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> Review-Finding: p1/red-team#1 reader=red-team Review-Finding: supabase/migrations/008_close_anon_writes.sql:9:published-number reader=core Review-Finding: p2/outside-voice#3 reader=outside-voice --- .../telemetry-data-fixes-and-lockdown.md | 1 + docs/privacy.md | 7 +- supabase/config.sh | 8 + supabase/functions/community-pulse/index.ts | 161 +----------- supabase/functions/telemetry-ingest/index.ts | 59 ++++- supabase/migrations/004_attack_telemetry.sql | 17 +- .../migrations/005_revoke_authenticated.sql | 43 ++++ .../006_live_traffic_and_pulse_summary.sql | 140 ++++++++++ .../007_drop_unused_installation_columns.sql | 28 ++ supabase/migrations/008_close_anon_writes.sql | 42 +++ supabase/verify-rls.sh | 240 ++++++++++-------- test-setup.ts | 5 + test/state-root-one-variable.test.ts | 1 + test/supabase-telemetry-schema.test.ts | 96 +++++++ test/telemetry-no-live-endpoint.test.ts | 126 +++++++++ 15 files changed, 704 insertions(+), 270 deletions(-) create mode 100644 changelog.d/telemetry-data-fixes-and-lockdown.md create mode 100644 supabase/migrations/005_revoke_authenticated.sql create mode 100644 supabase/migrations/006_live_traffic_and_pulse_summary.sql create mode 100644 supabase/migrations/007_drop_unused_installation_columns.sql create mode 100644 supabase/migrations/008_close_anon_writes.sql create mode 100644 test/supabase-telemetry-schema.test.ts create mode 100644 test/telemetry-no-live-endpoint.test.ts diff --git a/changelog.d/telemetry-data-fixes-and-lockdown.md b/changelog.d/telemetry-data-fixes-and-lockdown.md new file mode 100644 index 000000000..ba2d51749 --- /dev/null +++ b/changelog.d/telemetry-data-fixes-and-lockdown.md @@ -0,0 +1 @@ +**gstack's telemetry backend only accepts writes through its own validated endpoint, and the community dashboard now counts real use.** The public key can no longer read or write the telemetry tables, and a signed-in Supabase user is denied everything. Test runs never send telemetry, test traffic is filtered out of `gstack-community-dashboard`, and its weekly active count is now real installs instead of update checks. The ingest endpoint also keeps whether an event came from a test and stores prompt-injection detections, which it used to drop. Clients older than v0.11.16.0 (March 2026) no longer report usage. diff --git a/docs/privacy.md b/docs/privacy.md index f5e2dc082..edd0d397e 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -26,8 +26,11 @@ no, and `gstack-config set telemetry off` turns it off at any time. - **Where it goes:** [Supabase](https://supabase.com) (an open-source Firebase alternative). The schema is in [`supabase/migrations/`](../supabase/migrations/), so you can verify what's collected. The publishable key in the repo is a - public key; row-level security denies direct access, and edge functions - enforce schema checks, event allowlists and field length limits. + public key that can neither read nor write those tables; every write goes + through edge functions that enforce schema checks, event allowlists and + field length limits. gstack never uses Supabase sign-in, and signed-in users + are denied everything too. gstack's own test suite never sends telemetry + to the live project. **Update check (on by default, regardless of telemetry).** Every interactive skill start runs `gstack-update-check`. When its cache is stale (60 minutes after an diff --git a/supabase/config.sh b/supabase/config.sh index bfc739bc4..2ee3ed612 100644 --- a/supabase/config.sh +++ b/supabase/config.sh @@ -6,3 +6,11 @@ GSTACK_SUPABASE_URL="https://frugpmstpnojnhfyimgv.supabase.co" GSTACK_SUPABASE_ANON_KEY="sb_publishable_tR4i6cyMIrYTE3s6OyHGHw_ppx2p6WK" + +# Tests must never reach the live project (old test runs left about 554,000 +# fixture rows in it). test-setup.ts sets this for every `bun test`; with it +# set, a client that falls back to this file finds no endpoint and sends nothing. +if [ -n "${GSTACK_TEST_BLOCK_LIVE_TELEMETRY:-}" ]; then + GSTACK_SUPABASE_URL="" + GSTACK_SUPABASE_ANON_KEY="" +fi diff --git a/supabase/functions/community-pulse/index.ts b/supabase/functions/community-pulse/index.ts index d2a16664b..758125e3a 100644 --- a/supabase/functions/community-pulse/index.ts +++ b/supabase/functions/community-pulse/index.ts @@ -1,6 +1,7 @@ // gstack community-pulse edge function -// Returns aggregated community stats for the dashboard: -// weekly active count, top skills, crash clusters, version distribution. +// Returns aggregated community stats for the dashboard: weekly active +// community-tier installs, update checks, top skills, crash clusters, version +// distribution and the prompt-injection aggregate, all from live traffic. // Uses server-side cache (community_pulse_cache table) to prevent DoS. // // Fail-closed contract (#1947): success responses carry `status: "ok"` so @@ -40,7 +41,7 @@ Deno.serve(async () => { cached = null; // cache miss/failure is non-fatal — recompute decides } - if (cached?.refreshed_at) { + if (cached?.refreshed_at && cached.data && "weekly_update_checks" in cached.data) { const age = Date.now() - new Date(cached.refreshed_at).getTime(); if (age < CACHE_MAX_AGE_MS) { // Serving the cache means this (new) backend is healthy; assert the @@ -53,153 +54,15 @@ Deno.serve(async () => { } try { - // Cache is stale or missing — recompute - const weekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString(); - const twoWeeksAgo = new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(); + // Cache is stale or missing — recompute. community_pulse_summary() + // (migration 006) counts every live row of the last 7 days in SQL: test + // traffic, fixture-shaped session ids and probe versions are left out, and + // weekly_active is distinct community-tier installs, not update checks. + const { data: summary, error: summaryErr } = await supabase.rpc("community_pulse_summary"); + if (summaryErr) throw summaryErr; + if (!summary || typeof summary !== "object") throw new Error("empty pulse summary"); - // Weekly active (update checks this week) - const { count: thisWeek, error: thisWeekErr } = await supabase - .from("update_checks") - .select("*", { count: "exact", head: true }) - .gte("checked_at", weekAgo); - if (thisWeekErr) throw thisWeekErr; - - // Last week (for change %) - const { count: lastWeek, error: lastWeekErr } = await supabase - .from("update_checks") - .select("*", { count: "exact", head: true }) - .gte("checked_at", twoWeeksAgo) - .lt("checked_at", weekAgo); - if (lastWeekErr) throw lastWeekErr; - - const current = thisWeek ?? 0; - const previous = lastWeek ?? 0; - const changePct = previous > 0 - ? Math.round(((current - previous) / previous) * 100) - : 0; - - // Top skills (last 7 days) - const { data: skillRows, error: skillErr } = await supabase - .from("telemetry_events") - .select("skill") - .eq("event_type", "skill_run") - .gte("event_timestamp", weekAgo) - .not("skill", "is", null) - .limit(1000); - if (skillErr) throw skillErr; - - const skillCounts: Record = {}; - for (const row of skillRows ?? []) { - if (row.skill) { - skillCounts[row.skill] = (skillCounts[row.skill] ?? 0) + 1; - } - } - const topSkills = Object.entries(skillCounts) - .sort(([, a], [, b]) => b - a) - .slice(0, 10) - .map(([skill, count]) => ({ skill, count })); - - // Crash clusters (top 5) - const { data: crashes, error: crashErr } = await supabase - .from("crash_clusters") - .select("error_class, gstack_version, total_occurrences, identified_users") - .limit(5); - if (crashErr) throw crashErr; - - // Version distribution (last 7 days) - const versionCounts: Record = {}; - const { data: versionRows, error: versionErr } = await supabase - .from("telemetry_events") - .select("gstack_version") - .eq("event_type", "skill_run") - .gte("event_timestamp", weekAgo) - .limit(1000); - if (versionErr) throw versionErr; - - for (const row of versionRows ?? []) { - if (row.gstack_version) { - versionCounts[row.gstack_version] = (versionCounts[row.gstack_version] ?? 0) + 1; - } - } - const topVersions = Object.entries(versionCounts) - .sort(([, a], [, b]) => b - a) - .slice(0, 5) - .map(([version, count]) => ({ version, count })); - - // Security events — aggregate attack_attempt events from the last 7 days. - // Fields emitted by gstack-telemetry-log --event-type attack_attempt: - // security_url_domain, security_payload_hash, security_confidence, - // security_layer, security_verdict. - const { data: attackRows, error: attackErr } = await supabase - .from("telemetry_events") - .select("security_url_domain, security_layer, security_verdict, installation_id") - .eq("event_type", "attack_attempt") - .gte("event_timestamp", weekAgo) - .limit(5000); - if (attackErr) throw attackErr; - - // k-anonymity threshold. A domain (or layer) must be reported by at least - // K_ANON distinct installations to appear in the aggregate. Without this, - // a single user's attack log leaks their targeted domains to every other - // gstack user who polls /community-pulse. With it, the dashboard shows - // only community-wide patterns. - const K_ANON = 5; - - const attacksTotal = attackRows?.length ?? 0; - const domainCounts: Record = {}; - const domainInstallations: Record> = {}; - const layerCounts: Record = {}; - const layerInstallations: Record> = {}; - const verdictCounts: Record = {}; - for (const row of attackRows ?? []) { - const iid = row.installation_id ?? ""; - if (row.security_url_domain) { - domainCounts[row.security_url_domain] = (domainCounts[row.security_url_domain] ?? 0) + 1; - if (iid) { - (domainInstallations[row.security_url_domain] ??= new Set()).add(iid); - } - } - if (row.security_layer) { - layerCounts[row.security_layer] = (layerCounts[row.security_layer] ?? 0) + 1; - if (iid) { - (layerInstallations[row.security_layer] ??= new Set()).add(iid); - } - } - if (row.security_verdict) { - // Verdict distribution is low-cardinality (block/warn/log_only) and - // aggregates population-wide with no re-identification risk, so no - // k-anon filter. - verdictCounts[row.security_verdict] = (verdictCounts[row.security_verdict] ?? 0) + 1; - } - } - const topAttackDomains = Object.entries(domainCounts) - .filter(([domain]) => (domainInstallations[domain]?.size ?? 0) >= K_ANON) - .sort(([, a], [, b]) => b - a) - .slice(0, 10) - .map(([domain, count]) => ({ domain, count })); - const topAttackLayers = Object.entries(layerCounts) - .filter(([layer]) => (layerInstallations[layer]?.size ?? 0) >= K_ANON) - .sort(([, a], [, b]) => b - a) - .map(([layer, count]) => ({ layer, count })); - const attackVerdictDistribution = Object.entries(verdictCounts) - .sort(([, a], [, b]) => b - a) - .map(([verdict, count]) => ({ verdict, count })); - - const result = { - status: "ok", - weekly_active: current, - change_pct: changePct, - top_skills: topSkills, - crashes: crashes ?? [], - versions: topVersions, - // Security aggregate for the /security-dashboard view - security: { - attacks_last_7_days: attacksTotal, - top_attack_domains: topAttackDomains, - top_attack_layers: topAttackLayers, - verdict_distribution: attackVerdictDistribution, - }, - }; + const result = { status: "ok", ...(summary as Record) }; // Upsert cache await supabase diff --git a/supabase/functions/telemetry-ingest/index.ts b/supabase/functions/telemetry-ingest/index.ts index f4e9cc8a9..72eaedfd3 100644 --- a/supabase/functions/telemetry-ingest/index.ts +++ b/supabase/functions/telemetry-ingest/index.ts @@ -21,6 +21,48 @@ interface TelemetryEvent { used_browse?: boolean; sessions?: number; installation_id?: string; + source?: string; + security_url_domain?: string; + security_payload_hash?: string; + security_confidence?: number; + security_layer?: string; + security_verdict?: string; +} + +// bin/gstack-telemetry-log writes source from --source or +// GSTACK_TELEMETRY_SOURCE (default "live"). Keep it so test and eval traffic +// can be told apart from real use; anything unexpected is "other", never "live". +const SOURCES = ["live", "test", "eval", "ci"]; +function normalizeSource(source: unknown): string { + if (source === undefined || source === null || source === "") return "live"; + return SOURCES.includes(String(source)) ? String(source) : "other"; +} + +// Prompt-injection detections (browse/src/security.ts via gstack-telemetry-log +// --event-type attack_attempt; columns from migration 004). Domain only, a +// salted hash, a 0..1 score and two enums; anything else is dropped. +const SECURITY_LAYERS = [ + "stackone_content", "testsavant_content", "transcript_classifier", + "aria_regex", "canary", "deberta_content", +]; +const SECURITY_VERDICTS = ["block", "warn", "log_only"]; +function securityFields(event: TelemetryEvent) { + const domain = typeof event.security_url_domain === "string" && + /^[A-Za-z0-9.-]{1,253}$/.test(event.security_url_domain) + ? event.security_url_domain.toLowerCase() : null; + const hash = typeof event.security_payload_hash === "string" && + /^[0-9a-f]{16,128}$/.test(event.security_payload_hash) + ? event.security_payload_hash : null; + const confidence = typeof event.security_confidence === "number" && + event.security_confidence >= 0 && event.security_confidence <= 1 + ? event.security_confidence : null; + return { + security_url_domain: domain, + security_payload_hash: hash, + security_confidence: confidence, + security_layer: SECURITY_LAYERS.includes(String(event.security_layer)) ? event.security_layer : null, + security_verdict: SECURITY_VERDICTS.includes(String(event.security_verdict)) ? event.security_verdict : null, + }; } const MAX_BATCH_SIZE = 100; @@ -45,15 +87,14 @@ Deno.serve(async (req) => { return new Response(`Batch too large (max ${MAX_BATCH_SIZE})`, { status: 400 }); } - // Use the anon key, not the service role key. - // The service role key bypasses Row Level Security (RLS) and grants full - // unrestricted database access — wildly over-privileged for a public - // telemetry endpoint that only needs INSERT on two tables. - // The anon key + properly configured RLS INSERT policies is correct. - // See: https://supabase.com/docs/guides/database/postgres/row-level-security + // Service role key, as update-check and community-pulse use. The public + // anon key can no longer write to these tables (migration 008): the RLS + // insert policies it relied on also let anyone write arbitrary rows + // straight through PostgREST. This function is now the only write path, + // and it validates every field before the insert below. const supabase = createClient( Deno.env.get("SUPABASE_URL") ?? "", - Deno.env.get("SUPABASE_ANON_KEY") ?? "" + Deno.env.get("SUPABASE_SERVICE_ROLE_KEY") ?? "" ); // Validate and transform events @@ -72,6 +113,7 @@ Deno.serve(async (req) => { // Validate event_type. Activation-funnel events (v1.x) join the originals: // onboarding (P0 setup nudge), first_task_scaffold_shown (P4 first-run // scaffold), handoff (P1 office-hours → next skill), route (gstack router). + // attack_attempt carries the prompt-injection fields (migration 004). const validTypes = [ "skill_run", "upgrade_prompted", @@ -80,6 +122,7 @@ Deno.serve(async (req) => { "first_task_scaffold_shown", "handoff", "route", + "attack_attempt", ]; if (!validTypes.includes(event.event_type)) continue; @@ -100,6 +143,8 @@ Deno.serve(async (req) => { used_browse: event.used_browse === true, concurrent_sessions: typeof event.sessions === "number" ? event.sessions : 1, installation_id: event.installation_id ? String(event.installation_id).slice(0, 64) : null, + source: normalizeSource(event.source), + ...(event.event_type === "attack_attempt" ? securityFields(event) : {}), }); // Track installations for upsert diff --git a/supabase/migrations/004_attack_telemetry.sql b/supabase/migrations/004_attack_telemetry.sql index 0c0e4211a..8704eb66a 100644 --- a/supabase/migrations/004_attack_telemetry.sql +++ b/supabase/migrations/004_attack_telemetry.sql @@ -25,20 +25,23 @@ -- * salt is per-device local file (~/.gstack/security/device-salt) — -- preventing cross-device rainbow table attacks +-- IF NOT EXISTS: this migration was not applied to the live project until +-- after 006 was written, so it must be safe to run twice. It must run before +-- 006, whose view and function read these columns. ALTER TABLE telemetry_events - ADD COLUMN security_url_domain TEXT, - ADD COLUMN security_payload_hash TEXT, - ADD COLUMN security_confidence NUMERIC, - ADD COLUMN security_layer TEXT, - ADD COLUMN security_verdict TEXT; + ADD COLUMN IF NOT EXISTS security_url_domain TEXT, + ADD COLUMN IF NOT EXISTS security_payload_hash TEXT, + ADD COLUMN IF NOT EXISTS security_confidence NUMERIC, + ADD COLUMN IF NOT EXISTS security_layer TEXT, + ADD COLUMN IF NOT EXISTS security_verdict TEXT; -- Top-domains query: ORDER BY count DESC WHERE event_type='attack_attempt' -- AND event_timestamp > now() - interval '7 days' -CREATE INDEX idx_telemetry_attack_domain +CREATE INDEX IF NOT EXISTS idx_telemetry_attack_domain ON telemetry_events (security_url_domain, event_timestamp) WHERE event_type = 'attack_attempt'; -- Layer-distribution query -CREATE INDEX idx_telemetry_attack_layer +CREATE INDEX IF NOT EXISTS idx_telemetry_attack_layer ON telemetry_events (security_layer, event_timestamp) WHERE event_type = 'attack_attempt'; diff --git a/supabase/migrations/005_revoke_authenticated.sql b/supabase/migrations/005_revoke_authenticated.sql new file mode 100644 index 000000000..9011d1e56 --- /dev/null +++ b/supabase/migrations/005_revoke_authenticated.sql @@ -0,0 +1,43 @@ +-- 005_revoke_authenticated.sql +-- gstack never uses Supabase Auth, so a signed-in user (the `authenticated` +-- role) must reach nothing. Supabase's default grants give `authenticated` +-- the same table privileges as `anon`, and views run as their owner, so +-- RLS does not protect them: migration 002 revoked crash_clusters and +-- skill_sequences from anon only. On 2026-11-19 someone signed up through the +-- public Auth endpoint, so this closes that path. +-- +-- This mirrors the lockdown applied to the live project on 2026-11-19, and +-- also makes both views security_invoker so they read with the caller's +-- privileges and RLS instead of their owner's. 008 closes anon's write path. + +BEGIN; +REVOKE ALL ON ALL TABLES IN SCHEMA public FROM authenticated; +REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM authenticated; +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM authenticated; +REVOKE ALL ON public.crash_clusters, public.skill_sequences FROM anon, authenticated; +ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE ALL ON TABLES FROM authenticated; +ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE ALL ON SEQUENCES FROM authenticated; +ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE EXECUTE ON FUNCTIONS FROM authenticated; +ALTER VIEW public.crash_clusters SET (security_invoker = true); +ALTER VIEW public.skill_sequences SET (security_invoker = true); +COMMIT; + +-- The lines above change the running role's defaults only. Objects that +-- supabase_admin creates later still default to granting `authenticated`. +-- Only a member of supabase_admin can change that; on the hosted project the +-- migration role (postgres) is not one, so this says so and moves on. +DO $$ +BEGIN + ALTER DEFAULT PRIVILEGES FOR ROLE supabase_admin IN SCHEMA public REVOKE ALL ON TABLES FROM authenticated; + ALTER DEFAULT PRIVILEGES FOR ROLE supabase_admin IN SCHEMA public REVOKE ALL ON SEQUENCES FROM authenticated; + ALTER DEFAULT PRIVILEGES FOR ROLE supabase_admin IN SCHEMA public REVOKE EXECUTE ON FUNCTIONS FROM authenticated; +EXCEPTION + WHEN insufficient_privilege OR undefined_object THEN + RAISE NOTICE 'supabase_admin default privileges unchanged (%): objects it creates later still grant authenticated', SQLERRM; +END +$$; + +-- Postgres also grants EXECUTE on every new function to PUBLIC, which +-- `authenticated` inherits, and the default-privilege line above does not +-- remove that. Every function a later migration adds must therefore +-- `REVOKE EXECUTE ... FROM PUBLIC, anon, authenticated` itself (006 does). diff --git a/supabase/migrations/006_live_traffic_and_pulse_summary.sql b/supabase/migrations/006_live_traffic_and_pulse_summary.sql new file mode 100644 index 000000000..d60d6554b --- /dev/null +++ b/supabase/migrations/006_live_traffic_and_pulse_summary.sql @@ -0,0 +1,140 @@ +-- 006_live_traffic_and_pulse_summary.sql +-- Separates real use from test traffic and moves community-pulse's counting +-- into SQL. +-- +-- Why: until v1.86.0.0 test/telemetry.test.ts posted to this project, and +-- older checkouts still do when their tests run; about 41% of skill_run rows +-- to 2026-11-19 carry test-fixture session ids (`a-1`, `inj-3`, ...). The +-- community-pulse function also counted only the first 1,000 rows of a week +-- and reported update-check rows as "weekly active". +-- +-- What counts as live (telemetry_live_events): +-- * source is 'live' (rows from before this column default to 'live'), +-- * a numeric gstack version other than 0.0.0 (probes used 0.0.0-*), +-- * an event type the ingest function accepts, and +-- * for skill_run, a session id in the shape bin/gstack-skill-start mints +-- (- or --); for upgrade_prompted, the +-- shape bin/gstack-update-check mints (update--). +-- + +ALTER TABLE telemetry_events ADD COLUMN IF NOT EXISTS source TEXT NOT NULL DEFAULT 'live'; + +CREATE INDEX IF NOT EXISTS idx_telemetry_received_at ON telemetry_events (received_at); + +CREATE OR REPLACE VIEW telemetry_live_events WITH (security_invoker = true) AS +SELECT * +FROM telemetry_events +WHERE source = 'live' + AND gstack_version ~ '^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?$' + AND gstack_version NOT LIKE '0.0.0%' + AND event_type IN ('skill_run', 'upgrade_prompted', 'upgrade_completed', 'onboarding', + 'first_task_scaffold_shown', 'handoff', 'route', 'attack_attempt') + AND ( + (event_type = 'skill_run' AND session_id ~ '^[0-9]+-[0-9]+(-[0-9A-Za-z]+)?$') + OR (event_type = 'upgrade_prompted' AND session_id ~ '^update-[0-9]+-[0-9]+$') + OR event_type NOT IN ('skill_run', 'upgrade_prompted') + ); + +REVOKE ALL ON telemetry_live_events FROM PUBLIC, anon, authenticated; + +-- Same columns as 001's view; it now reads live rows only. +CREATE OR REPLACE VIEW crash_clusters WITH (security_invoker = true) AS +SELECT + error_class, + gstack_version, + COUNT(*) as total_occurrences, + COUNT(DISTINCT installation_id) as identified_users, + COUNT(*) - COUNT(installation_id) as anonymous_occurrences, + MIN(event_timestamp) as first_seen, + MAX(event_timestamp) as last_seen +FROM telemetry_live_events +WHERE outcome = 'error' AND error_class IS NOT NULL +GROUP BY error_class, gstack_version +ORDER BY total_occurrences DESC; + +REVOKE ALL ON crash_clusters FROM PUBLIC, anon, authenticated; + +-- One call returns everything community-pulse shows, counted over every live +-- row of the last 7 days (by server receive time). weekly_active is distinct +-- community-tier installs that ran a skill; anonymous-tier use has no id and +-- is not in it. Attack domains and layers need 5 distinct installs (k-anonymity). +CREATE OR REPLACE FUNCTION community_pulse_summary() +RETURNS jsonb +LANGUAGE sql +STABLE +SECURITY INVOKER +SET search_path = public +AS $$ +WITH week AS ( + SELECT * FROM telemetry_live_events WHERE received_at >= now() - interval '7 days' +), +prev AS ( + SELECT * FROM telemetry_live_events + WHERE received_at >= now() - interval '14 days' AND received_at < now() - interval '7 days' +), +runs AS ( + SELECT * FROM week WHERE event_type = 'skill_run' AND skill IS NOT NULL AND skill NOT LIKE '\_%' +), +active AS ( + SELECT + (SELECT COUNT(DISTINCT installation_id) FROM runs) AS this_week, + (SELECT COUNT(DISTINCT installation_id) FROM prev + WHERE event_type = 'skill_run' AND skill IS NOT NULL AND skill NOT LIKE '\_%') AS last_week +), +attacks AS ( + SELECT * FROM week WHERE event_type = 'attack_attempt' +) +SELECT jsonb_build_object( + 'weekly_active', (SELECT this_week FROM active), + 'change_pct', (SELECT CASE WHEN last_week > 0 + THEN round(((this_week - last_week)::numeric / last_week) * 100)::int + ELSE 0 END FROM active), + 'weekly_update_checks', ( + SELECT COUNT(*) FROM update_checks + WHERE checked_at >= now() - interval '7 days' + AND gstack_version ~ '^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?$' + AND gstack_version NOT LIKE '0.0.0%' + ), + 'top_skills', COALESCE(( + SELECT jsonb_agg(jsonb_build_object('skill', skill, 'count', n) ORDER BY n DESC, skill) + FROM (SELECT skill, COUNT(*) AS n FROM runs GROUP BY skill ORDER BY n DESC, skill LIMIT 10) t + ), '[]'::jsonb), + 'crashes', COALESCE(( + SELECT jsonb_agg(jsonb_build_object( + 'error_class', error_class, 'gstack_version', gstack_version, + 'total_occurrences', total_occurrences, 'identified_users', identified_users)) + FROM (SELECT * FROM crash_clusters LIMIT 5) t + ), '[]'::jsonb), + 'versions', COALESCE(( + SELECT jsonb_agg(jsonb_build_object('version', gstack_version, 'count', n) ORDER BY n DESC, gstack_version) + FROM (SELECT gstack_version, COUNT(*) AS n FROM runs GROUP BY gstack_version + ORDER BY n DESC, gstack_version LIMIT 5) t + ), '[]'::jsonb), + 'security', jsonb_build_object( + 'attacks_last_7_days', (SELECT COUNT(*) FROM attacks), + 'top_attack_domains', COALESCE(( + SELECT jsonb_agg(jsonb_build_object('domain', security_url_domain, 'count', n) ORDER BY n DESC, security_url_domain) + FROM (SELECT security_url_domain, COUNT(*) AS n FROM attacks + WHERE security_url_domain IS NOT NULL + GROUP BY security_url_domain + HAVING COUNT(DISTINCT installation_id) >= 5 + ORDER BY n DESC, security_url_domain LIMIT 10) t + ), '[]'::jsonb), + 'top_attack_layers', COALESCE(( + SELECT jsonb_agg(jsonb_build_object('layer', security_layer, 'count', n) ORDER BY n DESC, security_layer) + FROM (SELECT security_layer, COUNT(*) AS n FROM attacks + WHERE security_layer IS NOT NULL + GROUP BY security_layer + HAVING COUNT(DISTINCT installation_id) >= 5) t + ), '[]'::jsonb), + 'verdict_distribution', COALESCE(( + SELECT jsonb_agg(jsonb_build_object('verdict', security_verdict, 'count', n) ORDER BY n DESC, security_verdict) + FROM (SELECT security_verdict, COUNT(*) AS n FROM attacks + WHERE security_verdict IS NOT NULL GROUP BY security_verdict) t + ), '[]'::jsonb) + ) +); +$$; + +REVOKE EXECUTE ON FUNCTION community_pulse_summary() FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION community_pulse_summary() TO service_role; diff --git a/supabase/migrations/007_drop_unused_installation_columns.sql b/supabase/migrations/007_drop_unused_installation_columns.sql new file mode 100644 index 000000000..8cfebf3ac --- /dev/null +++ b/supabase/migrations/007_drop_unused_installation_columns.sql @@ -0,0 +1,28 @@ +-- 007_drop_unused_installation_columns.sql +-- The live installations table carries seven columns that no migration in +-- this repo defines and no gstack code reads or writes: user_id, email, +-- config_snapshot, analytics_snapshot, retro_history, last_backup_at and +-- backup_version. On 2026-11-19 all seven were empty in all 37,566 rows. +-- An email column has no place in a telemetry table (docs/privacy.md: no +-- personal data), so drop them. The guard refuses if any row holds a value, +-- so this can never delete data; it does nothing where the columns are absent. + +DO $$ +DECLARE + col text; + filled bigint; +BEGIN + LOCK TABLE public.installations IN ACCESS EXCLUSIVE MODE; + FOREACH col IN ARRAY ARRAY['user_id', 'email', 'config_snapshot', 'analytics_snapshot', + 'retro_history', 'last_backup_at', 'backup_version'] LOOP + IF EXISTS (SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'installations' AND column_name = col) THEN + EXECUTE format('SELECT COUNT(*) FROM public.installations WHERE %I IS NOT NULL', col) INTO filled; + IF filled > 0 THEN + RAISE EXCEPTION 'installations.% holds % values; refusing to drop it', col, filled; + END IF; + EXECUTE format('ALTER TABLE public.installations DROP COLUMN %I', col); + END IF; + END LOOP; +END +$$; diff --git a/supabase/migrations/008_close_anon_writes.sql b/supabase/migrations/008_close_anon_writes.sql new file mode 100644 index 000000000..be8251c6e --- /dev/null +++ b/supabase/migrations/008_close_anon_writes.sql @@ -0,0 +1,42 @@ +-- 008_close_anon_writes.sql +-- Closes the last direct write path. Since v0.11.16.0 every gstack client +-- writes through the edge functions (telemetry-ingest, update-check); only +-- older clients POST straight to the tables with the public key. The policies +-- kept for them in 002 also let anyone write arbitrary rows: probes on +-- 2026-11-16..19 added about 2,000 junk events and 500 fake installs. +-- +-- What this drops: clients older than v0.11.16.0 (released 2026-03-24) stop +-- landing. In the 30 days to 2026-11-19 that was 245 events from 23 installs +-- and 163 update checks. Their sync keeps the batch and retries, so nothing +-- breaks on their side; it just never arrives. +-- +-- Deploy order: apply 004, 005, 006 and 007; deploy telemetry-ingest and +-- community-pulse (the new ingest writes 004's and 006's columns, and the new +-- pulse calls 006's function); confirm one ingest event lands; then apply +-- this. Run earlier, it refuses every new event. Undo by re-granting anon's +-- table privileges and recreating the policies this file drops (created in +-- 001 and 003). + +DROP POLICY IF EXISTS "anon_insert_only" ON telemetry_events; +DROP POLICY IF EXISTS "anon_insert_only" ON installations; +DROP POLICY IF EXISTS "anon_insert_only" ON update_checks; +DROP POLICY IF EXISTS "anon_update_tracking" ON installations; +DROP POLICY IF EXISTS "anon_update_last_seen" ON installations; + +REVOKE ALL ON ALL TABLES IN SCHEMA public FROM anon; +REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM anon; +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM anon; +ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE ALL ON TABLES FROM anon; +ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE ALL ON SEQUENCES FROM anon; +ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE EXECUTE ON FUNCTIONS FROM anon; + +DO $$ +BEGIN + ALTER DEFAULT PRIVILEGES FOR ROLE supabase_admin IN SCHEMA public REVOKE ALL ON TABLES FROM anon; + ALTER DEFAULT PRIVILEGES FOR ROLE supabase_admin IN SCHEMA public REVOKE ALL ON SEQUENCES FROM anon; + ALTER DEFAULT PRIVILEGES FOR ROLE supabase_admin IN SCHEMA public REVOKE EXECUTE ON FUNCTIONS FROM anon; +EXCEPTION + WHEN insufficient_privilege OR undefined_object THEN + RAISE NOTICE 'supabase_admin default privileges unchanged (%): objects it creates later still grant anon', SQLERRM; +END +$$; diff --git a/supabase/verify-rls.sh b/supabase/verify-rls.sh index 3657776a1..c21bcf86f 100755 --- a/supabase/verify-rls.sh +++ b/supabase/verify-rls.sh @@ -1,14 +1,30 @@ #!/usr/bin/env bash -# verify-rls.sh — smoke test after deploying 002_tighten_rls.sql +# verify-rls.sh — smoke test the telemetry project's access rules after +# applying migrations 005-008. # -# Verifies: -# - SELECT denied on all tables and views (security fix) -# - UPDATE denied on installations (security fix) -# - INSERT still allowed on tables (kept for old client compat) +# Verifies, for the public key (anon) and for a signed-in user +# (a JWT whose role is `authenticated`): +# - every table, view and the pulse function is unreadable +# - no table accepts an INSERT or UPDATE (008 closed the legacy inserts; +# every write now goes through the telemetry-ingest edge function) +# +# A check passes only on a permission denial (Postgres error 42501) in the +# response body. A missing table or function fails, except community_benchmarks, +# which may be absent (ABSENT). An invalid JWT, a constraint error or any other +# response fails, so a wrong key or an unapplied migration cannot pass. +# +# Writes nothing: insert probes send an empty row, so a table that wrongly +# accepts the insert still rejects the row on its NOT NULL columns, and the +# check reports that as a failure. +# +# The signed-in checks need a user JWT. Set GSTACK_VERIFY_AUTH_JWT to one, or +# SUPABASE_JWT_SECRET (Dashboard > Settings > JWT, legacy secret) to mint a +# five-minute token for a random user id. Neither is ever printed. Without +# either, the signed-in checks are skipped and the verdict is INCOMPLETE. # -# Run manually after deploying the migration: # bash supabase/verify-rls.sh set -uo pipefail +BASH_COMPAT=50 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" . "$SCRIPT_DIR/config.sh" @@ -18,131 +34,145 @@ KEY="$GSTACK_SUPABASE_ANON_KEY" PASS=0 FAIL=0 TOTAL=0 +SKIPPED=0 -# check [data] -# expected: "deny" (want 401/403) or "allow" (want 200/201) +# check [data] +# Every check expects a permission denial, and decides from the response body: +# - PASS only when the body carries Postgres error 42501 (permission denied). +# - PGRST205 or PGRST202 (table or function not found) is ABSENT, which +# counts as a pass for community_benchmarks only. For any other object it +# is a FAIL: the migration that creates it is not applied. +# - Anything else is a FAIL: data, an empty read, an update that matched no +# row, an invalid JWT (401 PGRST301), a write stopped only by a constraint +# (400 or 409), or a failed connection. check() { - local desc="$1" - local expected="$2" - local method="$3" - local path="$4" - local data="${5:-}" + local label="$1" bearer="$2" desc="$3" method="$4" path="$5" data="${6:-}" TOTAL=$(( TOTAL + 1 )) local resp_file - # Use mktemp strictly. Don't fall back to a predictable $$-based path — - # that's a race/overwrite footgun on shared machines. resp_file="$(mktemp "${TMPDIR:-/tmp}/verify-rls-XXXXXX")" || { echo "verify-rls: mktemp failed, aborting" >&2 return 1 } + local hdr_file + hdr_file="$(mktemp "${TMPDIR:-/tmp}/verify-rls-hdr-XXXXXX")" || { + rm -f "$resp_file"; echo "verify-rls: mktemp failed, aborting" >&2 + return 1 + } + chmod 600 "$hdr_file" 2>/dev/null || true + printf 'apikey: %s\nAuthorization: Bearer %s\nContent-Type: application/json\n' "$KEY" "$bearer" > "$hdr_file" local http_code - if [ "$method" = "GET" ]; then - http_code="$(curl -s -o "$resp_file" -w '%{http_code}' --max-time 10 \ - "${URL}/rest/v1/${path}" \ - -H "apikey: ${KEY}" \ - -H "Authorization: Bearer ${KEY}" \ - -H "Content-Type: application/json" 2>/dev/null)" || http_code="000" - elif [ "$method" = "POST" ]; then - http_code="$(curl -s -o "$resp_file" -w '%{http_code}' --max-time 10 \ - -X POST "${URL}/rest/v1/${path}" \ - -H "apikey: ${KEY}" \ - -H "Authorization: Bearer ${KEY}" \ - -H "Content-Type: application/json" \ - -H "Prefer: return=minimal" \ - -d "$data" 2>/dev/null)" || http_code="000" - elif [ "$method" = "PATCH" ]; then - http_code="$(curl -s -o "$resp_file" -w '%{http_code}' --max-time 10 \ - -X PATCH "${URL}/rest/v1/${path}" \ - -H "apikey: ${KEY}" \ - -H "Authorization: Bearer ${KEY}" \ - -H "Content-Type: application/json" \ - -d "$data" 2>/dev/null)" || http_code="000" - fi - - # Trim to last 3 chars (the HTTP code) in case of concatenation + case "$method" in + GET) http_code="$(curl -s -o "$resp_file" -w '%{http_code}' --max-time 10 -H @"$hdr_file" \ + "${URL}/rest/v1/${path}" 2>/dev/null)" || http_code="000" ;; + POST) http_code="$(curl -s -o "$resp_file" -w '%{http_code}' --max-time 10 -H @"$hdr_file" \ + -H "Prefer: return=minimal" -X POST "${URL}/rest/v1/${path}" -d "$data" 2>/dev/null)" || http_code="000" ;; + PATCH) http_code="$(curl -s -o "$resp_file" -w '%{http_code}' --max-time 10 -H @"$hdr_file" \ + -X PATCH "${URL}/rest/v1/${path}" -d "$data" 2>/dev/null)" || http_code="000" ;; + esac + rm -f "$hdr_file" 2>/dev/null || true http_code="$(echo "$http_code" | grep -oE '[0-9]{3}$' || echo "000")" - if [ "$expected" = "deny" ]; then - case "$http_code" in - 401|403) - echo " PASS $desc (HTTP $http_code, denied)" - PASS=$(( PASS + 1 )) ;; - 200|204) - # For GETs: 200+empty means RLS filtering (pass). 200+data means leak (fail). - # For PATCH: 204 means no rows matched — could be RLS or missing row. - if [ "$method" = "GET" ]; then - body="$(cat "$resp_file" 2>/dev/null || echo "")" - if [ "$body" = "[]" ] || [ -z "$body" ]; then - echo " PASS $desc (HTTP $http_code, empty — RLS filtering)" - PASS=$(( PASS + 1 )) - else - echo " FAIL $desc (HTTP $http_code, got data!)" - FAIL=$(( FAIL + 1 )) - fi - else - # PATCH 204 = no rows affected. RLS blocked the update or row doesn't exist. - # Either way, the attacker can't modify data. - echo " PASS $desc (HTTP $http_code, no rows affected)" - PASS=$(( PASS + 1 )) - fi ;; - 000) - echo " WARN $desc (connection failed)" - FAIL=$(( FAIL + 1 )) ;; - *) - echo " WARN $desc (HTTP $http_code — unexpected)" - FAIL=$(( FAIL + 1 )) ;; - esac - elif [ "$expected" = "allow" ]; then - case "$http_code" in - 200|201|204|409) - # 409 = conflict (duplicate key) — INSERT policy works, row already exists - echo " PASS $desc (HTTP $http_code, allowed as expected)" - PASS=$(( PASS + 1 )) ;; - 401|403) - echo " FAIL $desc (HTTP $http_code, denied — should be allowed)" - FAIL=$(( FAIL + 1 )) ;; - 000) - echo " WARN $desc (connection failed)" - FAIL=$(( FAIL + 1 )) ;; - *) - echo " WARN $desc (HTTP $http_code — unexpected)" - FAIL=$(( FAIL + 1 )) ;; - esac - fi - + local body object + body="$(cat "$resp_file" 2>/dev/null || echo "")" rm -f "$resp_file" 2>/dev/null || true + object="${path%%\?*}" + object="${object#rpc/}" + + if [ "$http_code" = "000" ]; then + echo " FAIL [$label] $desc (connection failed)" + FAIL=$(( FAIL + 1 )) + elif [[ "$body" == *'"42501"'* ]]; then + echo " PASS [$label] $desc (HTTP $http_code, permission denied)" + PASS=$(( PASS + 1 )) + elif [[ "$body" == *PGRST205* || "$body" == *PGRST202* ]]; then + if [ "$object" = "community_benchmarks" ]; then + echo " ABSENT [$label] $desc (HTTP $http_code, not present)" + PASS=$(( PASS + 1 )) + else + echo " FAIL [$label] $desc (HTTP $http_code: $object not found; is its migration applied?)" + FAIL=$(( FAIL + 1 )) + fi + elif [[ "$body" == *PGRST301* ]]; then + echo " FAIL [$label] $desc (HTTP $http_code: JWT rejected, so nothing was checked)" + FAIL=$(( FAIL + 1 )) + elif [ "$http_code" = "200" ] || [ "$http_code" = "204" ] || [ "$http_code" = "201" ]; then + if [ "$method" = "GET" ] && { [ "$body" = "[]" ] || [ -z "$body" ]; }; then + echo " FAIL [$label] $desc (HTTP $http_code: readable, empty only because of row filtering)" + elif [ "$method" = "PATCH" ]; then + echo " FAIL [$label] $desc (HTTP $http_code: update permitted, no row matched)" + else + echo " FAIL [$label] $desc (HTTP $http_code: got data!)" + fi + FAIL=$(( FAIL + 1 )) + elif [ "$http_code" = "400" ] || [ "$http_code" = "409" ]; then + echo " FAIL [$label] $desc (HTTP $http_code: write permitted, stopped only by a constraint)" + FAIL=$(( FAIL + 1 )) + else + echo " FAIL [$label] $desc (HTTP $http_code, not a permission denial)" + FAIL=$(( FAIL + 1 )) + fi } -echo "RLS Verification (after 002_tighten_rls.sql)" +check_all() { + local label="$1" bearer="$2" + echo "Reads ($label):" + for rel in telemetry_events installations update_checks community_pulse_cache community_benchmarks \ + crash_clusters telemetry_live_events; do + check "$label" "$bearer" "SELECT $rel" GET "${rel}?select=*&limit=1" + done + check "$label" "$bearer" "SELECT skill_sequences" GET "skill_sequences?select=skill_a&limit=1" + check "$label" "$bearer" "CALL community_pulse_summary" POST "rpc/community_pulse_summary" '{}' + echo "" + echo "Writes ($label):" + for rel in telemetry_events installations update_checks; do + check "$label" "$bearer" "INSERT $rel" POST "$rel" '{}' + done + check "$label" "$bearer" "UPDATE installations" PATCH "installations?installation_id=eq.verify_rls_none" '{"gstack_version":"x"}' + echo "" +} + +mint_jwt() { + [ -n "${SUPABASE_JWT_SECRET:-}" ] || return 1 + command -v python3 >/dev/null 2>&1 || { echo "verify-rls: python3 is needed to mint a JWT" >&2; return 1; } + python3 - <<'PY' +import base64, hashlib, hmac, json, os, time, uuid +def b64(b): return base64.urlsafe_b64encode(b).rstrip(b"=").decode() +now = int(time.time()) +head = b64(json.dumps({"alg": "HS256", "typ": "JWT"}).encode()) +body = b64(json.dumps({"role": "authenticated", "aud": "authenticated", "sub": str(uuid.uuid4()), + "iat": now, "exp": now + 300}).encode()) +sig = hmac.new(os.environ["SUPABASE_JWT_SECRET"].encode(), f"{head}.{body}".encode(), hashlib.sha256).digest() +print(f"{head}.{body}.{b64(sig)}") +PY +} + +echo "Access verification (migrations 005-008)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" -echo "Read denial (should be blocked):" -check "SELECT telemetry_events" deny GET "telemetry_events?select=*&limit=1" -check "SELECT installations" deny GET "installations?select=*&limit=1" -check "SELECT update_checks" deny GET "update_checks?select=*&limit=1" -check "SELECT crash_clusters" deny GET "crash_clusters?select=*&limit=1" -check "SELECT skill_sequences" deny GET "skill_sequences?select=skill_a&limit=1" +check_all "anon" "$KEY" -echo "" -echo "Update denial (should be blocked):" -check "UPDATE installations" deny PATCH "installations?installation_id=eq.test_verify_rls" '{"gstack_version":"hacked"}' +USER_JWT="${GSTACK_VERIFY_AUTH_JWT:-}" +[ -n "$USER_JWT" ] || USER_JWT="$(mint_jwt 2>/dev/null || true)" +if [ -n "$USER_JWT" ]; then + check_all "signed-in" "$USER_JWT" +else + echo "SKIP signed-in checks: set GSTACK_VERIFY_AUTH_JWT or SUPABASE_JWT_SECRET to run them" + SKIPPED=1 + echo "" +fi -echo "" -echo "Insert allowed (kept for old client compat):" -check "INSERT telemetry_events" allow POST "telemetry_events" '{"gstack_version":"verify_rls_test","os":"test","event_timestamp":"2026-01-01T00:00:00Z","outcome":"test"}' -check "INSERT update_checks" allow POST "update_checks" '{"gstack_version":"verify_rls_test","os":"test"}' -check "INSERT installations" allow POST "installations" '{"installation_id":"verify_rls_test"}' - -echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Results: $PASS passed, $FAIL failed (of $TOTAL checks)" if [ "$FAIL" -gt 0 ]; then echo "VERDICT: FAIL" exit 1 +elif [ "$SKIPPED" -gt 0 ]; then + echo "VERDICT: INCOMPLETE — public-key checks passed; signed-in checks not run" + exit 2 else - echo "VERDICT: PASS — reads/updates blocked, inserts allowed" + echo "VERDICT: PASS — the public key and signed-in users can read and write nothing" exit 0 fi diff --git a/test-setup.ts b/test-setup.ts index c170db1b9..e19d10d8d 100644 --- a/test-setup.ts +++ b/test-setup.ts @@ -50,6 +50,11 @@ for (const name of ['GBRAIN_DATABASE_URL', 'GBRAIN_DIRECT_DATABASE_URL', 'DATABA delete process.env[name]; process.stderr.write(`test-setup: stripped inherited ${name}; tests must not reach a live gbrain\n`); } +// Telemetry clients fall back to supabase/config.sh, the live project, when a +// test leaves GSTACK_SUPABASE_URL unset; that is how about 554,000 test rows +// reached it. With this set, config.sh yields no endpoint and nothing is sent. +// test/telemetry-no-live-endpoint.test.ts holds the tripwire. +process.env.GSTACK_TEST_BLOCK_LIVE_TELEMETRY = '1'; // One fixture root per test process (GST-1695). TMPDIR (TMP and TEMP on Windows) // points at it, so os.tmpdir() here, and $TMPDIR in a child spawned through // node:child_process or with this process's env, put each fixture inside it; a diff --git a/test/state-root-one-variable.test.ts b/test/state-root-one-variable.test.ts index 4de1b4815..10560ca9d 100644 --- a/test/state-root-one-variable.test.ts +++ b/test/state-root-one-variable.test.ts @@ -28,6 +28,7 @@ for (const variable of ['GSTACK_HOME', 'GSTACK_STATE_ROOT', 'GSTACK_STATE_DIR'] ...(process.env as Record), HOME: home, GSTACK_HOME: '', GSTACK_STATE_ROOT: '', GSTACK_STATE_DIR: '', CLAUDE_PLUGIN_DATA: '', GSTACK_TEST_LEGACY_ROOT: '', [variable]: root, + GSTACK_SUPABASE_URL: 'https://telemetry.fixture.invalid', GSTACK_SUPABASE_ANON_KEY: 'fixture-anon-key', }; expect(resolveStateRoot(env)).toBe(root); const run = (cmd: string, args: string[], input?: string) => diff --git a/test/supabase-telemetry-schema.test.ts b/test/supabase-telemetry-schema.test.ts new file mode 100644 index 000000000..b044d67e9 --- /dev/null +++ b/test/supabase-telemetry-schema.test.ts @@ -0,0 +1,96 @@ +/** + * The telemetry backend in supabase/ matches what the client sends and keeps + * the public key and signed-in users out. These are source checks: the edge + * functions run on Deno in Supabase and the migrations on the live project, + * neither of which the free suite can reach. + */ +import { describe, test, expect } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const MIGRATIONS = path.join(ROOT, 'supabase', 'migrations'); +const INGEST = fs.readFileSync(path.join(ROOT, 'supabase', 'functions', 'telemetry-ingest', 'index.ts'), 'utf-8'); + +function emittedEventTypes(): Set { + const types = new Set(); + const scan = (file: string) => { + for (const m of fs.readFileSync(file, 'utf-8').matchAll(/--event-type['", ]+([a-z_]+)/g)) types.add(m[1]); + }; + const walk = (dir: string, keep: (name: string) => boolean) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const p = path.join(dir, entry.name); + if (entry.isDirectory()) { if (!['node_modules', 'test', 'dist'].includes(entry.name)) walk(p, keep); } + else if (keep(entry.name)) scan(p); + } + }; + walk(path.join(ROOT, 'bin'), () => true); + walk(path.join(ROOT, 'browse', 'src'), n => n.endsWith('.ts')); + walk(ROOT, n => n.endsWith('.tmpl')); + scan(path.join(ROOT, 'setup')); + return types; +} + +describe('telemetry-ingest accepts what the client sends', () => { + test('every event type the client emits is on the ingest allowlist', () => { + const allow = new Set([...(INGEST.match(/const validTypes = \[([^\]]*)\]/)?.[1] ?? '').matchAll(/"([a-z_]+)"/g)].map(m => m[1])); + const emitted = emittedEventTypes(); + expect(emitted.size).toBeGreaterThanOrEqual(6); + expect([...emitted].filter(t => !allow.has(t))).toEqual([]); + }); + + test('ingest stores source, so test and eval traffic stay separable', () => { + expect(INGEST).toContain('source: normalizeSource(event.source)'); + }); + + test('ingest writes with the service role key, the only write path once 008 drops the anon policies', () => { + expect(INGEST).toContain('Deno.env.get("SUPABASE_SERVICE_ROLE_KEY")'); + expect(INGEST).not.toContain('SUPABASE_ANON_KEY'); + }); +}); + +describe('migrations keep the public key and signed-in users out', () => { + const files = fs.readdirSync(MIGRATIONS).filter(f => f.endsWith('.sql')).sort(); + const later = files.filter(f => Number(f.slice(0, 3)) >= 5); + + test('every view, function or table created from 005 on is revoked from anon and authenticated', () => { + const missing: string[] = []; + for (const f of later) { + const sql = fs.readFileSync(path.join(MIGRATIONS, f), 'utf-8'); + for (const m of sql.matchAll(/CREATE (?:OR REPLACE )?(VIEW|FUNCTION|TABLE)(?: IF NOT EXISTS)? (?:public\.)?([a-z_]+)/gi)) { + const name = m[2]; + const revoke = new RegExp(`REVOKE [A-Z ]+ON (?:FUNCTION )?(?:public\\.)?${name}\\b[^;]*FROM [^;]*\\banon\\b[^;]*\\bauthenticated\\b`, 'i'); + if (!revoke.test(sql)) missing.push(`${f}: ${m[1]} ${name}`); + } + } + expect(missing).toEqual([]); + }); + + test('every view created from 005 on runs with the caller\'s privileges', () => { + for (const f of later) { + const sql = fs.readFileSync(path.join(MIGRATIONS, f), 'utf-8'); + for (const m of sql.matchAll(/CREATE (?:OR REPLACE )?VIEW ([a-z_]+)([^\n]*)/gi)) { + expect(m[2], `${f}: ${m[1]}`).toContain('security_invoker = true'); + } + } + }); + + test('the older views are switched to security_invoker and revoked from both roles', () => { + const sql = fs.readFileSync(path.join(MIGRATIONS, '005_revoke_authenticated.sql'), 'utf-8'); + for (const view of ['crash_clusters', 'skill_sequences']) { + expect(sql).toContain(`ALTER VIEW public.${view} SET (security_invoker = true);`); + } + expect(sql).toContain('REVOKE ALL ON public.crash_clusters, public.skill_sequences FROM anon, authenticated;'); + expect(sql).toContain('REVOKE ALL ON ALL TABLES IN SCHEMA public FROM authenticated;'); + }); + + test('no migration leaves an anon insert or update policy standing', () => { + const created = new Set(); + for (const f of files) { + const sql = fs.readFileSync(path.join(MIGRATIONS, f), 'utf-8'); + for (const m of sql.matchAll(/CREATE POLICY "([^"]+)" ON ([a-z_]+)/g)) created.add(`${m[1]} ON ${m[2]}`); + for (const m of sql.matchAll(/DROP POLICY IF EXISTS "([^"]+)" ON ([a-z_]+)/g)) created.delete(`${m[1]} ON ${m[2]}`); + } + expect([...created].filter(p => /^anon_(insert|update)/.test(p))).toEqual([]); + }); +}); diff --git a/test/telemetry-no-live-endpoint.test.ts b/test/telemetry-no-live-endpoint.test.ts new file mode 100644 index 000000000..b74d1c6cf --- /dev/null +++ b/test/telemetry-no-live-endpoint.test.ts @@ -0,0 +1,126 @@ +/** + * Tests never reach the live telemetry project. Before v1.86.0.0 every run of + * test/telemetry.test.ts posted fixture events to it, and + * test/state-root-one-variable.test.ts still did until this file landed: + * about 554,000 of the 1.34M skill_run rows there are test fixtures. + * + * The guard: test-setup.ts sets GSTACK_TEST_BLOCK_LIVE_TELEMETRY, and + * supabase/config.sh (the fallback every telemetry client sources when + * GSTACK_SUPABASE_URL is unset) then yields no endpoint. + */ +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const BIN = path.join(ROOT, 'bin'); +const CONFIG = path.join(ROOT, 'supabase', 'config.sh'); +const LIVE_HOST = (fs.readFileSync(CONFIG, 'utf-8').match(/GSTACK_SUPABASE_URL="https:\/\/([^"/]+)"/) ?? [])[1]; +const CONFIG_CLIENTS = ['gstack-community-dashboard', 'gstack-security-dashboard', 'gstack-telemetry-sync', 'gstack-update-check']; + +let tmp: string; +let curlLog: string; + +function run(args: string[], env: Record = {}) { + const merged: Record = {}; + for (const [k, v] of Object.entries({ ...process.env, ...env })) if (v !== undefined) merged[k] = v; + return spawnSync('bash', args, { cwd: ROOT, env: merged, encoding: 'utf-8', timeout: 20_000 }); +} + +function curlCalls(): string[] { + return fs.existsSync(curlLog) ? fs.readFileSync(curlLog, 'utf-8').split('\n').filter(Boolean) : []; +} + +beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-no-live-')); + curlLog = path.join(tmp, 'curl.log'); + fs.mkdirSync(path.join(tmp, 'bin')); + fs.mkdirSync(path.join(tmp, 'state', 'analytics'), { recursive: true }); + fs.writeFileSync(path.join(tmp, 'bin', 'curl'), `#!/bin/sh\nprintf '%s ' "$@" >> "$GSTACK_TEST_CURL_LOG"\necho >> "$GSTACK_TEST_CURL_LOG"\nprintf '200'\n`, { mode: 0o755 }); + fs.writeFileSync(path.join(tmp, 'state', 'config.yaml'), 'telemetry: anonymous\n'); + fs.writeFileSync( + path.join(tmp, 'state', 'analytics', 'skill-usage.jsonl'), + '{"v":1,"ts":"2026-11-19T00:00:00Z","event_type":"skill_run","skill":"qa","session_id":"1-2","gstack_version":"1.0.0.0","os":"linux","outcome":"success"}\n', + ); +}); + +afterEach(() => { + fs.rmSync(tmp, { recursive: true, force: true }); +}); + +function clientEnv(extra: Record = {}) { + return { + GSTACK_HOME: path.join(tmp, 'state'), GSTACK_STATE_ROOT: undefined, GSTACK_STATE_DIR: undefined, + HOME: path.join(tmp, 'home'), GSTACK_DIR: ROOT, GSTACK_TEST_CURL_LOG: curlLog, + PATH: `${path.join(tmp, 'bin')}:${process.env.PATH ?? ''}`, + GSTACK_SUPABASE_URL: undefined, GSTACK_SUPABASE_ANON_KEY: undefined, + ...extra, + }; +} + +describe('telemetry never reaches the live project from tests', () => { + test('the test preload sets the guard', () => { + expect(process.env.GSTACK_TEST_BLOCK_LIVE_TELEMETRY).toBe('1'); + expect(LIVE_HOST).toBeTruthy(); + }); + + test('config.sh yields no endpoint under the guard, and the live one without it', () => { + const show = '. "$1"; printf "[%s][%s]" "$GSTACK_SUPABASE_URL" "${GSTACK_SUPABASE_ANON_KEY:+key}"'; + const guarded = run(['-c', show, 'x', CONFIG]); + expect(guarded.status).toBe(0); + expect(guarded.stdout).toBe('[][]'); + const unguarded = run(['-c', show, 'x', CONFIG], { GSTACK_TEST_BLOCK_LIVE_TELEMETRY: '' }); + expect(unguarded.stdout).toBe(`[https://${LIVE_HOST}][key]`); + }); + + test('every client that reads config.sh reads it only as the fallback for an unset GSTACK_SUPABASE_URL', () => { + for (const name of CONFIG_CLIENTS) { + const src = fs.readFileSync(path.join(BIN, name), 'utf-8'); + expect(src, name).toContain('if [ -z "${GSTACK_SUPABASE_URL:-}" ] && [ -f "$GSTACK_DIR/supabase/config.sh" ]; then'); + } + const sourcing = fs.readdirSync(BIN).filter(f => { + const p = path.join(BIN, f); + return fs.statSync(p).isFile() && fs.readFileSync(p, 'utf-8').includes('. "$GSTACK_DIR/supabase/config.sh"'); + }); + expect(sourcing.sort()).toEqual([...CONFIG_CLIENTS].sort()); + }); + + test('telemetry sync with telemetry on and no endpoint set sends nothing', () => { + const r = run([path.join(BIN, 'gstack-telemetry-sync')], clientEnv()); + expect(r.status).toBe(0); + expect(curlCalls()).toEqual([]); + }); + + test('the community dashboard with no endpoint set sends nothing', () => { + const r = run([path.join(BIN, 'gstack-community-dashboard')], clientEnv()); + expect(r.status).toBe(0); + expect(r.stdout).toContain('Supabase not configured'); + expect(curlCalls()).toEqual([]); + }); + + test('control: the curl stub does record a send to an explicit fixture endpoint', () => { + const r = run([path.join(BIN, 'gstack-telemetry-sync')], clientEnv({ + GSTACK_SUPABASE_URL: 'https://telemetry.fixture.invalid', GSTACK_SUPABASE_ANON_KEY: 'fixture-anon-key', + })); + expect(r.status).toBe(0); + const calls = curlCalls(); + expect(calls).toHaveLength(1); + expect(calls[0]).toContain('https://telemetry.fixture.invalid/functions/v1/telemetry-ingest'); + }); + + test('no test file names the live project', () => { + const offenders: string[] = []; + const walk = (dir: string) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const p = path.join(dir, entry.name); + if (entry.isDirectory()) { if (entry.name !== 'node_modules') walk(p); continue; } + if (!/\.(ts|js|sh|json|jsonl)$/.test(entry.name)) continue; + if (fs.readFileSync(p, 'utf-8').includes(LIVE_HOST!)) offenders.push(path.relative(ROOT, p)); + } + }; + walk(path.join(ROOT, 'test')); + expect(offenders).toEqual([]); + }); +});