gstack/bin/gstack-telemetry-sync

#!/usr/bin/env bash
# gstack-telemetry-sync — sync local JSONL events to Supabase
#
# Fire-and-forget, backgrounded, rate-limited to once per 5 minutes.
# Strips local-only fields before sending. Respects privacy tiers.
# Posts to the telemetry-ingest edge function (not PostgREST directly).
#
# Env overrides (for testing):
#   GSTACK_HOME           — relocate the state root (chain: bin/gstack-state-root.sh)
#   GSTACK_DIR                 — override auto-detected gstack root
#   GSTACK_SUPABASE_URL        — override Supabase project URL
set -uo pipefail

GSTACK_DIR="${GSTACK_DIR:-$(cd "$(dirname "$0")/.." && pwd)}"
. "$(dirname "$0")/gstack-state-root.sh" 2>/dev/null || { echo "$0: cannot resolve the gstack state root: $(dirname "$0")/gstack-state-root.sh is missing. fix: reinstall with ./setup or /gstack-upgrade (docs/state-root.md)" >&2; exit 1; }
gstack_state_root_select
STATE_DIR="$_gstack_sr_root"

# Egress receipt helpers (_receipted_curl): receipt-before-send, fail-closed.
. "$GSTACK_DIR/bin/gstack-egress-lib.sh"
ANALYTICS_DIR="$STATE_DIR/analytics"
JSONL_FILE="$ANALYTICS_DIR/skill-usage.jsonl"
CURSOR_FILE="$ANALYTICS_DIR/.last-sync-line"
RATE_FILE="$ANALYTICS_DIR/.last-sync-time"
CONFIG_CMD="$GSTACK_DIR/bin/gstack-config"

# Source Supabase config if not overridden by env
if [ -z "${GSTACK_SUPABASE_URL:-}" ] && [ -f "$GSTACK_DIR/supabase/config.sh" ]; then
  . "$GSTACK_DIR/supabase/config.sh"
fi
SUPABASE_URL="${GSTACK_SUPABASE_URL:-}"
ANON_KEY="${GSTACK_SUPABASE_ANON_KEY:-}"

# ─── Pre-checks ──────────────────────────────────────────────
# No Supabase URL configured yet → exit silently
[ -z "$SUPABASE_URL" ] && exit 0

# No JSONL file → nothing to sync
[ -f "$JSONL_FILE" ] || exit 0

# Rate limit: once per 5 minutes
if [ -f "$RATE_FILE" ]; then
  STALE=$(find "$RATE_FILE" -mmin +5 2>/dev/null || true)
  [ -z "$STALE" ] && exit 0
fi

# ─── Read tier ───────────────────────────────────────────────
TIER="$("$CONFIG_CMD" get telemetry 2>/dev/null || true)"
TIER="${TIER:-off}"
[ "$TIER" = "off" ] && exit 0

# ─── Read cursor ─────────────────────────────────────────────
CURSOR=0
if [ -f "$CURSOR_FILE" ]; then
  CURSOR="$(cat "$CURSOR_FILE" 2>/dev/null | tr -d ' \n\r\t')"
  # Validate: must be a non-negative integer
  case "$CURSOR" in *[!0-9]*) CURSOR=0 ;; esac
fi

# Safety: if cursor exceeds file length, reset
TOTAL_LINES="$(wc -l < "$JSONL_FILE" | tr -d ' \n\r\t')"
if [ "$CURSOR" -gt "$TOTAL_LINES" ] 2>/dev/null; then
  CURSOR=0
fi

# Nothing new to sync
[ "$CURSOR" -ge "$TOTAL_LINES" ] 2>/dev/null && exit 0

# ─── Read unsent lines ───────────────────────────────────────
SKIP=$(( CURSOR + 1 ))
UNSENT="$(tail -n "+$SKIP" "$JSONL_FILE" 2>/dev/null || true)"
[ -z "$UNSENT" ] && exit 0

# ─── Strip local-only fields and build batch ─────────────────
# Edge function expects raw JSONL field names (v, ts, sessions) —
# no column renaming needed (the function maps them internally).
BATCH="["
FIRST=true
COUNT=0

while IFS= read -r LINE; do
  # Skip empty or malformed lines
  [ -z "$LINE" ] && continue
  echo "$LINE" | grep -q '^{' || continue

  # Strip local-only fields (keep v, ts, sessions as-is for edge function).
  # jq del() is structural — a value containing an escaped quote (repo names,
  # branch names) can't smuggle the field past a regex or corrupt the strip.
  # The sed path stays only as a jq-less fallback.
  if command -v jq >/dev/null 2>&1; then
    if [ "$TIER" = "anonymous" ]; then
      CLEAN="$(printf '%s' "$LINE" | jq -c 'del(._repo_slug, ._branch, .repo, .installation_id)' 2>/dev/null)" || CLEAN=""
    else
      CLEAN="$(printf '%s' "$LINE" | jq -c 'del(._repo_slug, ._branch, .repo)' 2>/dev/null)" || CLEAN=""
    fi
    # A line jq can't parse is malformed telemetry — drop it rather than
    # forwarding bytes the strip never touched.
    [ -z "$CLEAN" ] && continue
  else
    CLEAN="$(echo "$LINE" | sed \
      -e 's/,"_repo_slug":"[^"]*"//g' \
      -e 's/,"_branch":"[^"]*"//g' \
      -e 's/,"repo":"[^"]*"//g')"
    if [ "$TIER" = "anonymous" ]; then
      CLEAN="$(echo "$CLEAN" | sed 's/,"installation_id":"[^"]*"//g; s/,"installation_id":null//g')"
    fi
  fi

  if [ "$FIRST" = "true" ]; then
    FIRST=false
  else
    BATCH="$BATCH,"
  fi
  BATCH="$BATCH$CLEAN"
  COUNT=$(( COUNT + 1 ))

  # Batch size limit
  [ "$COUNT" -ge 100 ] && break
done <<< "$UNSENT"

BATCH="$BATCH]"

# Nothing to send after filtering
[ "$COUNT" -eq 0 ] && exit 0

# ─── POST to edge function ───────────────────────────────────
# Create response file atomically. If mktemp fails, refuse to continue rather
# than fall back to a predictable $$-based path (race + overwrite footgun).
RESP_FILE="$(mktemp "${TMPDIR:-/tmp}/gstack-sync-XXXXXX")" || {
  echo "gstack-telemetry-sync: mktemp failed — skipping this run" >&2
  exit 0
}
trap 'rm -f "$RESP_FILE"' EXIT

# Egress receipt BEFORE the send (fail-closed): the batch is written to a
# temp file, the receipt hashes those exact bytes, and curl sends the SAME
# file. On refusal nothing is sent and the cursor does not advance, so the
# batch stays buffered and the next run retries. The helper consumes the
# payload file.
PAYLOAD_FILE="$(mktemp "${TMPDIR:-/tmp}/gstack-sync-payload-XXXXXX")" || {
  echo "gstack-telemetry-sync: mktemp failed — skipping this run" >&2
  exit 0
}
printf '%s' "$BATCH" > "$PAYLOAD_FILE"
DEST_HOST="${SUPABASE_URL#*://}"
DEST_HOST="${DEST_HOST%%/*}"
HTTP_CODE="$(GSTACK_HOME="$STATE_DIR" _receipted_curl closed telemetry-sync "$DEST_HOST" telemetry-events "telemetry=$TIER" "$PAYLOAD_FILE" \
  curl -s -w '%{http_code}' --max-time 10 \
  -X POST "${SUPABASE_URL}/functions/v1/telemetry-ingest" \
  -H "Content-Type: application/json" \
  -H "apikey: ${ANON_KEY}" \
  -o "$RESP_FILE" || echo "000")"

# Best-effort response-status record for the receipt above (overrides the
# helper's generic exit-code outcome with the HTTP status).
if [ -n "${_GSTACK_EGRESS_LAST_RECEIPT:-}" ]; then
  GSTACK_HOME="$STATE_DIR" "$GSTACK_DIR/bin/gstack-egress-receipt" outcome "$_GSTACK_EGRESS_LAST_RECEIPT" "$HTTP_CODE" 2>/dev/null || true
fi

# ─── Update cursor on success (2xx) ─────────────────────────
case "$HTTP_CODE" in
  2*)
    # Parse inserted count from response — only advance if events were actually inserted.
    # Advance by SENT count (not inserted count) because we can't map inserted back to
    # source lines. If inserted==0, something is systemically wrong — don't advance.
    INSERTED="$(grep -o '"inserted":[0-9]*' "$RESP_FILE" 2>/dev/null | grep -o '[0-9]*' || echo "0")"
    # Check for upsert errors (installation tracking failures) — log but don't block cursor advance
    UPSERT_ERRORS="$(grep -o '"upsertErrors"' "$RESP_FILE" 2>/dev/null || true)"
    if [ -n "$UPSERT_ERRORS" ]; then
      echo "[gstack-telemetry-sync] Warning: installation upsert errors in response" >&2
    fi
    if [ "${INSERTED:-0}" -gt 0 ] 2>/dev/null; then
      NEW_CURSOR=$(( CURSOR + COUNT ))
      echo "$NEW_CURSOR" > "$CURSOR_FILE" 2>/dev/null || true
    fi
    ;;
esac

rm -f "$RESP_FILE" 2>/dev/null || true

# Update rate limit marker
touch "$RATE_FILE" 2>/dev/null || true

exit 0