gstack/test/telemetry-repo-strip.test.ts

/**
 * Telemetry "no repo identity egress" invariant.
 *
 * The telemetry consent copy promises a user's repo name is recorded locally
 * only and stripped before any upload (scripts/resolvers/preamble/
 * generate-telemetry-prompt.ts). The producers that write repo/branch identity
 * into the local skill-usage.jsonl (the preamble's inline bash moved into the
 * skill-start/skill-end scripts in token-reduction Phase 1):
 *
 *   - gstack-skill-start (skill_run event)   → "repo"
 *     (bin/gstack-skill-start)
 *   - gstack-skill-end (completion event)    → (no repo identity today,
 *     scanned so drift is caught)             (bin/gstack-skill-end)
 *   - gstack-telemetry-log                   → "_repo_slug", "_branch"
 *     (bin/gstack-telemetry-log)
 *
 * gstack-telemetry-sync MUST strip every one of those fields before the remote
 * POST (bin/gstack-telemetry-sync). The script has TWO strip paths — jq del()
 * is PRIMARY (structural, escape-proof), sed is the jq-less fallback — and
 * this test enforces the contract on both:
 *
 *   1. Coverage — every repo/branch field the producers emit is also stripped,
 *      by every jq del() list AND by the sed expressions. Catches "added a new
 *      repo field, forgot to strip it" (the rename-to-_repo landmine, or any
 *      future producer drift) on whichever path a machine takes.
 *   2. Behavior — run the ACTUAL jq expression and the ACTUAL sed strip
 *      expressions from the sync script over a sample event line and assert no
 *      repo/branch field survives, while benign fields do. Catches a
 *      broken/edited filter, not just a missing line. The jq leg also pins the
 *      malformed-line contract: a line jq can't parse is dropped, never
 *      forwarded unstripped.
 *   3. Floor — the three known fields are always in the stripped set, so deleting
 *      a strip rule fails CI even if a producer also stops emitting it.
 */

import { describe, test, expect } from 'bun:test';
import { spawnSync } from 'bun';
import fs from 'fs';
import path from 'path';

const ROOT = path.resolve(__dirname, '..');
const SYNC = path.join(ROOT, 'bin', 'gstack-telemetry-sync');
const SKILL_START = path.join(ROOT, 'bin', 'gstack-skill-start');
const SKILL_END = path.join(ROOT, 'bin', 'gstack-skill-end');
const TEL_LOG = path.join(ROOT, 'bin', 'gstack-telemetry-log');

// Fields that identify the user's repo/branch. The promise is that NONE of
// these reach the network. Add to this floor if a new identity field is born.
const REPO_IDENTITY_FLOOR = ['repo', '_repo_slug', '_branch'];

const isRepoIdentity = (field: string) => /repo|branch/i.test(field);

/** Pull every `sed -e 's/.../g'` expression out of the sync script. */
function extractSedExprs(scriptText: string): string[] {
  return [...scriptText.matchAll(/-e\s+'(s\/[^']*)'/g)].map((m) => m[1]);
}

/** Pull every `jq -c 'del(...)'` filter out of the sync script, verbatim. */
function extractJqDelFilters(scriptText: string): string[] {
  return [...scriptText.matchAll(/jq -c '(del\([^']*\))'/g)].map((m) => m[1]);
}

/** The JSON keys a jq del() filter removes, e.g. `del(._repo_slug, .repo)`. */
function fieldsFromJqDel(filter: string): string[] {
  return [...filter.matchAll(/\.([A-Za-z_][A-Za-z0-9_]*)/g)].map((m) => m[1]);
}

/** The JSON key a strip expression targets, e.g. `,"repo":"[^"]*"` -> `repo`. */
function fieldFromSedExpr(expr: string): string | null {
  const m = expr.match(/,"([A-Za-z_][A-Za-z0-9_]*)":/);
  return m ? m[1] : null;
}

/**
 * Repo/branch JSON keys a producer writes INTO skill-usage.jsonl — the only
 * file gstack-telemetry-sync reads and uploads. Scoped to the emission lines
 * that target the synced file so local-only sinks (e.g. the timeline log, which
 * carries "branch" but is never synced) don't count against the egress invariant.
 */
function emittedRepoFields(lines: string[]): string[] {
  const text = lines.join('\n');
  const keys = [...text.matchAll(/"([A-Za-z_][A-Za-z0-9_]*)":/g)].map((m) => m[1]);
  return [...new Set(keys.filter(isRepoIdentity))];
}

describe('telemetry no-repo-identity-egress invariant', () => {
  const syncText = fs.readFileSync(SYNC, 'utf-8');
  const sedExprs = extractSedExprs(syncText);
  const strippedRepoExprs = sedExprs.filter((e) => {
    const f = fieldFromSedExpr(e);
    return f !== null && isRepoIdentity(f);
  });
  const strippedFields = new Set(
    strippedRepoExprs.map(fieldFromSedExpr).filter((f): f is string => f !== null),
  );
  const jqFilters = extractJqDelFilters(syncText);

  // Repo-identity fields the producers emit into the synced file — computed
  // once, asserted against BOTH strip paths (jq primary, sed fallback). Only
  // emission lines that target the synced file (skill-usage.jsonl) count: the
  // skill-start/skill-end scripts append directly (the former inline preamble
  // bash); gstack-telemetry-log builds the synced event with a
  // `printf '{"v":1,...` line into $JSONL_FILE (= skill-usage.jsonl). The
  // timeline log carries "branch" but is local-only and never synced.
  const skillStartSynced = fs
    .readFileSync(SKILL_START, 'utf-8')
    .split('\n')
    .filter((l) => l.includes('skill-usage.jsonl'));
  const skillEndSynced = fs
    .readFileSync(SKILL_END, 'utf-8')
    .split('\n')
    .filter((l) => l.includes('skill-usage.jsonl'));
  const telLogSynced = fs
    .readFileSync(TEL_LOG, 'utf-8')
    .split('\n')
    .filter((l) => l.includes('"v":1') || l.includes('skill-usage'));
  const emitted = new Set<string>([
    ...emittedRepoFields(skillStartSynced),
    ...emittedRepoFields(skillEndSynced),
    ...emittedRepoFields(telLogSynced),
  ]);

  test('floor: the three known repo-identity fields are stripped', () => {
    for (const field of REPO_IDENTITY_FLOOR) {
      expect(strippedFields.has(field)).toBe(true);
    }
  });

  test('coverage: every repo/branch field the producers emit into skill-usage.jsonl is stripped (sed fallback path)', () => {
    // gstack-skill-start must emit "repo" — guards against the test silently
    // passing because a regex stopped matching the producer.
    expect(emitted.has('repo')).toBe(true);
    for (const field of emitted) {
      expect(
        strippedFields.has(field),
        `producer emits repo-identity field "${field}" but gstack-telemetry-sync's sed fallback does not strip it (would leak to remote)`,
      ).toBe(true);
    }
  });

  test('coverage: every jq del() list (the PRIMARY strip path) covers every emitted repo-identity field', () => {
    // Both tiers run a del() filter; each must strip full repo identity on its
    // own — a machine only ever takes one branch.
    expect(jqFilters.length).toBeGreaterThanOrEqual(2);
    expect(emitted.has('repo')).toBe(true); // producer-regex canary, as above
    for (const filter of jqFilters) {
      const delFields = new Set(fieldsFromJqDel(filter));
      for (const field of [...emitted, ...REPO_IDENTITY_FLOOR]) {
        expect(
          delFields.has(field),
          `jq filter "${filter}" does not del repo-identity field "${field}" (primary strip path would leak it to remote)`,
        ).toBe(true);
      }
    }
  });

  test('behavior: the real sed expressions remove repo identity, keep benign fields', () => {
    const sample =
      '{"v":1,"ts":"2026-06-02T00:00:00Z","skill":"design-shotgun",' +
      '"repo":"my-secret-repo","_repo_slug":"acme-my-secret-repo","_branch":"feature-x",' +
      '"sessions":3,"installation_id":"abc123"}';

    const sedArgs: string[] = [];
    for (const e of strippedRepoExprs) {
      sedArgs.push('-e', e);
    }
    const out = spawnSync(['sed', ...sedArgs], {
      stdin: Buffer.from(sample),
      timeout: 30_000,
    });
    const cleaned = out.stdout.toString();

    // No repo/branch identity survives, value or key.
    expect(cleaned).not.toContain('my-secret-repo');
    expect(cleaned).not.toContain('feature-x');
    expect(cleaned).not.toContain('"repo"');
    expect(cleaned).not.toContain('_repo_slug');
    expect(cleaned).not.toContain('_branch');

    // Benign fields are untouched — the strip is surgical, not a blanket wipe.
    expect(cleaned).toContain('"skill":"design-shotgun"');
    expect(cleaned).toContain('"sessions":3');
    expect(cleaned).toContain('"ts":"2026-06-02T00:00:00Z"');
  });

  test('behavior: the real jq del() filters strip repo identity and drop malformed lines', () => {
    if (!Bun.which('jq')) return; // jq-less machine: the sed-fallback behavior test above is the live path

    const sample =
      '{"v":1,"ts":"2026-06-02T00:00:00Z","skill":"design-shotgun",' +
      '"repo":"my-secret-repo","_repo_slug":"acme-my-secret-repo","_branch":"feature-x",' +
      '"sessions":3,"installation_id":"abc123"}';

    // The identified-tier filter (no installation_id in its del list) and the
    // anonymous-tier filter (installation_id included) — run each verbatim.
    const identified = jqFilters.find((f) => !f.includes('installation_id'));
    const anonymous = jqFilters.find((f) => f.includes('installation_id'));
    expect(identified).toBeTruthy();
    expect(anonymous).toBeTruthy();

    const runJq = (filter: string, input: string) => {
      const out = spawnSync(['jq', '-c', filter], { stdin: Buffer.from(input), timeout: 30_000 });
      return { exitCode: out.exitCode, stdout: out.stdout.toString().trim() };
    };

    const id = runJq(identified!, sample);
    expect(id.exitCode).toBe(0);
    // No repo/branch identity survives, value or key.
    expect(id.stdout).not.toContain('my-secret-repo');
    expect(id.stdout).not.toContain('feature-x');
    expect(id.stdout).not.toContain('"repo"');
    expect(id.stdout).not.toContain('_repo_slug');
    expect(id.stdout).not.toContain('_branch');
    // Benign fields are untouched; identified tier keeps installation_id.
    expect(id.stdout).toContain('"skill":"design-shotgun"');
    expect(id.stdout).toContain('"sessions":3');
    expect(id.stdout).toContain('"installation_id":"abc123"');

    // Anonymous tier additionally drops installation_id.
    const anon = runJq(anonymous!, sample);
    expect(anon.exitCode).toBe(0);
    expect(anon.stdout).not.toContain('installation_id');
    expect(anon.stdout).not.toContain('my-secret-repo');

    // Malformed line: jq fails and emits nothing — the sync script's
    // `|| CLEAN=""` + `[ -z "$CLEAN" ] && continue` drops it, so bytes the
    // strip never touched are never forwarded.
    const bad = runJq(identified!, '{"v":1,"repo":"my-secret-repo"');
    expect(bad.exitCode).not.toBe(0);
    expect(bad.stdout).toBe('');
  });
});