gstack/lib/gstack-memory-helpers.ts

/**
 * gstack-memory-helpers — shared helpers for the V1 memory ingest + retrieval pipeline.
 *
 * Imported by:
 *   - bin/gstack-memory-ingest.ts (Lane A)
 *   - bin/gstack-gbrain-sync.ts   (Lane B)
 *   - bin/gstack-brain-context-load.ts (Lane C)
 *   - scripts/gen-skill-docs.ts (manifest validation)
 *
 * Design refs in the plan:
 *   §"Eng review additions" — DRY refactor (Section 1A)
 *   §"V1 final scope clarification" — schema_version: 1 standardization (Section 2A)
 *   ED1 — engine-tier cache lives in ~/.gstack/.gbrain-engine-cache.json (60s TTL)
 *
 * NOTE: secretScanFile() currently shells out to `gitleaks` from PATH; the vendored
 * binary install is part of Lane E (setup-gbrain). When gitleaks is missing, the
 * helper warns once and returns an empty findings list — fail-safe defaults.
 */

import { existsSync, readFileSync, writeFileSync, mkdirSync, statSync } from "fs";
import { appendJsonl } from "./jsonl-store";
import { dirname, join } from "path";
import { execFileSync } from "child_process";
import { homedir } from "os";

// ── Types ──────────────────────────────────────────────────────────────────

export interface SecretFinding {
  rule_id: string;
  description: string;
  line: number;
  redacted_match: string;
}

export interface SecretScanResult {
  scanned: boolean;
  findings: SecretFinding[];
  scanner: "gitleaks" | "missing" | "error";
}

export type EngineTier = "pglite" | "supabase" | "unknown";

export interface EngineDetect {
  engine: EngineTier;
  supabase_url?: string;
  detected_at: number;
  schema_version: 1;
}

export interface GbrainManifestQuery {
  id: string;
  kind: "vector" | "list" | "filesystem";
  render_as: string;
  // kind=vector
  query?: string;
  // kind=list
  filter?: Record<string, unknown>;
  sort?: string;
  // kind=filesystem
  glob?: string;
  tail?: number;
  // common
  limit?: number;
}

export interface GbrainManifest {
  schema: number; // gbrain.schema in frontmatter; V1 = 1
  context_queries: GbrainManifestQuery[];
}

export interface ErrorContextEntry {
  ts: string;
  op: string;
  duration_ms: number;
  outcome: "ok" | "error";
  error?: string;
  schema_version: 1;
  last_writer: string;
}

// ── Public: canonicalizeRemote ────────────────────────────────────────────

/**
 * Normalize a git remote URL to a canonical form: `host/org/repo` (no scheme,
 * no trailing `.git`). Used as the dedup key for cross-Mac transcript routing
 * (per ED1 — gbrain-side session_id dedup uses repo as a tag).
 *
 * Examples:
 *   https://github.com/garrytan/gstack.git → github.com/garrytan/gstack
 *   git@github.com:garrytan/gstack.git     → github.com/garrytan/gstack
 *   ssh://git@gitlab.com/foo/bar           → gitlab.com/foo/bar
 *   (empty / null)                         → ""
 */
export function canonicalizeRemote(url: string | null | undefined): string {
  if (!url) return "";
  let s = url.trim();
  if (!s) return "";
  // strip surrounding quotes that some configs add
  s = s.replace(/^['"]|['"]$/g, "");
  // git@host:path/repo  →  host/path/repo
  const scpMatch = s.match(/^[^@\s]+@([^:]+):(.+)$/);
  if (scpMatch) {
    s = `${scpMatch[1]}/${scpMatch[2]}`;
  } else {
    // strip scheme (https://, ssh://, git://, http://)
    s = s.replace(/^[a-z][a-z0-9+.-]*:\/\//i, "");
    // strip user@ prefix on URL-style remotes
    s = s.replace(/^[^@\/]+@/, "");
  }
  // strip trailing slash(es) first, so a URL written with a trailing slash
  // still matches the `.git$` suffix below (e.g. ".../repo.git/" must
  // canonicalize to ".../repo", not ".../repo.git").
  s = s.replace(/\/+$/, "");
  // strip trailing .git
  s = s.replace(/\.git$/i, "");
  // re-strip trailing slash(es): a path remote ending in a `.git` directory
  // component ("/repo/.git") exposes a new trailing slash once `.git` is
  // stripped, which would split the repo into a second identity.
  s = s.replace(/\/+$/, "");
  // collapse multiple slashes (after path normalization)
  s = s.replace(/\/{2,}/g, "/");
  return s.toLowerCase();
}

// ── Public: secretScanFile (gitleaks wrapper) ─────────────────────────────

let _gitleaksAvailability: boolean | null = null;

function gitleaksAvailable(): boolean {
  if (_gitleaksAvailability !== null) return _gitleaksAvailability;
  try {
    execFileSync("gitleaks", ["version"], {
      env: process.env,
      stdio: "ignore",
      timeout: 2_000,
    });
    _gitleaksAvailability = true;
  } catch {
    _gitleaksAvailability = false;
    // Only warn once per process — Lane E will vendor the binary.
    process.stderr.write(
      "[gstack-memory-helpers] gitleaks not in PATH; secret scanning disabled. " +
      "Run /setup-gbrain to install (or `brew install gitleaks`).\n"
    );
  }
  return _gitleaksAvailability;
}

/**
 * Scan a file for embedded secrets using gitleaks. Returns findings list
 * (empty if clean). When gitleaks is not in PATH, returns scanned=false with
 * scanner="missing" — caller decides whether to skip the file or proceed.
 *
 * Per D19: gitleaks runs at ingest time before any put_page / put_file write.
 * Replaces the inadequate regex scanner in bin/gstack-brain-sync (which only
 * applies to staged git diffs).
 */
export function secretScanFile(path: string): SecretScanResult {
  if (!existsSync(path)) {
    return { scanned: false, findings: [], scanner: "error" };
  }
  if (!gitleaksAvailable()) {
    return { scanned: false, findings: [], scanner: "missing" };
  }
  try {
    // gitleaks detect --no-git --source <path> --report-format json --report-path -
    // Returns 0 on clean, 1 on findings, 126/127 on bad invocation.
    const out = execFileSync(
      "gitleaks",
      ["detect", "--no-git", "--source", path, "--report-format", "json", "--report-path", "/dev/stdout", "--exit-code", "0"],
      { encoding: "utf-8", env: process.env, maxBuffer: 16 * 1024 * 1024 }
    );
    const trimmed = out.trim();
    if (!trimmed) return { scanned: true, findings: [], scanner: "gitleaks" };
    const parsed = JSON.parse(trimmed) as Array<{
      RuleID: string;
      Description: string;
      StartLine: number;
      Match?: string;
      Secret?: string;
    }>;
    const findings: SecretFinding[] = (parsed || []).map((f) => ({
      rule_id: f.RuleID || "unknown",
      description: f.Description || "",
      line: f.StartLine || 0,
      redacted_match: redactMatch(f.Secret || f.Match || ""),
    }));
    return { scanned: true, findings, scanner: "gitleaks" };
  } catch (err) {
    return {
      scanned: false,
      findings: [],
      scanner: "error",
    };
  }
}

function redactMatch(s: string): string {
  if (!s) return "";
  if (s.length <= 8) return "[REDACTED]";
  return `${s.slice(0, 4)}...${s.slice(-4)}`;
}

// ── Public: detectEngineTier (cached) ─────────────────────────────────────

const ENGINE_CACHE_TTL_MS = 60 * 1000;

function gstackHome(): string {
  return process.env.GSTACK_HOME || join(homedir(), ".gstack");
}

function engineCachePath(): string {
  return join(gstackHome(), ".gbrain-engine-cache.json");
}

function errorLogPath(): string {
  return join(gstackHome(), ".gbrain-errors.jsonl");
}

/**
 * Detect which gbrain engine is active (PGLite vs Supabase) and cache the
 * answer for 60s in ~/.gstack/.gbrain-engine-cache.json. Caching avoids
 * fork+exec'ing `gbrain doctor --json` on every skill start.
 *
 * Per ED1 (state files local-only): this cache is gitignored from the brain
 * repo. Per Section 2A: schema_version: 1 + last_writer field for forensic
 * tracing.
 */
export function detectEngineTier(): EngineDetect {
  // Try cache first
  if (existsSync(engineCachePath())) {
    try {
      const stat = statSync(engineCachePath());
      const ageMs = Date.now() - stat.mtimeMs;
      if (ageMs < ENGINE_CACHE_TTL_MS) {
        const cached = JSON.parse(readFileSync(engineCachePath(), "utf-8")) as EngineDetect;
        if (cached.schema_version === 1) return cached;
      }
    } catch {
      // Cache corrupt; fall through to fresh detect.
    }
  }

  const fresh = freshDetectEngineTier();
  try {
    mkdirSync(dirname(engineCachePath()), { recursive: true });
    writeFileSync(
      engineCachePath(),
      JSON.stringify({ ...fresh, last_writer: "gstack-memory-helpers.detectEngineTier" }, null, 2),
      "utf-8"
    );
  } catch {
    // Cache write failure is non-fatal.
  }
  return fresh;
}

// Returns gbrain's config.json path, honoring GBRAIN_HOME env var with a
// fallback to ~/.gbrain. gbrain >=0.25 dropped the top-level `engine` field
// from doctor output, so this file is the only reliable source for engine
// detection on that version. See #1415.
function gbrainConfigPath(): string {
  const root = process.env.GBRAIN_HOME || join(homedir(), ".gbrain");
  return join(root, "config.json");
}

// Best-effort JSONL append to ~/.gstack/.gbrain-errors.jsonl. Never throws.
function logGbrainError(kind: string, detail: string): void {
  try {
    const path = errorLogPath();
    mkdirSync(dirname(path), { recursive: true });
    appendJsonl(path, { ts: new Date().toISOString(), kind, detail: detail.slice(0, 500) });
  } catch { /* logging is best-effort */ }
}

function freshDetectEngineTier(): EngineDetect {
  const now = Date.now();
  let parsed: Record<string, unknown> | null = null;

  // execFileSync (not execSync) avoids shell redirection — portable to
  // environments where `2>/dev/null` is bash-specific. The stdio array
  // suppresses stderr without invoking a shell.
  try {
    const out = execFileSync("gbrain", ["doctor", "--json", "--fast"], {
      encoding: "utf-8",
      timeout: 5000,
      stdio: ["ignore", "pipe", "ignore"],
    });
    parsed = JSON.parse(out);
  } catch (err: unknown) {
    // execFileSync throws on non-zero exit; stdout is still on the error
    // object. gbrain doctor exits 1 whenever health_score < 100, which is
    // essentially always on fresh installs (resolver_health warnings are
    // normal). Recover stdout and re-parse. See #1415.
    try {
      const stdout = (err as { stdout?: Buffer | string })?.stdout ?? "";
      const stdoutStr = typeof stdout === "string" ? stdout : stdout.toString("utf-8");
      if (stdoutStr) parsed = JSON.parse(stdoutStr);
    } catch (parseErr) {
      logGbrainError("doctor_parse_failure", String(parseErr));
    }
  }

  let engine: EngineTier =
    parsed?.engine === "supabase" ? "supabase" :
    parsed?.engine === "pglite"   ? "pglite"   : "unknown";

  // gbrain >=0.25 ships schema_version:2 doctor output which dropped the
  // top-level `engine` field. Fall back to gbrain's config.json (respects
  // GBRAIN_HOME). "supabase" here means "remote postgres" — gbrain config
  // uses engine:"postgres" for real Supabase AND any other remote postgres
  // (e.g. local-postgres-for-testing). Downstream sync code treats them the
  // same, so the label compression is intentional.
  if (engine === "unknown") {
    try {
      const cfg = JSON.parse(readFileSync(gbrainConfigPath(), "utf-8"));
      if (cfg?.engine === "pglite") engine = "pglite";
      else if (cfg?.engine === "postgres" || cfg?.database_url) engine = "supabase";
    } catch (cfgErr) {
      logGbrainError("config_read_failure", String(cfgErr));
    }
  }

  return {
    engine,
    supabase_url: parsed?.supabase_url as string | undefined,
    detected_at: now,
    schema_version: 1,
  };
}

// ── Public: parseSkillManifest ────────────────────────────────────────────

/**
 * Parse the `gbrain:` section out of a SKILL.md.tmpl frontmatter block.
 * Returns null if no manifest is declared OR if the file has no frontmatter.
 *
 * Schema validation (full kind/required-fields check) lives in
 * scripts/gen-skill-docs.ts and runs at generation time. This parser is the
 * runtime read path used by gstack-brain-context-load; it tolerates extra
 * fields and relies on validation having already happened upstream.
 */
export function parseSkillManifest(skillFilePath: string): GbrainManifest | null {
  if (!existsSync(skillFilePath)) return null;
  const content = readFileSync(skillFilePath, "utf-8");
  const frontmatter = extractFrontmatter(content);
  if (!frontmatter) return null;
  const gbrain = extractGbrainBlock(frontmatter);
  if (!gbrain) return null;
  return gbrain;
}

function extractFrontmatter(content: string): string | null {
  // Supports both `---\n...\n---` (YAML) and `+++\n...\n+++` (TOML, rare).
  const yamlMatch = content.match(/^---\s*\n([\s\S]*?)\n---\s*\n/);
  if (yamlMatch) return yamlMatch[1];
  return null;
}

function extractGbrainBlock(frontmatter: string): GbrainManifest | null {
  // Naive YAML extraction — finds the `gbrain:` key and parses its sub-tree.
  // Real YAML parsing avoided to keep zero-deps; gen-skill-docs validates the
  // shape strictly at build time.
  const lines = frontmatter.split("\n");
  const start = lines.findIndex((l) => /^gbrain\s*:/.test(l));
  if (start === -1) return null;

  // Collect indented lines under `gbrain:` until next top-level key or EOF
  const block: string[] = [];
  for (let i = start + 1; i < lines.length; i++) {
    const line = lines[i];
    if (/^[A-Za-z_][A-Za-z0-9_-]*\s*:/.test(line)) break; // next top-level key
    block.push(line);
  }

  const text = block.join("\n");
  // Extract schema number
  const schemaMatch = text.match(/\n\s*schema\s*:\s*(\d+)/);
  const schema = schemaMatch ? parseInt(schemaMatch[1], 10) : 1;

  // Extract context_queries items
  const queries: GbrainManifestQuery[] = [];
  const cqMatch = text.match(/\n\s*context_queries\s*:\s*\n([\s\S]+)/);
  if (cqMatch) {
    const cqText = cqMatch[1];
    // Split using a positive lookahead so each chunk begins with the list-item dash.
    // Pattern: line starting with 4-6 spaces + "-" + whitespace.
    const rawItems = cqText.split(/(?=^[ ]{4,6}-\s)/m);
    const items = rawItems.filter((s) => /^[ ]{4,6}-\s/.test(s));
    for (const item of items) {
      const q: Partial<GbrainManifestQuery> = {};
      // Strip the leading list-item marker so id/kind/etc. regexes can use line-start.
      const body = item.replace(/^[ ]{4,6}-\s+/, "      ");
      const idM = body.match(/(?:^|\n)\s*id\s*:\s*([^\n]+)/);
      const kindM = body.match(/(?:^|\n)\s*kind\s*:\s*([^\n]+)/);
      const renderM = body.match(/(?:^|\n)\s*render_as\s*:\s*"?([^"\n]+?)"?\s*$/m);
      const queryM = body.match(/(?:^|\n)\s*query\s*:\s*"?([^"\n]+?)"?\s*$/m);
      const limitM = body.match(/(?:^|\n)\s*limit\s*:\s*(\d+)/);
      const globM = body.match(/(?:^|\n)\s*glob\s*:\s*"?([^"\n]+?)"?\s*$/m);
      const sortM = body.match(/(?:^|\n)\s*sort\s*:\s*([^\n]+)/);
      const tailM = body.match(/(?:^|\n)\s*tail\s*:\s*(\d+)/);
      const filterMap = parseFilterMap(body);

      if (idM) q.id = idM[1].trim();
      if (kindM) {
        const k = kindM[1].trim();
        if (k === "vector" || k === "list" || k === "filesystem") q.kind = k;
      }
      if (renderM) q.render_as = renderM[1].trim();
      if (queryM) q.query = queryM[1].trim();
      if (limitM) q.limit = parseInt(limitM[1], 10);
      if (globM) q.glob = globM[1].trim();
      if (sortM) q.sort = sortM[1].trim();
      if (tailM) q.tail = parseInt(tailM[1], 10);
      if (filterMap) q.filter = filterMap;

      if (q.id && q.kind && q.render_as) {
        queries.push(q as GbrainManifestQuery);
      }
    }
  }

  return { schema, context_queries: queries };
}

/**
 * Parse a nested `filter:` block map out of a single context_queries item body.
 *
 * The block is a YAML map nested under the `filter:` key:
 *
 *   filter:
 *     type: timeline
 *     tags_contains: "repo:{repo_slug}"
 *
 * Each sub-key sits one indent level deeper than `filter:`. Surrounding quotes
 * are stripped and template vars ({repo_slug}, now-7d, ...) are left intact for
 * downstream substitution, matching how dispatchList stringifies each value
 * into a `--filter k=v` argument. Returns undefined when there is no `filter:`
 * block or it is empty.
 */
function parseFilterMap(body: string): Record<string, string> | undefined {
  const lines = body.split("\n");
  const filterIdx = lines.findIndex((l) => /^\s*filter\s*:\s*$/.test(l));
  if (filterIdx === -1) return undefined;
  const filterIndent = lines[filterIdx].match(/^\s*/)![0].length;

  const filter: Record<string, string> = {};
  for (let i = filterIdx + 1; i < lines.length; i++) {
    const line = lines[i];
    if (line.trim() === "") continue; // tolerate blank lines within the block
    const indent = line.match(/^\s*/)![0].length;
    if (indent <= filterIndent) break; // dedent to a sibling key ends the block
    const kv = line.match(/^\s*([A-Za-z0-9_]+)\s*:\s*"?(.*?)"?\s*$/);
    if (kv) filter[kv[1]] = kv[2].trim();
  }
  return Object.keys(filter).length > 0 ? filter : undefined;
}

// ── Public: withErrorContext ──────────────────────────────────────────────

const ERROR_LOG_PATH = join(gstackHome(), ".gbrain-errors.jsonl");

/**
 * Wrap an op with structured error logging. Logs success/failure + duration
 * to ~/.gstack/.gbrain-errors.jsonl for forensic debugging. Replaces ad-hoc
 * try/catch sites across the three Bun helpers (Section 2B).
 *
 * On error: the error is RE-THROWN after logging — caller still owns flow.
 */
export async function withErrorContext<T>(
  op: string,
  fn: () => T | Promise<T>,
  caller: string = "unknown"
): Promise<T> {
  const t0 = Date.now();
  try {
    const result = await fn();
    logErrorContext({
      ts: new Date().toISOString(),
      op,
      duration_ms: Date.now() - t0,
      outcome: "ok",
      schema_version: 1,
      last_writer: caller,
    });
    return result;
  } catch (err) {
    logErrorContext({
      ts: new Date().toISOString(),
      op,
      duration_ms: Date.now() - t0,
      outcome: "error",
      error: err instanceof Error ? err.message : String(err),
      schema_version: 1,
      last_writer: caller,
    });
    throw err;
  }
}

function logErrorContext(entry: ErrorContextEntry): void {
  try {
    const path = errorLogPath();
    mkdirSync(dirname(path), { recursive: true });
    appendJsonl(path, entry);
  } catch {
    // Logging failure is non-fatal — never block the op.
  }
}

// Test-only export for resetting the gitleaks availability cache between tests.
export function _resetGitleaksAvailabilityCache(): void {
  _gitleaksAvailability = null;
}