mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-09 22:48:57 +02:00
feat(code-intelligence): provider contract Phase 1 — GBrain, Sourcebot, Graphify behind one ask-once offer
Open a large repo (1,000+ tracked files) and gstack can offer code intelligence ONCE, with the trade-offs stated: GBrain (semantic memory + code, sends content to YOUR gbrain DB, per-repo consent), Sourcebot (self-hosted whole-repo search, local on localhost), Graphify (local tree-sitter graph, nothing leaves the machine, user-installed), or No indexing — a decline persists machine-wide so no skill ever asks again. Small repos never see the question; grep stays the always-working default and provider-OFF degrades silently (PROVIDER_UNAVAILABLE -> file-only). Ported: lib/code-intelligence/ (contract + 3 verified adapters + picker + selection + suggest, MIT headers), the gstack-code-intelligence CLI (suggest/select/consent/index/search/status), 31 offline tests (fake CLI shims + injected fetch), and the provider-contract design doc. Verified live on this repo: suggest fires at 1,233 files with real availability detail per provider. Hardened per review: the per-remote trust store is the SINGLE consent authority — a gstack-gbrain-repo-policy deny tier vetoes any recorded code-intelligence consent (fail-closed on an unreadable store, pinned by three tests); both send-capable adapters are registered as fail-closed MODULE_SINKS in the egress tripwire so a refactor can't drop their receipts; and local-compute vs remote-send consents are never bundled. setup-gbrain gains the provider-choice Step 0. The fork's Phases 2-4 glue-collapse is explicitly NOT ported. Ported from time-attack/gstack (GStack 2); consent unification ours. Co-authored-by: Sina Matian <sina@time-attack.dev> Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Sina Matian
Claude Fable 5
parent
53deeeb116
commit
3aba218303
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* code-intelligence/contract — the OPTIONAL, repo-oriented provider contract.
|
||||
*
|
||||
* gstack does not maintain a home-grown indexer. It defines this small contract
|
||||
* and external providers (GBrain, Sourcebot, Graphify) implement it. The whole
|
||||
* contract is OPTIONAL: when no provider is available/consented,
|
||||
* `resolveCodeProvider()` returns null and callers degrade to grep / the
|
||||
* file-only decision store. Never a dependency, always an enhancement — the same
|
||||
* reliability contract as lib/gstack-decision-semantic.ts.
|
||||
*
|
||||
* Repo-oriented, not document-store (settled): register_source / refresh /
|
||||
* search / status are required; add / delete / export are optional capabilities
|
||||
* a provider MAY advertise. A document-CRUD-required contract would misrepresent
|
||||
* whole-repo code-search and code-graph tools. See
|
||||
* docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md.
|
||||
*/
|
||||
|
||||
export type CodeProviderId = "gbrain" | "sourcebot" | "graphify";
|
||||
|
||||
export type CodeProviderCapability =
|
||||
| "register_source"
|
||||
| "refresh"
|
||||
| "search"
|
||||
| "status"
|
||||
| "add"
|
||||
| "delete"
|
||||
| "export";
|
||||
|
||||
export const REQUIRED_CAPABILITIES: readonly CodeProviderCapability[] = [
|
||||
"register_source",
|
||||
"refresh",
|
||||
"search",
|
||||
"status",
|
||||
] as const;
|
||||
|
||||
export const OPTIONAL_CAPABILITIES: readonly CodeProviderCapability[] = [
|
||||
"add",
|
||||
"delete",
|
||||
"export",
|
||||
] as const;
|
||||
|
||||
export interface RepoRef {
|
||||
/** Source id the provider registers this repo under. */
|
||||
id: string;
|
||||
/** Local worktree path. */
|
||||
path: string;
|
||||
/** Remote URL, when the provider clones/manages it. */
|
||||
remoteUrl?: string;
|
||||
}
|
||||
|
||||
export interface SourceRef {
|
||||
id: string;
|
||||
}
|
||||
|
||||
export interface SourceStatus {
|
||||
id: string;
|
||||
state: "registered" | "indexing" | "ready" | "absent" | "unknown";
|
||||
/** Pages / files / graph nodes, when the provider reports a count. */
|
||||
itemCount?: number;
|
||||
detail?: string;
|
||||
/** True when the provider only implements a partial status probe. */
|
||||
partial?: boolean;
|
||||
}
|
||||
|
||||
export interface CodeSearchHit {
|
||||
/** Slug, file path, or symbol id — whatever the provider keys results on. */
|
||||
ref: string;
|
||||
score?: number;
|
||||
snippet?: string;
|
||||
kind?: "document" | "file" | "symbol" | "graph-node";
|
||||
}
|
||||
|
||||
export interface OpOptions {
|
||||
/**
|
||||
* Env override for spawned processes. Production callers leave this unset;
|
||||
* tests inject a synthetic env (fake CLI on PATH). Matches the existing
|
||||
* gbrain helpers.
|
||||
*/
|
||||
env?: NodeJS.ProcessEnv;
|
||||
/** Timeout in ms for the underlying op. */
|
||||
timeout?: number;
|
||||
/**
|
||||
* Explicit per-repo consent that repo content may leave the machine. Required
|
||||
* for non-local providers on register_source / refresh / add.
|
||||
*/
|
||||
consented?: boolean;
|
||||
}
|
||||
|
||||
export interface SearchOptions extends OpOptions {
|
||||
/** Restrict to a registered source. */
|
||||
source?: string;
|
||||
limit?: number;
|
||||
minScore?: number;
|
||||
}
|
||||
|
||||
export interface CodeProvider {
|
||||
readonly id: CodeProviderId;
|
||||
readonly label: string;
|
||||
readonly capabilities: ReadonlySet<CodeProviderCapability>;
|
||||
/** True when no repo content leaves the machine (Graphify). */
|
||||
readonly local: boolean;
|
||||
|
||||
has(capability: CodeProviderCapability): boolean;
|
||||
|
||||
registerSource(repo: RepoRef, opts?: OpOptions): Promise<SourceStatus>;
|
||||
refresh(source: SourceRef, opts?: OpOptions): Promise<SourceStatus>;
|
||||
search(query: string, opts?: SearchOptions): Promise<CodeSearchHit[]>;
|
||||
status(source?: SourceRef, opts?: OpOptions): Promise<SourceStatus>;
|
||||
|
||||
add?(doc: { slug: string; body: string }, opts?: OpOptions): Promise<SourceStatus>;
|
||||
delete?(slug: string, opts?: OpOptions): Promise<SourceStatus>;
|
||||
export?(source: SourceRef, opts?: OpOptions): Promise<string>;
|
||||
}
|
||||
|
||||
export const CODE_PROVIDER_FAILURES = Object.freeze([
|
||||
"PROVIDER_UNAVAILABLE",
|
||||
"PROVIDER_NOT_CONSENTED",
|
||||
"CAPABILITY_UNSUPPORTED",
|
||||
"SOURCE_NOT_REGISTERED",
|
||||
"PROVIDER_TIMEOUT",
|
||||
"PROVIDER_ERROR",
|
||||
] as const);
|
||||
|
||||
export type CodeProviderFailure = (typeof CODE_PROVIDER_FAILURES)[number];
|
||||
|
||||
const FAILURE_SET = new Set<string>(CODE_PROVIDER_FAILURES);
|
||||
|
||||
/**
|
||||
* Typed provider failure. Mirrors runtime/context.js ContextError discipline:
|
||||
* the code set is closed and the constructor throws on an unknown code, so a
|
||||
* typo can never mint an untyped failure.
|
||||
*/
|
||||
export class CodeProviderError extends Error {
|
||||
readonly code: CodeProviderFailure;
|
||||
readonly providerId?: CodeProviderId;
|
||||
|
||||
constructor(code: CodeProviderFailure, message: string, providerId?: CodeProviderId) {
|
||||
if (!FAILURE_SET.has(code)) throw new TypeError(`Unknown code-provider failure code: ${code}`);
|
||||
super(message);
|
||||
this.name = "CodeProviderError";
|
||||
this.code = code;
|
||||
this.providerId = providerId;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enforce that a provider advertises every required capability. Called by each
|
||||
* adapter constructor so an incomplete provider fails fast, not at first search.
|
||||
*/
|
||||
export function assertRequiredCapabilities(
|
||||
id: CodeProviderId,
|
||||
capabilities: ReadonlySet<CodeProviderCapability>,
|
||||
): void {
|
||||
const missing = REQUIRED_CAPABILITIES.filter((cap) => !capabilities.has(cap));
|
||||
if (missing.length) {
|
||||
throw new TypeError(`Code provider ${id} is missing required capabilities: ${missing.join(", ")}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Guard for optional ops: throw CAPABILITY_UNSUPPORTED (never a silent no-op)
|
||||
* when a provider is asked for a capability it does not advertise.
|
||||
*/
|
||||
export function assertCapability(provider: CodeProvider, capability: CodeProviderCapability): void {
|
||||
if (!provider.has(capability)) {
|
||||
throw new CodeProviderError(
|
||||
"CAPABILITY_UNSUPPORTED",
|
||||
`${provider.label} does not support "${capability}"`,
|
||||
provider.id,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Repo-scoped egress consent gate. Non-local providers must not move repo
|
||||
* content off the machine without explicit per-repo consent. Local providers
|
||||
* (nothing leaves the machine) are exempt.
|
||||
*/
|
||||
export function assertEgressConsent(provider: CodeProvider, opts?: OpOptions): void {
|
||||
if (provider.local) return;
|
||||
if (opts?.consented === true) return;
|
||||
throw new CodeProviderError(
|
||||
"PROVIDER_NOT_CONSENTED",
|
||||
`${provider.label} would send repo content off this machine; per-repo indexing consent is required`,
|
||||
provider.id,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
/**
|
||||
* GBrain adapter — full contract fit over the existing gbrain CLI chokepoint.
|
||||
*
|
||||
* Reuses lib/gbrain-exec.ts (spawnGbrain, seeded DATABASE_URL) and
|
||||
* lib/gbrain-sources.ts (ensureSourceRegistered, probeSource, sourcePageCount)
|
||||
* rather than re-issuing raw commands, so the DATABASE_URL / GBRAIN_HOME /
|
||||
* Windows-shim guarantees carry over unchanged. GBrain's native primitive is
|
||||
* document-by-slug (put/delete/get/export) PLUS a repo axis (sources add/sync),
|
||||
* so it advertises all seven capabilities.
|
||||
*/
|
||||
|
||||
import { spawnSync } from "child_process";
|
||||
import { sha256Hex, writeReceipt } from "../egress-receipt.js";
|
||||
import { spawnGbrain, buildGbrainEnv, NEEDS_SHELL_ON_WINDOWS } from "../gbrain-exec";
|
||||
import { ensureSourceRegistered, probeSource, sourcePageCount } from "../gbrain-sources";
|
||||
import {
|
||||
assertCapability,
|
||||
assertEgressConsent,
|
||||
assertRequiredCapabilities,
|
||||
CodeProviderError,
|
||||
type CodeProvider,
|
||||
type CodeProviderCapability,
|
||||
type CodeSearchHit,
|
||||
type OpOptions,
|
||||
type RepoRef,
|
||||
type SearchOptions,
|
||||
type SourceRef,
|
||||
type SourceStatus,
|
||||
} from "./contract";
|
||||
|
||||
const CAPABILITIES: CodeProviderCapability[] = [
|
||||
"register_source",
|
||||
"refresh",
|
||||
"search",
|
||||
"status",
|
||||
"add",
|
||||
"delete",
|
||||
"export",
|
||||
];
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
|
||||
/**
|
||||
* Parse `gbrain search` text output (`[score] slug -- snippet`) into hits.
|
||||
* gbrain's search prints text, not JSON (verified in
|
||||
* lib/gstack-decision-semantic.ts). Exported for deterministic unit testing.
|
||||
*/
|
||||
export function parseGbrainSearch(stdout: string, minScore: number, limit: number): CodeSearchHit[] {
|
||||
const hits: CodeSearchHit[] = [];
|
||||
for (const line of stdout.split("\n")) {
|
||||
const m = line.match(/^\[([\d.]+)\]\s+(\S+)\s+--\s+(.*)$/);
|
||||
if (!m) continue;
|
||||
const score = parseFloat(m[1]);
|
||||
if (!Number.isFinite(score) || score < minScore) continue;
|
||||
hits.push({ ref: m[2], score, snippet: m[3].trim(), kind: "document" });
|
||||
}
|
||||
return hits.slice(0, limit);
|
||||
}
|
||||
|
||||
export class GbrainProvider implements CodeProvider {
|
||||
readonly id = "gbrain" as const;
|
||||
readonly label = "GBrain";
|
||||
readonly capabilities = new Set<CodeProviderCapability>(CAPABILITIES);
|
||||
/** GBrain federates into a (possibly remote) DB, so content can leave the machine. */
|
||||
readonly local = false;
|
||||
|
||||
constructor() {
|
||||
assertRequiredCapabilities(this.id, this.capabilities);
|
||||
}
|
||||
|
||||
has(capability: CodeProviderCapability): boolean {
|
||||
return this.capabilities.has(capability);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fail-closed egress receipt for the write ops (register/refresh/add). The
|
||||
* gbrain subprocess owns the wire bytes, so the receipt records destination
|
||||
* + payload class; sha256 is only known for `add` (the exact document body).
|
||||
*/
|
||||
#receipt(payloadClass: string, opts: OpOptions, body?: string): void {
|
||||
writeReceipt({
|
||||
env: opts.env,
|
||||
sink: "gbrain",
|
||||
host: "gbrain-db (user-configured DATABASE_URL)",
|
||||
payloadClass,
|
||||
bytes: body == null ? 0 : Buffer.byteLength(body),
|
||||
sha256: body == null ? null : sha256Hex(body),
|
||||
consent: "code-intelligence provider=gbrain + per-repo consented=true",
|
||||
});
|
||||
}
|
||||
|
||||
async registerSource(repo: RepoRef, opts: OpOptions = {}): Promise<SourceStatus> {
|
||||
assertEgressConsent(this, opts);
|
||||
this.#receipt("repo-source-registration (sent by gbrain subprocess)", opts);
|
||||
try {
|
||||
const result = await ensureSourceRegistered(repo.id, repo.path, {
|
||||
federated: true,
|
||||
env: opts.env,
|
||||
});
|
||||
return {
|
||||
id: repo.id,
|
||||
state: result.state.status === "match" ? "registered" : "unknown",
|
||||
detail: result.changed ? "registered" : "already registered",
|
||||
};
|
||||
} catch (err) {
|
||||
throw this.#wrap(err);
|
||||
}
|
||||
}
|
||||
|
||||
async refresh(source: SourceRef, opts: OpOptions = {}): Promise<SourceStatus> {
|
||||
assertEgressConsent(this, opts);
|
||||
this.#receipt("repo-code-index (sent by gbrain subprocess)", opts);
|
||||
const timeout = opts.timeout ?? DEFAULT_TIMEOUT_MS;
|
||||
// Two passes, verified end-to-end against real Postgres-backed gbrain 0.42.56:
|
||||
// 1. default sync (markdown strategy) — indexes docs.
|
||||
// 2. `sync --strategy code` — the ACTUAL code-indexing pass. Without it code
|
||||
// is never indexed (the whole point of a code provider); `code-def` stays
|
||||
// "not_built" and search only finds incidental doc mentions. `--full`
|
||||
// forces it past the per-source checkpoint the markdown pass advanced.
|
||||
this.#assertOk(spawnGbrain(["sync", "--source", source.id], { baseEnv: opts.env, timeout }));
|
||||
this.#assertOk(spawnGbrain(["sync", "--source", source.id, "--strategy", "code", "--full"], { baseEnv: opts.env, timeout }));
|
||||
return this.status(source, opts);
|
||||
}
|
||||
|
||||
async search(query: string, opts: SearchOptions = {}): Promise<CodeSearchHit[]> {
|
||||
if (!query.trim()) return [];
|
||||
// `gbrain search` is global and has no `--source` flag; `--limit` is real
|
||||
// (verified against gbrain 0.42.x --help).
|
||||
const args = ["search", query];
|
||||
if (opts.limit) args.push("--limit", String(opts.limit));
|
||||
const r = spawnGbrain(args, { baseEnv: opts.env, timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS });
|
||||
this.#assertOk(r);
|
||||
return parseGbrainSearch(r.stdout || "", opts.minScore ?? 0.1, opts.limit ?? 10);
|
||||
}
|
||||
|
||||
async status(source?: SourceRef, opts: OpOptions = {}): Promise<SourceStatus> {
|
||||
if (!source) {
|
||||
// No source given: liveness probe. `sources list` reachable = ready.
|
||||
this.#assertOk(spawnGbrain(["sources", "list", "--json"], {
|
||||
baseEnv: opts.env,
|
||||
timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS,
|
||||
}));
|
||||
return { id: "*", state: "ready" };
|
||||
}
|
||||
try {
|
||||
const probed = probeSource(source.id, opts.env);
|
||||
if (probed.status === "absent") return { id: source.id, state: "absent" };
|
||||
const count = sourcePageCount(source.id, opts.env);
|
||||
return {
|
||||
id: source.id,
|
||||
state: "ready",
|
||||
itemCount: count ?? undefined,
|
||||
detail: probed.registered_path,
|
||||
};
|
||||
} catch (err) {
|
||||
throw this.#wrap(err);
|
||||
}
|
||||
}
|
||||
|
||||
// Document ops (add/delete/export) are GBrain-only and secondary; they match
|
||||
// gbrain's documented CLI surface (`put <slug>` reads stdin; `delete <slug>`;
|
||||
// `export`) but could not be exercised against a live engine on the test host
|
||||
// (pglite WASM broken, garrytan/gbrain#223), so treat them as best-effort.
|
||||
async add(doc: { slug: string; body: string }, opts: OpOptions = {}): Promise<SourceStatus> {
|
||||
assertCapability(this, "add");
|
||||
assertEgressConsent(this, opts);
|
||||
this.#receipt("document-body (sent by gbrain subprocess)", opts, doc.body);
|
||||
// `gbrain put <slug>` reads the document body from stdin.
|
||||
this.#assertOk(this.#runInput(["put", doc.slug], doc.body, opts));
|
||||
return { id: doc.slug, state: "ready" };
|
||||
}
|
||||
|
||||
async delete(slug: string, opts: OpOptions = {}): Promise<SourceStatus> {
|
||||
assertCapability(this, "delete");
|
||||
// stdin closed ("") so any confirmation prompt gets EOF rather than hanging.
|
||||
this.#assertOk(this.#runInput(["delete", slug], "", opts));
|
||||
return { id: slug, state: "absent" };
|
||||
}
|
||||
|
||||
async export(_source: SourceRef, opts: OpOptions = {}): Promise<string> {
|
||||
assertCapability(this, "export");
|
||||
// `gbrain export` is brain-wide (no per-source flag); returns whatever it prints.
|
||||
const r = spawnGbrain(["export"], {
|
||||
baseEnv: opts.env,
|
||||
timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS,
|
||||
});
|
||||
this.#assertOk(r);
|
||||
return r.stdout || "";
|
||||
}
|
||||
|
||||
/** spawn gbrain with `input` on stdin, seeded env, Windows-shim aware. */
|
||||
#runInput(args: string[], input: string, opts: OpOptions) {
|
||||
return spawnSync("gbrain", args, {
|
||||
input,
|
||||
encoding: "utf-8",
|
||||
timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS,
|
||||
env: buildGbrainEnv({ baseEnv: opts.env }),
|
||||
shell: NEEDS_SHELL_ON_WINDOWS,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Throw a typed failure unless the spawn succeeded. Distinguishes a missing
|
||||
* CLI (ENOENT → PROVIDER_UNAVAILABLE, the degrade signal) from a timeout
|
||||
* (ETIMEDOUT/SIGTERM, status=null) and a real non-zero exit.
|
||||
*/
|
||||
#assertOk(r: {
|
||||
status: number | null;
|
||||
stderr?: string;
|
||||
error?: Error & { code?: string };
|
||||
signal?: NodeJS.Signals | null;
|
||||
}): void {
|
||||
if (r.status === 0) return;
|
||||
const stderr = (r.stderr || "").trim();
|
||||
if (r.error?.code === "ENOENT" || /command not found/.test(stderr)) {
|
||||
throw new CodeProviderError("PROVIDER_UNAVAILABLE", "gbrain CLI not on PATH", this.id);
|
||||
}
|
||||
if (r.error?.code === "ETIMEDOUT" || r.signal === "SIGTERM") {
|
||||
throw new CodeProviderError("PROVIDER_TIMEOUT", "gbrain timed out", this.id);
|
||||
}
|
||||
// Engine / DB / config problems are ENVIRONMENTAL — degrade to UNAVAILABLE
|
||||
// (caller falls back to file-only), not a hard PROVIDER_ERROR with a raw dump.
|
||||
// Covers the real case where gbrain's pglite engine fails to init its WASM
|
||||
// runtime (garrytan/gbrain#223) as well as unreachable/unconfigured databases.
|
||||
if (/PGLite|WASM|failed to initialize|Aborted|Cannot connect to database|not configured|config\.json|database (is )?un(reachable|available)/i.test(stderr)) {
|
||||
throw new CodeProviderError("PROVIDER_UNAVAILABLE", firstLine(stderr) || "gbrain engine unavailable", this.id);
|
||||
}
|
||||
throw new CodeProviderError("PROVIDER_ERROR", firstLine(stderr) || `gbrain exited ${r.status}`, this.id);
|
||||
}
|
||||
|
||||
#wrap(err: unknown): CodeProviderError {
|
||||
if (err instanceof CodeProviderError) return err;
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
// Same environmental-vs-real split as #assertOk: missing CLI, or engine/DB/
|
||||
// config problems, degrade to UNAVAILABLE so callers fall back to file-only.
|
||||
if (/not on PATH|command not found|PGLite|WASM|failed to initialize|Aborted|Cannot connect to database|not configured|config\.json/i.test(message)) {
|
||||
return new CodeProviderError("PROVIDER_UNAVAILABLE", firstLine(message), this.id);
|
||||
}
|
||||
return new CodeProviderError("PROVIDER_ERROR", firstLine(message), this.id);
|
||||
}
|
||||
}
|
||||
|
||||
/** First non-empty line, so a multi-line WASM/stack dump never reaches the user. */
|
||||
function firstLine(text: string): string {
|
||||
return (text || "").split("\n").map((l) => l.trim()).find(Boolean) ?? "";
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
/**
|
||||
* Graphify adapter — real CLI integration (github.com/Graphify-Labs/graphify).
|
||||
*
|
||||
* Graphify is a LOCAL tree-sitter knowledge graph. For CODE, `graphify <dir>`
|
||||
* and `graphify update <dir>` produce the SAME AST graph with NO LLM and NO
|
||||
* network (verified against graphify 0.9.23 — both emit `AST extraction on N
|
||||
* code files`, all node origins `ast`). The LLM backend (openai/gemini) is only
|
||||
* used to RENAME community clusters (`graphify label` / `cluster-only`) and to
|
||||
* ingest non-code docs (`graphify add`); it adds zero nodes/edges, and our parser
|
||||
* discards the `community=` field it touches — so an LLM mode would send code
|
||||
* off-machine for no change in search output, and is intentionally not offered.
|
||||
*
|
||||
* This adapter uses `graphify update <dir>` (writes `<dir>/graphify-out/graph.json`
|
||||
* and does clustering in one shot) and stays fully local — nothing leaves the
|
||||
* machine, so `local = true` and no egress consent is needed.
|
||||
*
|
||||
* Query is `graphify query "<q>" --graph <dir>/graphify-out/graph.json`; the
|
||||
* `--graph` flag points at the built graph so search never depends on cwd.
|
||||
*
|
||||
* Never auto-installed: install is `pip install graphifyy && graphify install`
|
||||
* (needs Python >= 3.10), a user action the picker surfaces. When the CLI is
|
||||
* absent every op throws PROVIDER_UNAVAILABLE and callers degrade to file-only.
|
||||
*
|
||||
* Path-based, not id-based: for Graphify a source "id" IS the absolute repo path
|
||||
* (that is where `graphify-out/` lives), unlike GBrain's short source ids.
|
||||
*/
|
||||
|
||||
import { spawnSync } from "child_process";
|
||||
import { existsSync, readFileSync } from "fs";
|
||||
import { join } from "path";
|
||||
import {
|
||||
assertCapability,
|
||||
assertRequiredCapabilities,
|
||||
CodeProviderError,
|
||||
type CodeProvider,
|
||||
type CodeProviderCapability,
|
||||
type CodeSearchHit,
|
||||
type OpOptions,
|
||||
type RepoRef,
|
||||
type SearchOptions,
|
||||
type SourceRef,
|
||||
type SourceStatus,
|
||||
} from "./contract";
|
||||
|
||||
const CAPABILITIES: CodeProviderCapability[] = ["register_source", "refresh", "search", "status", "export"];
|
||||
const OUT_DIR = "graphify-out";
|
||||
const GRAPH_JSON = "graph.json";
|
||||
const DEFAULT_TIMEOUT_MS = 120_000; // indexing a repo can take a while
|
||||
const NEEDS_SHELL_ON_WINDOWS = process.platform === "win32"; // graphify is a shim on Windows
|
||||
|
||||
export interface GraphifyOptions {
|
||||
/** Directory whose `graphify-out/` search/status/export read. Defaults to cwd. */
|
||||
root?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}
|
||||
|
||||
export class GraphifyProvider implements CodeProvider {
|
||||
readonly id = "graphify" as const;
|
||||
readonly label = "Graphify";
|
||||
readonly capabilities = new Set<CodeProviderCapability>(CAPABILITIES);
|
||||
/** Fully local — no repo content leaves the machine. */
|
||||
readonly local = true;
|
||||
readonly #root: string;
|
||||
readonly #env?: NodeJS.ProcessEnv;
|
||||
|
||||
constructor(opts: GraphifyOptions = {}) {
|
||||
this.#root = opts.root ?? process.cwd();
|
||||
this.#env = opts.env;
|
||||
assertRequiredCapabilities(this.id, this.capabilities);
|
||||
}
|
||||
|
||||
has(capability: CodeProviderCapability): boolean {
|
||||
return this.capabilities.has(capability);
|
||||
}
|
||||
|
||||
#run(args: string[], cwd: string, timeout: number) {
|
||||
return spawnSync("graphify", args, {
|
||||
cwd,
|
||||
encoding: "utf-8",
|
||||
timeout,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: this.#env,
|
||||
shell: NEEDS_SHELL_ON_WINDOWS,
|
||||
});
|
||||
}
|
||||
|
||||
#assertOk(r: { status: number | null; stderr?: string; error?: Error & { code?: string }; signal?: NodeJS.Signals | null }): void {
|
||||
if (r.status === 0) return;
|
||||
const stderr = (r.stderr || "").trim();
|
||||
if (r.error?.code === "ENOENT" || /command not found/.test(stderr)) {
|
||||
throw new CodeProviderError("PROVIDER_UNAVAILABLE", "graphify CLI not on PATH (install: pip install graphifyy && graphify install)", this.id);
|
||||
}
|
||||
if (r.error?.code === "ETIMEDOUT" || r.signal === "SIGTERM") {
|
||||
throw new CodeProviderError("PROVIDER_TIMEOUT", "graphify timed out", this.id);
|
||||
}
|
||||
throw new CodeProviderError("PROVIDER_ERROR", stderr || `graphify exited ${r.status}`, this.id);
|
||||
}
|
||||
|
||||
/** Build the graph over repo.path locally (no LLM, no egress consent needed). */
|
||||
async registerSource(repo: RepoRef, opts: OpOptions = {}): Promise<SourceStatus> {
|
||||
this.#assertOk(this.#run(["update", repo.path], repo.path, opts.timeout ?? DEFAULT_TIMEOUT_MS));
|
||||
return this.status({ id: repo.path }, opts);
|
||||
}
|
||||
|
||||
/** Re-parse and rebuild the graph (same local `graphify update` path). */
|
||||
async refresh(source: SourceRef, opts: OpOptions = {}): Promise<SourceStatus> {
|
||||
this.#assertOk(this.#run(["update", source.id], source.id, opts.timeout ?? DEFAULT_TIMEOUT_MS));
|
||||
return this.status(source, opts);
|
||||
}
|
||||
|
||||
/**
|
||||
* `graphify query "<q>" --graph <graph.json>` traces the graph and prints
|
||||
* `NODE ...` / `EDGE ...` lines (plus a `Traversal:` header). We pass `--graph`
|
||||
* explicitly so the query reads the indexed repo's graph regardless of cwd.
|
||||
*/
|
||||
async search(query: string, opts: SearchOptions = {}): Promise<CodeSearchHit[]> {
|
||||
if (!query.trim()) return [];
|
||||
const root = opts.source ?? this.#root;
|
||||
const graphPath = join(root, OUT_DIR, GRAPH_JSON);
|
||||
const r = this.#run(["query", query, "--graph", graphPath], root, opts.timeout ?? DEFAULT_TIMEOUT_MS);
|
||||
this.#assertOk(r);
|
||||
return parseGraphifyQuery(r.stdout || "", opts.limit ?? 10);
|
||||
}
|
||||
|
||||
async status(source?: SourceRef, _opts: OpOptions = {}): Promise<SourceStatus> {
|
||||
const dir = source?.id ?? this.#root;
|
||||
const graphPath = join(dir, OUT_DIR, GRAPH_JSON);
|
||||
if (!existsSync(graphPath)) return { id: dir, state: "absent" };
|
||||
let itemCount: number | undefined;
|
||||
try {
|
||||
const graph = JSON.parse(readFileSync(graphPath, "utf-8")) as { nodes?: unknown[] };
|
||||
if (Array.isArray(graph.nodes)) itemCount = graph.nodes.length;
|
||||
} catch {
|
||||
// graph.json present but unparseable — still ready, just no count.
|
||||
}
|
||||
return { id: dir, state: "ready", itemCount, detail: graphPath };
|
||||
}
|
||||
|
||||
async export(source: SourceRef, _opts: OpOptions = {}): Promise<string> {
|
||||
assertCapability(this, "export");
|
||||
const graphPath = join(source.id, OUT_DIR, GRAPH_JSON);
|
||||
if (!existsSync(graphPath)) {
|
||||
throw new CodeProviderError("SOURCE_NOT_REGISTERED", `no graph at ${graphPath}; index it first`, this.id);
|
||||
}
|
||||
return readFileSync(graphPath, "utf-8");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse real `graphify query` output into hits. The format (graphify 0.9.23):
|
||||
* Traversal: BFS depth=2 | Start: ['query()'] | ... | 4 nodes found
|
||||
* NODE query() [src=db.py loc=L4 community=login]
|
||||
* EDGE query() --calls [EXTRACTED context=call]--> login() at=auth.py:L8
|
||||
* The file lives mid-line (`src=<file> loc=L<n>` on NODE, `at=<file>:L<n>` on
|
||||
* EDGE), so the ref is `<file>:L<n>`. The `Traversal:` header and any other line
|
||||
* are skipped. Exported for deterministic unit testing against the real format.
|
||||
*/
|
||||
export function parseGraphifyQuery(stdout: string, limit: number): CodeSearchHit[] {
|
||||
const hits: CodeSearchHit[] = [];
|
||||
for (const raw of stdout.split("\n")) {
|
||||
const line = raw.trim();
|
||||
let ref: string | undefined;
|
||||
const node = line.match(/^NODE\b.*?\[src=(\S+)\s+loc=(L\d+)/);
|
||||
const edge = line.match(/^EDGE\b.*?\bat=(\S+?):(L\d+)\b/);
|
||||
if (node) ref = `${node[1]}:${node[2]}`;
|
||||
else if (edge) ref = `${edge[1]}:${edge[2]}`;
|
||||
else continue; // skip the Traversal header and anything non-NODE/EDGE
|
||||
hits.push({ ref, snippet: line, kind: "graph-node" });
|
||||
if (hits.length >= limit) break;
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
/** Whether the `graphify` CLI is installed (for the picker's availability probe). */
|
||||
export function graphifyInstalled(env?: NodeJS.ProcessEnv): boolean {
|
||||
const r = spawnSync("graphify", ["--version"], {
|
||||
encoding: "utf-8",
|
||||
timeout: 5_000,
|
||||
stdio: ["ignore", "ignore", "ignore"],
|
||||
env,
|
||||
shell: NEEDS_SHELL_ON_WINDOWS,
|
||||
});
|
||||
return r.status === 0;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
/**
|
||||
* code-intelligence — the OPTIONAL, repo-oriented provider contract.
|
||||
* See docs/designs/CODE_INTELLIGENCE_PROVIDER_CONTRACT.md.
|
||||
*/
|
||||
|
||||
export * from "./contract";
|
||||
export { GbrainProvider, parseGbrainSearch } from "./gbrain-adapter";
|
||||
export { GraphifyProvider, parseGraphifyQuery, type GraphifyOptions } from "./graphify-adapter";
|
||||
export { SourcebotProvider, parseSourcebotSearch, type SourcebotOptions } from "./sourcebot-adapter";
|
||||
export {
|
||||
readSelection,
|
||||
setProvider,
|
||||
setConsent,
|
||||
hasConsent,
|
||||
setRoot,
|
||||
getRoot,
|
||||
type Selection,
|
||||
} from "./selection";
|
||||
export {
|
||||
LARGE_REPO_FILE_THRESHOLD,
|
||||
shouldOfferIndexing,
|
||||
trackedFileCount,
|
||||
type Suggestion,
|
||||
type SuggestReason,
|
||||
} from "./suggest";
|
||||
export {
|
||||
RECOMMENDED_ORDER,
|
||||
providerById,
|
||||
resolveSelectedProvider,
|
||||
detectAvailable,
|
||||
type PickerOptions,
|
||||
type Availability,
|
||||
} from "./picker";
|
||||
@@ -0,0 +1,100 @@
|
||||
/**
|
||||
* Picker — constructs the code-intelligence provider the user selected, and
|
||||
* offers the recommendation order (GBrain first) for the selection UX.
|
||||
*
|
||||
* `resolveSelectedProvider()` reads the persisted selection and constructs that
|
||||
* provider, or returns null when nothing is selected — the provider-OFF path,
|
||||
* where callers degrade to grep / the file-only decision store. Availability is
|
||||
* proven at call time: a selected provider whose tool/server is absent throws
|
||||
* PROVIDER_UNAVAILABLE from its ops, which callers catch and degrade on. The
|
||||
* `detectAvailable()` probe drives the `options`/`status` display.
|
||||
*
|
||||
* GBrain is recommended first. Graphify is NEVER auto-installed — it appears in
|
||||
* the options only once its CLI is present (a user install).
|
||||
*/
|
||||
|
||||
import { localEngineStatus } from "../gbrain-local-status";
|
||||
import { GbrainProvider } from "./gbrain-adapter";
|
||||
import { GraphifyProvider, graphifyInstalled, type GraphifyOptions } from "./graphify-adapter";
|
||||
import { SourcebotProvider, type SourcebotOptions } from "./sourcebot-adapter";
|
||||
import { readSelection, getRoot } from "./selection";
|
||||
import type { CodeProvider, CodeProviderId } from "./contract";
|
||||
|
||||
/** Recommendation order — GBrain first. */
|
||||
export const RECOMMENDED_ORDER: readonly CodeProviderId[] = ["gbrain", "sourcebot", "graphify"];
|
||||
|
||||
export interface PickerOptions {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
graphify?: GraphifyOptions;
|
||||
sourcebot?: SourcebotOptions;
|
||||
}
|
||||
|
||||
/** Construct a provider by id (no availability check — ops degrade at call time). */
|
||||
export function providerById(id: CodeProviderId, opts: PickerOptions = {}): CodeProvider {
|
||||
switch (id) {
|
||||
case "gbrain":
|
||||
return new GbrainProvider();
|
||||
case "graphify": {
|
||||
// Default the graph root to the repo Graphify last indexed, so `search`
|
||||
// reads the same graph `index` built (not whatever cwd happens to be).
|
||||
const root = opts.graphify?.root ?? getRoot("graphify", opts.env);
|
||||
return new GraphifyProvider({ env: opts.env, ...opts.graphify, ...(root ? { root } : {}) });
|
||||
}
|
||||
case "sourcebot":
|
||||
return new SourcebotProvider({ env: opts.env, ...opts.sourcebot });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The provider the user selected, constructed, or null when none is selected.
|
||||
* Null is the provider-OFF path: callers MUST degrade to grep / file-only.
|
||||
*/
|
||||
export function resolveSelectedProvider(opts: PickerOptions = {}): CodeProvider | null {
|
||||
const { provider } = readSelection(opts.env);
|
||||
return provider ? providerById(provider, opts) : null;
|
||||
}
|
||||
|
||||
export interface Availability {
|
||||
id: CodeProviderId;
|
||||
available: boolean;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe which providers are usable right now, in recommendation order. Used by
|
||||
* the `options`/`status` display. GBrain via the real localEngineStatus();
|
||||
* Graphify via its CLI status; Sourcebot via an HTTP liveness probe.
|
||||
*/
|
||||
export async function detectAvailable(opts: PickerOptions = {}): Promise<Availability[]> {
|
||||
const gbrainStatus = localEngineStatus({ env: opts.env });
|
||||
const gbrainOk = gbrainStatus === "ok" || gbrainStatus === "timeout";
|
||||
|
||||
// Available = the CLI is installed and selectable (NOT "a graph already exists
|
||||
// here"). A freshly installed Graphify with no graph yet is still available.
|
||||
const graphifyOk = graphifyInstalled(opts.env);
|
||||
let graphifyDetail = "graphify CLI not installed (pip install graphifyy, Python >= 3.10)";
|
||||
if (graphifyOk) {
|
||||
try {
|
||||
const s = await new GraphifyProvider({ env: opts.env, ...opts.graphify }).status();
|
||||
graphifyDetail = s.state === "ready" ? "installed; graph built in this repo" : "installed; run `index` to build a graph";
|
||||
} catch {
|
||||
graphifyDetail = "installed";
|
||||
}
|
||||
}
|
||||
|
||||
let sourcebotOk = false;
|
||||
let sourcebotDetail = "server unreachable";
|
||||
try {
|
||||
const s = await new SourcebotProvider({ env: opts.env, ...opts.sourcebot }).status();
|
||||
sourcebotOk = s.state === "ready";
|
||||
sourcebotDetail = s.detail ?? "";
|
||||
} catch {
|
||||
sourcebotOk = false;
|
||||
}
|
||||
|
||||
return [
|
||||
{ id: "gbrain", available: gbrainOk, detail: `gbrain engine: ${gbrainStatus}` },
|
||||
{ id: "sourcebot", available: sourcebotOk, detail: sourcebotDetail },
|
||||
{ id: "graphify", available: graphifyOk, detail: graphifyDetail },
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* selection — persists the user's chosen code-intelligence provider and their
|
||||
* per-repo indexing consent. Stored at `$GSTACK_HOME/code-intelligence.json`
|
||||
* (default `~/.gstack/`), the same home the rest of gstack uses.
|
||||
*
|
||||
* Consent is per-repo (keyed by absolute repo path), because indexing consent
|
||||
* is "may THIS repo's content be indexed by the selected provider" — a decision
|
||||
* a user makes per project, not once for the machine. No selection at all is the
|
||||
* provider-OFF default: callers degrade to grep / the file-only decision store.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "fs";
|
||||
import { homedir } from "os";
|
||||
import { dirname, join, resolve } from "path";
|
||||
import { execFileSync, spawnSync } from "child_process";
|
||||
import type { CodeProviderId } from "./contract";
|
||||
|
||||
export interface Selection {
|
||||
provider: CodeProviderId | null;
|
||||
/** Absolute repo path → consented. */
|
||||
consents: Record<string, boolean>;
|
||||
/** Provider id → the absolute repo path it last indexed (so search finds it). */
|
||||
roots: Record<string, string>;
|
||||
/** User explicitly chose no indexing — never offer again. */
|
||||
declined: boolean;
|
||||
}
|
||||
|
||||
const EMPTY: Selection = { provider: null, consents: {}, roots: {}, declined: false };
|
||||
|
||||
function storePath(env: NodeJS.ProcessEnv = process.env): string {
|
||||
const home = env.GSTACK_HOME || join(env.HOME || homedir(), ".gstack");
|
||||
return join(home, "code-intelligence.json");
|
||||
}
|
||||
|
||||
export function readSelection(env: NodeJS.ProcessEnv = process.env): Selection {
|
||||
const p = storePath(env);
|
||||
if (!existsSync(p)) return { ...EMPTY };
|
||||
try {
|
||||
const raw = JSON.parse(readFileSync(p, "utf-8")) as Partial<Selection>;
|
||||
return {
|
||||
provider: raw.provider ?? null,
|
||||
consents: raw.consents && typeof raw.consents === "object" ? raw.consents : {},
|
||||
roots: raw.roots && typeof raw.roots === "object" ? raw.roots : {},
|
||||
declined: raw.declined === true,
|
||||
};
|
||||
} catch {
|
||||
return { ...EMPTY };
|
||||
}
|
||||
}
|
||||
|
||||
function write(selection: Selection, env: NodeJS.ProcessEnv = process.env): void {
|
||||
const p = storePath(env);
|
||||
mkdirSync(dirname(p), { recursive: true });
|
||||
const tmp = `${p}.tmp.${process.pid}`;
|
||||
writeFileSync(tmp, JSON.stringify(selection, null, 2), "utf-8");
|
||||
renameSync(tmp, p);
|
||||
}
|
||||
|
||||
export function setProvider(provider: CodeProviderId | null, env: NodeJS.ProcessEnv = process.env): Selection {
|
||||
// Choosing a provider clears a prior decline; clearing to null records one,
|
||||
// so the session-start offer is never repeated after an explicit "none".
|
||||
const next = { ...readSelection(env), provider, declined: provider === null };
|
||||
write(next, env);
|
||||
return next;
|
||||
}
|
||||
|
||||
/** Record per-repo indexing consent (repo path resolved to absolute). */
|
||||
export function setConsent(repoPath: string, consented: boolean, env: NodeJS.ProcessEnv = process.env): Selection {
|
||||
const current = readSelection(env);
|
||||
const next: Selection = { ...current, consents: { ...current.consents, [resolve(repoPath)]: consented } };
|
||||
write(next, env);
|
||||
return next;
|
||||
}
|
||||
|
||||
/**
|
||||
* The per-remote trust store (gstack-gbrain-repo-policy) is the SINGLE
|
||||
* authority for consent-to-send: a `deny` tier vetoes any recorded
|
||||
* code-intelligence consent, so two stores can never disagree about whether
|
||||
* code may leave this repo (R1, fork port wave 2 review). Mirrors the
|
||||
* gbrain-sync chokepoint's polarity: no policy store → no veto (nothing was
|
||||
* ever set); unreadable store → veto (fail-closed — a policy the user set
|
||||
* must not be bypassed by a broken store).
|
||||
*/
|
||||
function repoPolicyVeto(repoPath: string, env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
const home = env.GSTACK_HOME || join(homedir(), ".gstack");
|
||||
if (!existsSync(join(home, "gbrain-repo-policy.json"))) return false;
|
||||
let url = "";
|
||||
try {
|
||||
url = execFileSync("git", ["-C", resolve(repoPath), "remote", "get-url", "origin"], {
|
||||
encoding: "utf-8", timeout: 5000,
|
||||
}).trim();
|
||||
} catch {
|
||||
return false; // no remote → policy (keyed by remote) has nothing set for this repo
|
||||
}
|
||||
if (!url) return false;
|
||||
const res = spawnSync(join(import.meta.dir, "..", "..", "bin", "gstack-gbrain-repo-policy"), ["get", url], {
|
||||
encoding: "utf-8", timeout: 10_000, env: { ...env } as NodeJS.ProcessEnv,
|
||||
});
|
||||
if (res.error || res.status !== 0) return true; // fail-closed
|
||||
const tier = (res.stdout || "").trim();
|
||||
return tier === "deny";
|
||||
}
|
||||
|
||||
export function hasConsent(repoPath: string, env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
if (readSelection(env).consents[resolve(repoPath)] !== true) return false;
|
||||
return !repoPolicyVeto(repoPath, env);
|
||||
}
|
||||
|
||||
/** Record the repo path a provider last indexed, so search reads the same graph. */
|
||||
export function setRoot(provider: CodeProviderId, repoPath: string, env: NodeJS.ProcessEnv = process.env): Selection {
|
||||
const current = readSelection(env);
|
||||
const next: Selection = { ...current, roots: { ...current.roots, [provider]: resolve(repoPath) } };
|
||||
write(next, env);
|
||||
return next;
|
||||
}
|
||||
|
||||
export function getRoot(provider: CodeProviderId, env: NodeJS.ProcessEnv = process.env): string | undefined {
|
||||
return readSelection(env).roots[provider];
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* Sourcebot adapter — real HTTP + config integration
|
||||
* (github.com/sourcebot-dev/sourcebot, YC F2025).
|
||||
*
|
||||
* Sourcebot is a self-hosted server that indexes repos declared in its
|
||||
* config.json and serves regex code search over `POST /api/search` (zoekt). So
|
||||
* the runtime drives it with plain HTTP + a config-file edit — no MCP:
|
||||
* - register_source: add `{ "type": "git", "url": "file:///abs/path" }` to the
|
||||
* server's config.json (it re-indexes automatically on config change).
|
||||
* - refresh: Sourcebot re-indexes on config change and on reindexIntervalMs;
|
||||
* there is no per-source trigger endpoint, so refresh reports current status.
|
||||
* - search: POST /api/search with a regex query, map files[] to hits.
|
||||
* - status: liveness GET against the base URL.
|
||||
* Declines the document ops (add/delete/export) — it is a whole-repo search index.
|
||||
*
|
||||
* Egress: a loopback base URL means the index runs on this machine, so no repo
|
||||
* content leaves it (local=true). A non-loopback base URL means content reaches
|
||||
* another host, so egress consent is required (local=false).
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync, writeFileSync, renameSync } from "fs";
|
||||
import { sha256Hex, writeReceipt } from "../egress-receipt.js";
|
||||
import {
|
||||
assertCapability,
|
||||
assertEgressConsent,
|
||||
assertRequiredCapabilities,
|
||||
CodeProviderError,
|
||||
type CodeProvider,
|
||||
type CodeProviderCapability,
|
||||
type CodeSearchHit,
|
||||
type OpOptions,
|
||||
type RepoRef,
|
||||
type SearchOptions,
|
||||
type SourceRef,
|
||||
type SourceStatus,
|
||||
} from "./contract";
|
||||
|
||||
const CAPABILITIES: CodeProviderCapability[] = ["register_source", "refresh", "search", "status"];
|
||||
const DEFAULT_URL = "http://localhost:3000";
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
|
||||
type FetchLike = typeof globalThis.fetch;
|
||||
|
||||
export interface SourcebotOptions {
|
||||
/** Base URL of the Sourcebot server. Defaults to SOURCEBOT_URL or http://localhost:3000. */
|
||||
baseUrl?: string;
|
||||
/** Path to the server's config.json (for register_source). Defaults to SOURCEBOT_CONFIG. */
|
||||
configPath?: string;
|
||||
/**
|
||||
* OPTIONAL API key for the Sourcebot REST API. Defaults to SOURCEBOT_API_KEY.
|
||||
* A local instance with anonymous access enabled
|
||||
* (`FORCE_ENABLE_ANONYMOUS_ACCESS=true`) serves `/api/search` with NO key —
|
||||
* verified keyless against a real Sourcebot v6.5.0. Only set a key when your
|
||||
* instance is login-gated; it is sent as `Authorization: Bearer <key>`.
|
||||
*/
|
||||
apiKey?: string;
|
||||
/** Injectable fetch for tests. */
|
||||
fetch?: FetchLike;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}
|
||||
|
||||
function isLoopback(url: string): boolean {
|
||||
try {
|
||||
const h = new URL(url).hostname.toLowerCase();
|
||||
return h === "localhost" || h === "127.0.0.1" || h === "::1" || h.endsWith(".localhost");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export class SourcebotProvider implements CodeProvider {
|
||||
readonly id = "sourcebot" as const;
|
||||
readonly label = "Sourcebot";
|
||||
readonly capabilities = new Set<CodeProviderCapability>(CAPABILITIES);
|
||||
readonly local: boolean;
|
||||
readonly #baseUrl: string;
|
||||
readonly #configPath?: string;
|
||||
readonly #apiKey?: string;
|
||||
readonly #fetch: FetchLike;
|
||||
|
||||
constructor(opts: SourcebotOptions = {}) {
|
||||
const env = opts.env ?? process.env;
|
||||
this.#baseUrl = (opts.baseUrl ?? env.SOURCEBOT_URL ?? DEFAULT_URL).replace(/\/$/, "");
|
||||
this.#configPath = opts.configPath ?? env.SOURCEBOT_CONFIG;
|
||||
this.#apiKey = opts.apiKey ?? env.SOURCEBOT_API_KEY;
|
||||
this.#fetch = opts.fetch ?? globalThis.fetch;
|
||||
this.local = isLoopback(this.#baseUrl);
|
||||
assertRequiredCapabilities(this.id, this.capabilities);
|
||||
}
|
||||
|
||||
#authHeaders(): Record<string, string> {
|
||||
return this.#apiKey ? { Authorization: `Bearer ${this.#apiKey}` } : {};
|
||||
}
|
||||
|
||||
has(capability: CodeProviderCapability): boolean {
|
||||
return this.capabilities.has(capability);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the repo as a local `git` connection in Sourcebot's config.json.
|
||||
* NOTE: Sourcebot silently SKIPS a local repo that has no `remote.origin.url`
|
||||
* (logs "Skipping <path> - remote.origin.url not found"); a freshly `git init`'d
|
||||
* repo must set an origin before it will index.
|
||||
*/
|
||||
async registerSource(repo: RepoRef, opts: OpOptions = {}): Promise<SourceStatus> {
|
||||
assertEgressConsent(this, opts); // no-op when the server is loopback (local)
|
||||
if (!this.#configPath) {
|
||||
throw new CodeProviderError(
|
||||
"PROVIDER_UNAVAILABLE",
|
||||
"set SOURCEBOT_CONFIG to the server's config.json path to register sources",
|
||||
this.id,
|
||||
);
|
||||
}
|
||||
let config: { connections?: Record<string, unknown> };
|
||||
try {
|
||||
config = existsSync(this.#configPath)
|
||||
? (JSON.parse(readFileSync(this.#configPath, "utf-8")) as typeof config)
|
||||
: {};
|
||||
} catch (err) {
|
||||
throw new CodeProviderError("PROVIDER_ERROR", `unreadable Sourcebot config: ${(err as Error).message}`, this.id);
|
||||
}
|
||||
config.connections = config.connections ?? {};
|
||||
config.connections[repo.id] = { type: "git", url: `file://${repo.path}` };
|
||||
// Atomic write so a running Sourcebot never reads a half-written config.
|
||||
const tmp = `${this.#configPath}.tmp.${process.pid}`;
|
||||
writeFileSync(tmp, JSON.stringify(config, null, 2), "utf-8");
|
||||
renameSync(tmp, this.#configPath);
|
||||
return { id: repo.id, state: "registered", detail: "Sourcebot re-indexes on config change" };
|
||||
}
|
||||
|
||||
/** Sourcebot re-indexes automatically; report current liveness. */
|
||||
async refresh(source: SourceRef, opts: OpOptions = {}): Promise<SourceStatus> {
|
||||
const live = await this.status(source, opts);
|
||||
return { ...live, detail: "Sourcebot re-indexes automatically (config change + reindexIntervalMs)" };
|
||||
}
|
||||
|
||||
async search(query: string, opts: SearchOptions = {}): Promise<CodeSearchHit[]> {
|
||||
if (!query.trim()) return [];
|
||||
const body = {
|
||||
query: opts.source ? `repo:${opts.source} ${query}` : query,
|
||||
matches: opts.limit ?? 20,
|
||||
isRegexEnabled: true,
|
||||
isCaseSensitivityEnabled: false,
|
||||
};
|
||||
const payload = await this.#post("/api/search", body, opts.timeout ?? DEFAULT_TIMEOUT_MS);
|
||||
return parseSourcebotSearch(payload, opts.limit ?? 20);
|
||||
}
|
||||
|
||||
async status(_source?: SourceRef, opts: OpOptions = {}): Promise<SourceStatus> {
|
||||
// `redirect: manual` so an auth-gated server (307 -> /login) reads as
|
||||
// not-usable instead of following to a 200 and falsely reporting "ready".
|
||||
try {
|
||||
const res = await this.#fetchWithTimeout(
|
||||
`${this.#baseUrl}/api/search`,
|
||||
{ method: "POST", headers: { "Content-Type": "application/json", ...this.#authHeaders() }, body: JSON.stringify({ query: "sourcebot", matches: 1, isRegexEnabled: false }), redirect: "manual" },
|
||||
opts.timeout ?? DEFAULT_TIMEOUT_MS,
|
||||
);
|
||||
if (res.status === 401 || res.status === 403) {
|
||||
return { id: "*", state: "unknown", partial: true, detail: "reachable but login-gated (enable anonymous access with FORCE_ENABLE_ANONYMOUS_ACCESS=true for local use, or set SOURCEBOT_API_KEY)" };
|
||||
}
|
||||
return { id: "*", state: res.ok ? "ready" : "unknown", partial: true, detail: `HTTP ${res.status}` };
|
||||
} catch {
|
||||
return { id: "*", state: "unknown", partial: true, detail: `unreachable at ${this.#baseUrl}` };
|
||||
}
|
||||
}
|
||||
|
||||
async #post(path: string, body: unknown, timeout: number): Promise<unknown> {
|
||||
let res: Response;
|
||||
try {
|
||||
res = await this.#fetchWithTimeout(`${this.#baseUrl}${path}`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Accept: "application/json", ...this.#authHeaders() },
|
||||
body: JSON.stringify(body),
|
||||
}, timeout);
|
||||
} catch (err) {
|
||||
throw new CodeProviderError("PROVIDER_UNAVAILABLE", `Sourcebot unreachable at ${this.#baseUrl}: ${(err as Error).message}`, this.id);
|
||||
}
|
||||
if (res.status === 401 || res.status === 403) {
|
||||
throw new CodeProviderError("PROVIDER_UNAVAILABLE", `Sourcebot is login-gated (HTTP ${res.status}); enable anonymous access (FORCE_ENABLE_ANONYMOUS_ACCESS=true) for local use, or set SOURCEBOT_API_KEY`, this.id);
|
||||
}
|
||||
if (!res.ok) throw new CodeProviderError("PROVIDER_ERROR", `Sourcebot ${path} returned HTTP ${res.status}`, this.id);
|
||||
try {
|
||||
return await res.json();
|
||||
} catch (err) {
|
||||
throw new CodeProviderError("PROVIDER_ERROR", `Sourcebot returned non-JSON: ${(err as Error).message}`, this.id);
|
||||
}
|
||||
}
|
||||
|
||||
async #fetchWithTimeout(url: string, init: RequestInit, timeout: number): Promise<Response> {
|
||||
// Every Sourcebot HTTP call routes through here. A loopback server keeps
|
||||
// content on this machine (no egress, no receipt); a non-loopback server
|
||||
// is an off-machine send and gets a fail-closed receipt BEFORE the fetch.
|
||||
if (!this.local) {
|
||||
const body = typeof init.body === "string" ? init.body : "";
|
||||
writeReceipt({
|
||||
sink: "sourcebot",
|
||||
host: new URL(url).host,
|
||||
payloadClass: "code-search-request",
|
||||
bytes: Buffer.byteLength(body),
|
||||
sha256: sha256Hex(body),
|
||||
consent: "code-intelligence provider=sourcebot (non-loopback SOURCEBOT_URL) + per-repo consented=true",
|
||||
});
|
||||
}
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeout);
|
||||
try {
|
||||
return await this.#fetch(url, { ...init, signal: controller.signal });
|
||||
} catch (err) {
|
||||
if ((err as Error)?.name === "AbortError") throw new CodeProviderError("PROVIDER_TIMEOUT", "Sourcebot request timed out", this.id);
|
||||
throw err;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface SourcebotMatchRange { start?: { lineNumber?: number } }
|
||||
interface SourcebotChunk { content?: string; matchRanges?: SourcebotMatchRange[] }
|
||||
interface SourcebotFile { fileName?: { text?: string }; repository?: string; chunks?: SourcebotChunk[] }
|
||||
|
||||
/**
|
||||
* Map a `POST /api/search` response `{ files: [...] }` to hits: one hit per file,
|
||||
* ref = file path, snippet = first matching chunk, kind = "file". Tolerant of a
|
||||
* missing/garbage payload (returns []). Exported for deterministic testing.
|
||||
*/
|
||||
export function parseSourcebotSearch(payload: unknown, limit: number): CodeSearchHit[] {
|
||||
const files = (payload as { files?: unknown })?.files;
|
||||
if (!Array.isArray(files)) return [];
|
||||
const hits: CodeSearchHit[] = [];
|
||||
for (const f of files as SourcebotFile[]) {
|
||||
const ref = f?.fileName?.text;
|
||||
if (typeof ref !== "string") continue;
|
||||
const chunk = f.chunks?.[0];
|
||||
const line = chunk?.matchRanges?.[0]?.start?.lineNumber;
|
||||
hits.push({
|
||||
ref: typeof line === "number" ? `${ref}:${line}` : ref,
|
||||
snippet: typeof chunk?.content === "string" ? chunk.content.trim() : undefined,
|
||||
kind: "file",
|
||||
});
|
||||
if (hits.length >= limit) break;
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* suggest — should the session-start indexing offer be made for this repo?
|
||||
*
|
||||
* The offer fires at most once per machine: never when a provider is already
|
||||
* selected, never after an explicit decline (`select none`), and never for
|
||||
* small repos where grep is already fast. Detection is cheap and local
|
||||
* (`git ls-files` count); a non-repo directory never triggers the offer.
|
||||
*/
|
||||
|
||||
import { spawnSync } from "child_process";
|
||||
import { resolve } from "path";
|
||||
import { readSelection } from "./selection";
|
||||
|
||||
// ponytail: single tracked-file-count knob for "large"; add a LOC signal if it misfires
|
||||
/** Tracked-file count at which indexing starts paying for itself. */
|
||||
export const LARGE_REPO_FILE_THRESHOLD = 1000;
|
||||
|
||||
export type SuggestReason =
|
||||
| "provider-selected"
|
||||
| "declined"
|
||||
| "not-a-repo"
|
||||
| "small-repo"
|
||||
| "large-repo";
|
||||
|
||||
export interface Suggestion {
|
||||
offer: boolean;
|
||||
reason: SuggestReason;
|
||||
fileCount: number | null;
|
||||
threshold: number;
|
||||
}
|
||||
|
||||
/** Count of git-tracked files, or null when the path is not a git repo. */
|
||||
export function trackedFileCount(repoPath: string): number | null {
|
||||
const result = spawnSync("git", ["-C", resolve(repoPath), "ls-files"], {
|
||||
encoding: "utf-8",
|
||||
maxBuffer: 64 * 1024 * 1024,
|
||||
});
|
||||
if (result.status !== 0 || typeof result.stdout !== "string") return null;
|
||||
const out = result.stdout.trim();
|
||||
return out ? out.split("\n").length : 0;
|
||||
}
|
||||
|
||||
export function shouldOfferIndexing(
|
||||
repoPath: string,
|
||||
opts: { env?: NodeJS.ProcessEnv; threshold?: number } = {},
|
||||
): Suggestion {
|
||||
const threshold = opts.threshold ?? LARGE_REPO_FILE_THRESHOLD;
|
||||
const selection = readSelection(opts.env);
|
||||
if (selection.provider) return { offer: false, reason: "provider-selected", fileCount: null, threshold };
|
||||
if (selection.declined) return { offer: false, reason: "declined", fileCount: null, threshold };
|
||||
const fileCount = trackedFileCount(repoPath);
|
||||
if (fileCount === null) return { offer: false, reason: "not-a-repo", fileCount, threshold };
|
||||
if (fileCount < threshold) return { offer: false, reason: "small-repo", fileCount, threshold };
|
||||
return { offer: true, reason: "large-repo", fileCount, threshold };
|
||||
}
|
||||
Reference in New Issue
Block a user