mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-20 15:13:47 +01:00
Merge oct8-misc: docs links, benchmark accounting, Codex originator buckets (triage wave)
This commit is contained in:
commit
67d800618b
12 files changed
+267
-38
No files matched your search
@@ -17,9 +17,13 @@ import { canonicalRemote } from "../lib/remote-identity";
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
type CodexOriginator = "desktop" | "exec" | "claude_code" | "other";
|
||||
type CodexOriginatorCounts = Record<CodexOriginator, number>;
|
||||
|
||||
interface Session {
|
||||
tool: "claude_code" | "codex" | "gemini";
|
||||
cwd: string;
|
||||
codexOriginator?: CodexOriginator;
|
||||
}
|
||||
|
||||
interface Repo {
|
||||
@@ -27,6 +31,7 @@ interface Repo {
|
||||
remote: string;
|
||||
paths: string[];
|
||||
sessions: { claude_code: number; codex: number; gemini: number };
|
||||
codex_originators: CodexOriginatorCounts;
|
||||
}
|
||||
|
||||
interface DiscoveryResult {
|
||||
@@ -35,7 +40,7 @@ interface DiscoveryResult {
|
||||
repos: Repo[];
|
||||
tools: {
|
||||
claude_code: { total_sessions: number; repos: number };
|
||||
codex: { total_sessions: number; repos: number };
|
||||
codex: { total_sessions: number; repos: number; originators: CodexOriginatorCounts };
|
||||
gemini: { total_sessions: number; repos: number };
|
||||
};
|
||||
total_sessions: number;
|
||||
@@ -283,6 +288,30 @@ export function extractCwdFromJsonl(filePath: string): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Separates interactive Codex Desktop work from codex exec runs and Claude
|
||||
// Code-driven calls. Missing, unknown and non-string values count as "other",
|
||||
// so the buckets always sum to the Codex session total.
|
||||
function normalizeCodexOriginator(raw: unknown): CodexOriginator {
|
||||
if (typeof raw !== "string") return "other";
|
||||
const value = raw.trim().toLowerCase();
|
||||
if (value === "codex desktop" || value === "codex_desktop") return "desktop";
|
||||
if (value === "codex_exec" || value === "codex exec") return "exec";
|
||||
if (value === "claude code" || value === "claude_code") return "claude_code";
|
||||
return "other";
|
||||
}
|
||||
|
||||
function countCodexOriginators(sessions: Session[]): CodexOriginatorCounts {
|
||||
const counts: CodexOriginatorCounts = { desktop: 0, exec: 0, claude_code: 0, other: 0 };
|
||||
for (const s of sessions) {
|
||||
if (s.tool === "codex") counts[s.codexOriginator ?? "other"]++;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
function formatCodexOriginators(o: CodexOriginatorCounts): string {
|
||||
return `desktop=${o.desktop}, exec=${o.exec}, claude_code=${o.claude_code}, other=${o.other}`;
|
||||
}
|
||||
|
||||
function scanCodex(since: Date): Session[] {
|
||||
const sessionsDir = process.env.CODEX_SESSIONS_DIR || join(homedir(), ".codex", "sessions");
|
||||
if (!existsSync(sessionsDir)) return [];
|
||||
@@ -332,7 +361,11 @@ function scanCodex(since: Date): Session[] {
|
||||
if (!firstLine) continue;
|
||||
const meta = JSON.parse(firstLine);
|
||||
if (meta.type === "session_meta" && meta.payload?.cwd) {
|
||||
sessions.push({ tool: "codex", cwd: meta.payload.cwd });
|
||||
sessions.push({
|
||||
tool: "codex",
|
||||
cwd: meta.payload.cwd,
|
||||
codexOriginator: normalizeCodexOriginator(meta.payload.originator),
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
console.error(`Warning: could not parse Codex session ${filePath}`);
|
||||
@@ -503,6 +536,7 @@ async function resolveAndDeduplicate(sessions: Session[]): Promise<Repo[]> {
|
||||
remote,
|
||||
paths: data.paths,
|
||||
sessions: sessionCounts,
|
||||
codex_originators: countCodexOriginators(data.sessions),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -545,13 +579,15 @@ async function main() {
|
||||
const codexRepos = new Set(repos.filter((r) => r.sessions.codex > 0).map((r) => r.remote)).size;
|
||||
const geminiRepos = new Set(repos.filter((r) => r.sessions.gemini > 0).map((r) => r.remote)).size;
|
||||
|
||||
const codexOriginators = countCodexOriginators(codexSessions);
|
||||
|
||||
const result: DiscoveryResult = {
|
||||
window: since,
|
||||
start_date: startDate,
|
||||
repos,
|
||||
tools: {
|
||||
claude_code: { total_sessions: ccSessions.length, repos: ccRepos },
|
||||
codex: { total_sessions: codexSessions.length, repos: codexRepos },
|
||||
codex: { total_sessions: codexSessions.length, repos: codexRepos, originators: codexOriginators },
|
||||
gemini: { total_sessions: geminiSessions.length, repos: geminiRepos },
|
||||
},
|
||||
total_sessions: allSessions.length,
|
||||
@@ -564,13 +600,14 @@ async function main() {
|
||||
// Summary format
|
||||
console.log(`Window: ${since} (since ${startDate})`);
|
||||
console.log(`Sessions: ${allSessions.length} total (CC: ${ccSessions.length}, Codex: ${codexSessions.length}, Gemini: ${geminiSessions.length})`);
|
||||
if (codexSessions.length > 0) console.log(` Codex originators: ${formatCodexOriginators(codexOriginators)}`);
|
||||
console.log(`Repos: ${repos.length} unique`);
|
||||
console.log("");
|
||||
for (const repo of repos) {
|
||||
const total = repo.sessions.claude_code + repo.sessions.codex + repo.sessions.gemini;
|
||||
const tools = [];
|
||||
if (repo.sessions.claude_code > 0) tools.push(`CC:${repo.sessions.claude_code}`);
|
||||
if (repo.sessions.codex > 0) tools.push(`Codex:${repo.sessions.codex}`);
|
||||
if (repo.sessions.codex > 0) tools.push(`Codex:${repo.sessions.codex} (${formatCodexOriginators(repo.codex_originators)})`);
|
||||
if (repo.sessions.gemini > 0) tools.push(`Gemini:${repo.sessions.gemini}`);
|
||||
console.log(` ${repo.name} (${total} sessions) — ${tools.join(", ")}`);
|
||||
console.log(` Remote: ${repo.remote}`);
|
||||
|
||||
+7
-1
@@ -127,7 +127,13 @@ environment variable should be set. gstack detects this and adjusts:
|
||||
rendering prose to nobody
|
||||
- Focuses on task completion and prose reporting
|
||||
|
||||
Set the env var in sessions_spawn: `env: { OPENCLAW_SESSION: "1" }`
|
||||
`sessions_spawn` has no per-spawn environment argument
|
||||
([openclaw/openclaw#51654](https://github.com/openclaw/openclaw/issues/51654)
|
||||
was closed without adding one), so an `env` field in the tool call is ignored.
|
||||
Set `OPENCLAW_SESSION=1` in the environment of the Claude ACP harness or the
|
||||
Gateway service instead, keep credentials out of that wrapper, and confirm the
|
||||
variable from inside the spawned process (for example `echo $OPENCLAW_SESSION`)
|
||||
rather than trusting the session's final message.
|
||||
|
||||
### Explicit override: GSTACK_SESSION_KIND
|
||||
|
||||
|
||||
@@ -1046,7 +1046,7 @@ runtime match; records preserve original provenance. The cookie workflow's custo
|
||||
input, the other 11 judge cases, dynamic agent tests, and local runs without
|
||||
scoped cache configuration stay fresh.
|
||||
Scheduled/manual full coverage and `test:release` always run fresh.
|
||||
See [testing policy](CONTRIBUTING.md#test-tiers) for commands and measured targets.
|
||||
See [testing policy](../CONTRIBUTING.md#test-tiers) for commands and measured targets.
|
||||
Anything that needs Aside
|
||||
itself (`test/skill-e2e-aside.test.ts`, the Aside qa/design E2E cases, the
|
||||
live render in `test/aside-render.test.ts`) runs only on a Mac with the Aside
|
||||
|
||||
+1
-2
@@ -16,7 +16,6 @@ Conventions:
|
||||
- [/browse](browse/SKILL.md): Drive a real browser through Aside: open a page, read it, click through a flow, take screenshots, check console errors.
|
||||
- [/canary](canary/SKILL.md): Post-deploy canary monitoring.
|
||||
- [/careful](careful/SKILL.md): Safety guardrails for destructive commands.
|
||||
- [/claude-code](claude-code/SKILL.md): Claude Code CLI second opinion for non-Claude Code hosts.
|
||||
- [/codex](codex/SKILL.md): OpenAI Codex CLI wrapper — three modes.
|
||||
- [/context-restore](context-restore/SKILL.md): Restore working context saved earlier by /context-save.
|
||||
- [/context-save](context-save/SKILL.md): Save working context.
|
||||
@@ -31,7 +30,7 @@ Conventions:
|
||||
- [/document-generate](document-generate/SKILL.md): Generate missing documentation from scratch for a feature, module, or entire project.
|
||||
- [/document-release](document-release/SKILL.md): Release documentation audit.
|
||||
- [/freeze](freeze/SKILL.md): Restrict file edits to a specific directory for the session.
|
||||
- [/gstack](gstack/SKILL.md): Router for the gstack skill suite.
|
||||
- [/gstack](SKILL.md): Router for the gstack skill suite.
|
||||
- [/gstack-upgrade](gstack-upgrade/SKILL.md): Upgrade gstack to the latest version.
|
||||
- [/guard](guard/SKILL.md): Full safety mode: destructive command warnings + directory-scoped edits.
|
||||
- [/health](health/SKILL.md): Code quality dashboard.
|
||||
|
||||
+11
-4
@@ -11,6 +11,10 @@
|
||||
*
|
||||
* Output: gstack/llms.txt at repo root.
|
||||
*
|
||||
* Skill links point at the Claude host's generated SKILL.md (the files this
|
||||
* checkout ships). Skills the Claude host skips (claude-code) have no such
|
||||
* file, so they are left out of the index rather than linked to a template.
|
||||
*
|
||||
* Refresh: invoked from scripts/gen-skill-docs.ts after SKILL.md generation
|
||||
* so it regenerates automatically on every skill change.
|
||||
*
|
||||
@@ -20,7 +24,8 @@
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { mkdirpSync } from '../lib/fs-utils';
|
||||
import { discoverTemplates } from './discover-skills';
|
||||
import { discoverTemplates, includesSkill } from './discover-skills';
|
||||
import claudeHost from '../hosts/claude';
|
||||
import { COMMAND_DESCRIPTIONS as BROWSE_COMMANDS } from '../browse/src/commands';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
@@ -29,6 +34,7 @@ const OUTPUT = path.join(ROOT, 'gstack', 'llms.txt');
|
||||
interface SkillEntry {
|
||||
name: string;
|
||||
description: string;
|
||||
output: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -36,7 +42,7 @@ interface SkillEntry {
|
||||
* `name` and `description`. description: | followed by indented lines is
|
||||
* the gstack convention; we collapse those into a single paragraph.
|
||||
*/
|
||||
function parseSkillFrontmatter(filePath: string): SkillEntry | null {
|
||||
function parseSkillFrontmatter(filePath: string): Omit<SkillEntry, 'output'> | null {
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
if (!content.startsWith('---')) return null;
|
||||
const end = content.indexOf('\n---', 3);
|
||||
@@ -136,6 +142,7 @@ export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<Gener
|
||||
const templates = discoverTemplates(root);
|
||||
const skills: SkillEntry[] = [];
|
||||
for (const t of templates) {
|
||||
if (!includesSkill(claudeHost, path.dirname(t.tmpl))) continue;
|
||||
const filePath = path.join(root, t.tmpl);
|
||||
const entry = parseSkillFrontmatter(filePath);
|
||||
if (!entry) {
|
||||
@@ -145,7 +152,7 @@ export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<Gener
|
||||
}
|
||||
continue;
|
||||
}
|
||||
skills.push(entry);
|
||||
skills.push({ ...entry, output: t.output });
|
||||
}
|
||||
skills.sort((a, b) => a.name.localeCompare(b.name));
|
||||
|
||||
@@ -168,7 +175,7 @@ export async function generateLlmsTxt(opts: GenerateOptions = {}): Promise<Gener
|
||||
lines.push('');
|
||||
for (const skill of skills) {
|
||||
const summary = oneLine(skill.description);
|
||||
lines.push(`- [/${skill.name}](${skill.name}/SKILL.md): ${summary}`);
|
||||
lines.push(`- [/${skill.name}](${skill.output}): ${summary}`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { resultFromCodexStream } from './helpers/providers/gpt';
|
||||
import { GeminiAdapter } from './helpers/providers/gemini';
|
||||
import { estimateCostUsd } from './helpers/pricing';
|
||||
import { formatMarkdown, formatTable, type BenchmarkReport } from './helpers/benchmark-runner';
|
||||
|
||||
describe('Codex token accounting', () => {
|
||||
const turn = JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 21_155, cached_input_tokens: 9_984, output_tokens: 5 } });
|
||||
|
||||
test('cached input is split out of input_tokens and priced at the cache-read rate', () => {
|
||||
const result = resultFromCodexStream(turn, { model: 'gpt-5.4' });
|
||||
expect(result.tokens).toEqual({ input: 11_171, cached: 9_984, output: 5 });
|
||||
// gpt-5.4: $2.50/MTok uncached, 10% of that for cache reads, $10/MTok output.
|
||||
expect(estimateCostUsd(result.tokens, 'gpt-5.4')).toBeCloseTo(0.030474, 5);
|
||||
});
|
||||
|
||||
test('turns accumulate both buckets', () => {
|
||||
const second = JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 200, cached_input_tokens: 150, output_tokens: 10 } });
|
||||
expect(resultFromCodexStream(`${turn}\n${second}`).tokens).toEqual({ input: 11_221, cached: 10_134, output: 15 });
|
||||
});
|
||||
|
||||
test('reports show new and cached input', () => {
|
||||
const report: BenchmarkReport = {
|
||||
prompt: 'p', workdir: '/tmp', startedAt: '2026-10-07T00:00:00Z', durationMs: 1,
|
||||
entries: [{ provider: 'gpt', family: 'gpt', available: true, result: resultFromCodexStream(turn, { model: 'gpt-5.4' }), costUsd: 0.03 }],
|
||||
};
|
||||
expect(formatTable(report)).toContain('11171+9984→5');
|
||||
expect(formatMarkdown(report)).toContain('11171+9984→5');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Gemini availability', () => {
|
||||
let home: string;
|
||||
|
||||
beforeEach(() => {
|
||||
home = fs.mkdtempSync(path.join(os.tmpdir(), 'gemini-avail-'));
|
||||
fs.mkdirSync(path.join(home, 'bin'));
|
||||
fs.writeFileSync(path.join(home, 'bin', 'gemini'), '#!/bin/sh\nexit 0\n', { mode: 0o755 });
|
||||
fs.mkdirSync(path.join(home, '.gemini'));
|
||||
fs.writeFileSync(path.join(home, '.gemini', 'oauth_creds.json'), '{}');
|
||||
});
|
||||
|
||||
afterEach(() => fs.rmSync(home, { recursive: true, force: true }));
|
||||
|
||||
// Bun fixes os.homedir() at startup, so each check runs in a child with the temp HOME.
|
||||
function availability(extraEnv: Record<string, string> = {}): { ok: boolean; reason?: string } {
|
||||
const env: Record<string, string> = { ...(process.env as Record<string, string>), HOME: home, PATH: `${path.join(home, 'bin')}:${process.env.PATH}`, ...extraEnv };
|
||||
delete env.GOOGLE_API_KEY;
|
||||
if (!extraEnv.GEMINI_API_KEY) delete env.GEMINI_API_KEY;
|
||||
const script = `const { GeminiAdapter } = await import(${JSON.stringify(path.join(import.meta.dir, 'helpers/providers/gemini.ts'))}); console.log(JSON.stringify(await new GeminiAdapter().available()));`;
|
||||
const child = spawnSync(process.execPath, ['-e', script], { env, encoding: 'utf-8', timeout: 30_000 });
|
||||
expect(child.status).toBe(0);
|
||||
return JSON.parse(child.stdout.trim().split('\n').pop()!);
|
||||
}
|
||||
|
||||
test('stored OAuth alone is not ready and names the API key fix', () => {
|
||||
const check = availability();
|
||||
expect(check.ok).toBe(false);
|
||||
expect(check.reason).toContain('stored Gemini OAuth');
|
||||
expect(check.reason).toContain('GEMINI_API_KEY');
|
||||
});
|
||||
|
||||
test('an API key is ready', () => {
|
||||
expect(availability({ GEMINI_API_KEY: 'fixture-not-a-key' })).toEqual({ ok: true });
|
||||
});
|
||||
});
|
||||
@@ -291,6 +291,95 @@ describe("gstack-global-discover", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("codex originator buckets", () => {
|
||||
let tmpDir: string;
|
||||
let codexDir: string;
|
||||
let repoDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), "gstack-codex-orig-"));
|
||||
const now = new Date();
|
||||
const y = now.getFullYear().toString();
|
||||
const m = String(now.getMonth() + 1).padStart(2, "0");
|
||||
const d = String(now.getDate()).padStart(2, "0");
|
||||
codexDir = join(tmpDir, "codex-home", "sessions", y, m, d);
|
||||
mkdirSync(codexDir, { recursive: true });
|
||||
repoDir = join(tmpDir, "fake-repo");
|
||||
mkdirSync(repoDir);
|
||||
spawnSync("git", ["init", "-q"], { cwd: repoDir, stdio: "pipe", timeout: 30_000 });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// Pass no argument to omit originator from the payload.
|
||||
function writeCodex(...originator: unknown[]) {
|
||||
const payload: Record<string, unknown> = { id: `t-${Math.random()}`, timestamp: new Date().toISOString(), cwd: repoDir };
|
||||
if (originator.length) payload.originator = originator[0];
|
||||
const name = `rollout-${new Date().toISOString().replace(/[:.]/g, "-")}-${Math.random().toString(36).slice(2)}.jsonl`;
|
||||
writeFileSync(join(codexDir, name), JSON.stringify({ timestamp: new Date().toISOString(), type: "session_meta", payload }) + "\n");
|
||||
}
|
||||
|
||||
function discover(format: "json" | "summary" = "json") {
|
||||
const r = spawnSync("bun", ["run", scriptPath, "--since", "1h", "--format", format], {
|
||||
encoding: "utf-8",
|
||||
timeout: 30_000,
|
||||
env: { ...process.env, CODEX_SESSIONS_DIR: join(tmpDir, "codex-home", "sessions") },
|
||||
});
|
||||
expect(r.status).toBe(0);
|
||||
return r.stdout;
|
||||
}
|
||||
|
||||
const originators = () => JSON.parse(discover()).tools.codex.originators;
|
||||
|
||||
test("Codex Desktop goes to desktop", () => {
|
||||
writeCodex("Codex Desktop");
|
||||
expect(originators()).toEqual({ desktop: 1, exec: 0, claude_code: 0, other: 0 });
|
||||
});
|
||||
|
||||
test("codex_exec goes to exec", () => {
|
||||
writeCodex("codex_exec");
|
||||
expect(originators()).toEqual({ desktop: 0, exec: 1, claude_code: 0, other: 0 });
|
||||
});
|
||||
|
||||
test("Claude Code goes to claude_code", () => {
|
||||
writeCodex("Claude Code");
|
||||
expect(originators()).toEqual({ desktop: 0, exec: 0, claude_code: 1, other: 0 });
|
||||
});
|
||||
|
||||
test("surrounding whitespace and case are ignored", () => {
|
||||
writeCodex(" CODEX DESKTOP ");
|
||||
expect(originators()).toEqual({ desktop: 1, exec: 0, claude_code: 0, other: 0 });
|
||||
});
|
||||
|
||||
test("missing, null, numeric and unknown originators go to other", () => {
|
||||
writeCodex();
|
||||
writeCodex(null);
|
||||
writeCodex(42);
|
||||
writeCodex("future-agent");
|
||||
const json = JSON.parse(discover());
|
||||
expect(json.tools.codex.total_sessions).toBe(4);
|
||||
expect(json.tools.codex.originators).toEqual({ desktop: 0, exec: 0, claude_code: 0, other: 4 });
|
||||
});
|
||||
|
||||
test("per-repo and global buckets sum to the codex total and print in the summary", () => {
|
||||
writeCodex("Codex Desktop");
|
||||
writeCodex("codex_exec");
|
||||
writeCodex("codex_exec");
|
||||
writeCodex("Claude Code");
|
||||
writeCodex();
|
||||
const json = JSON.parse(discover());
|
||||
const sum = (o: Record<string, number>) => Object.values(o).reduce((a, b) => a + b, 0);
|
||||
expect(json.tools.codex.total_sessions).toBe(5);
|
||||
expect(sum(json.tools.codex.originators)).toBe(5);
|
||||
const repo = json.repos.find((r: { paths: string[] }) => r.paths.includes(repoDir));
|
||||
expect(repo.codex_originators).toEqual({ desktop: 1, exec: 2, claude_code: 1, other: 1 });
|
||||
expect(sum(repo.codex_originators)).toBe(repo.sessions.codex);
|
||||
expect(discover("summary")).toContain("Codex originators: desktop=1, exec=2, claude_code=1, other=1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("discovery output structure", () => {
|
||||
test("repos have required fields", () => {
|
||||
const result = spawnSync(
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* one. Per-provider auth/timeout/rate-limit errors don't abort the batch.
|
||||
*/
|
||||
|
||||
import type { ProviderAdapter, RunOpts, RunResult } from './providers/types';
|
||||
import type { ProviderAdapter, RunOpts, RunResult, TokenUsage } from './providers/types';
|
||||
import { ClaudeAdapter } from './providers/claude';
|
||||
import { GptAdapter } from './providers/gpt';
|
||||
import { GeminiAdapter } from './providers/gemini';
|
||||
@@ -99,21 +99,21 @@ export async function runBenchmark(input: BenchmarkInput): Promise<BenchmarkRepo
|
||||
}
|
||||
|
||||
export function formatTable(report: BenchmarkReport): string {
|
||||
const header = `Model Latency In→Out Tokens Cost Quality Tool Calls Notes`;
|
||||
const header = `Model Latency In(+cached)→Out Tokens Cost Quality Tool Calls Notes`;
|
||||
const sep = '-'.repeat(header.length);
|
||||
const rows: string[] = [header, sep];
|
||||
for (const e of report.entries) {
|
||||
if (!e.available) {
|
||||
rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 20)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
|
||||
rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 24)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
|
||||
continue;
|
||||
}
|
||||
const r = e.result!;
|
||||
if (r.error) {
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(fmtTokens(r.tokens), 24)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
|
||||
continue;
|
||||
}
|
||||
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(fmtTokens(r.tokens), 24)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
|
||||
}
|
||||
return rows.join('\n');
|
||||
}
|
||||
@@ -130,8 +130,8 @@ export function formatMarkdown(report: BenchmarkReport): string {
|
||||
`**Workdir:** \`${report.workdir}\``,
|
||||
`**Total duration:** ${msToStr(report.durationMs)}`,
|
||||
'',
|
||||
'| Model | Latency | Tokens (in→out) | Cost | Quality | Tools | Notes |',
|
||||
'|-------|---------|-----------------|------|---------|-------|-------|',
|
||||
'| Model | Latency | Tokens (in+cached→out) | Cost | Quality | Tools | Notes |',
|
||||
'|-------|---------|------------------------|------|---------|-------|-------|',
|
||||
];
|
||||
for (const e of report.entries) {
|
||||
if (!e.available) {
|
||||
@@ -140,11 +140,11 @@ export function formatMarkdown(report: BenchmarkReport): string {
|
||||
}
|
||||
const r = e.result!;
|
||||
if (r.error) {
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${fmtTokens(r.tokens)} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
|
||||
continue;
|
||||
}
|
||||
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${fmtTokens(r.tokens)} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -153,6 +153,12 @@ function pad(s: string, n: number): string {
|
||||
return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length);
|
||||
}
|
||||
|
||||
function fmtTokens(tokens: TokenUsage): string {
|
||||
return tokens.cached === undefined
|
||||
? `${tokens.input}→${tokens.output}`
|
||||
: `${tokens.input}+${tokens.cached}→${tokens.output}`;
|
||||
}
|
||||
|
||||
function msToStr(ms: number): string {
|
||||
if (ms < 1000) return `${ms}ms`;
|
||||
return `${(ms / 1000).toFixed(1)}s`;
|
||||
|
||||
@@ -109,9 +109,10 @@ export function resultFromGeminiStream(
|
||||
/**
|
||||
* Gemini adapter — wraps the `gemini` CLI.
|
||||
*
|
||||
* Auth: GEMINI_API_KEY / GOOGLE_API_KEY (preferred), or ~/.gemini oauth.
|
||||
* Personal OAuth free-tier is no longer supported by gemini CLI — use an
|
||||
* AI Studio API key. Antigravity is a separate product/quota path.
|
||||
* Auth: GEMINI_API_KEY / GOOGLE_API_KEY. Stored ~/.gemini OAuth alone is
|
||||
* reported as not ready: personal OAuth free-tier is no longer supported by
|
||||
* gemini CLI — use an AI Studio API key. Antigravity is a separate
|
||||
* product/quota path.
|
||||
*
|
||||
* Headless flags always passed:
|
||||
* --output-format stream-json — NDJSON events (message/tool_use/result)
|
||||
@@ -134,17 +135,15 @@ export class GeminiAdapter implements ProviderAdapter {
|
||||
const legacyCfgDir = path.join(os.homedir(), '.config', 'gemini');
|
||||
const newCfgDir = path.join(os.homedir(), '.gemini');
|
||||
const newOauth = path.join(newCfgDir, 'oauth_creds.json');
|
||||
const hasCfg = !opts?.csoProducer && (fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth));
|
||||
// CLI accepts either name; Google AI Studio keys are usually GEMINI_API_KEY.
|
||||
const hasKey = !!(process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY);
|
||||
if (!hasCfg && !hasKey) {
|
||||
return {
|
||||
ok: false,
|
||||
reason:
|
||||
'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.',
|
||||
};
|
||||
}
|
||||
return { ok: true };
|
||||
if (process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY) return { ok: true };
|
||||
const hasOauth = !opts?.csoProducer && (fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth));
|
||||
return {
|
||||
ok: false,
|
||||
reason: hasOauth
|
||||
? 'Only stored Gemini OAuth found; personal OAuth free-tier is no longer supported by gemini CLI and headless runs can fail. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey.'
|
||||
: 'No Gemini auth found. Export GEMINI_API_KEY (or GOOGLE_API_KEY) from https://aistudio.google.com/app/apikey — personal OAuth free-tier is no longer supported by gemini CLI.',
|
||||
};
|
||||
}
|
||||
|
||||
async run(opts: RunOpts): Promise<RunResult> {
|
||||
|
||||
@@ -99,9 +99,13 @@ export class GptAdapter implements ProviderAdapter {
|
||||
}
|
||||
}
|
||||
|
||||
/** Map `codex exec --json` output while requiring evidence-bearing text for paid producers. */
|
||||
/**
|
||||
* Map `codex exec --json` output while requiring evidence-bearing text for paid producers.
|
||||
* Codex reports cached_input_tokens as a subset of input_tokens; split them so
|
||||
* tokens.input is uncached-only and tokens.cached is the disjoint cache-read count.
|
||||
*/
|
||||
export function resultFromCodexStream(raw:string,opts:{model?:string;durationMs?:number;producer?:boolean}={}):RunResult{
|
||||
let output='',input=0,out=0,toolCalls=0,modelUsed:string|undefined;
|
||||
let output='',input=0,cached=0,out=0,toolCalls=0,modelUsed:string|undefined;
|
||||
for(const line of raw.split('\n')){
|
||||
const text=line.trim();if(!text)continue;
|
||||
try{
|
||||
@@ -110,14 +114,14 @@ export function resultFromCodexStream(raw:string,opts:{model?:string;durationMs?
|
||||
if(obj.item.type==='agent_message'&&typeof obj.item.text==='string')output+=(output?'\n':'')+obj.item.text;
|
||||
else if(obj.item.type==='command_execution')toolCalls++;
|
||||
}else if(obj.type==='turn.completed'){
|
||||
const usage=obj.usage??{};input+=usage.input_tokens??0;out+=usage.output_tokens??0;
|
||||
const usage=obj.usage??{},turnCached=usage.cached_input_tokens??0;input+=Math.max(0,(usage.input_tokens??0)-turnCached);cached+=turnCached;out+=usage.output_tokens??0;
|
||||
if(typeof obj.model==='string'&&obj.model)modelUsed=obj.model;
|
||||
}
|
||||
}catch{/* Codex can mix diagnostic text into the JSONL stream. */}
|
||||
}
|
||||
const durationMs=opts.durationMs??0,resolvedModel=modelUsed||opts.model||CODEX_FRONTIER_MODEL;
|
||||
if(opts.producer&&!output.trim())return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:resolvedModel,error:{code:'unknown',reason:'empty output from codex CLI (exit 0)'}};
|
||||
return{output,tokens:{input,output:out},durationMs,toolCalls,modelUsed:resolvedModel};
|
||||
return{output,tokens:{input,cached,output:out},durationMs,toolCalls,modelUsed:resolvedModel};
|
||||
}
|
||||
|
||||
export interface CodexProducerPaths {
|
||||
|
||||
@@ -36,6 +36,13 @@ describe('gen-llms-txt — shape', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('every link in the index resolves to an existing file', () => {
|
||||
const links = [...generated.content.matchAll(/\]\(([^)]+)\)/g)].map((m) => m[1]);
|
||||
expect(links.length).toBeGreaterThanOrEqual(generated.skills.length);
|
||||
const missing = links.filter((href) => !fs.existsSync(path.join(ROOT, href)));
|
||||
expect(missing).toEqual([]);
|
||||
});
|
||||
|
||||
test('every browse command in COMMAND_DESCRIPTIONS appears in the index', () => {
|
||||
expect(generated.browseCommands.length).toBeGreaterThan(0);
|
||||
for (const cmd of generated.browseCommands) {
|
||||
|
||||
@@ -32,4 +32,9 @@ describe('OpenClaw native skills', () => {
|
||||
expect((parsed.description as string).length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('integration docs do not advertise an unsupported per-spawn env', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'docs/OPENCLAW.md'), 'utf-8');
|
||||
expect(content).not.toContain('env: { OPENCLAW_SESSION');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user