mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-04 10:26:52 +02:00
test: clean up the paid eval lane (B1-B4, B6, B7)
- B1: delete paid files that assert nothing or cannot pass meaningfully: skill-llm-eval-spec and skill-e2e-spec-execute (test.todo), gemini-e2e (+ gemini-session-runner; no gemini CLI in CI), ship-idempotency (red since v1.63), the two opus-4-7 *-sonnet overlay wrappers, conductor-prose (+ its source-evaluation replay), codex-e2e-plan-format; drop their keys, scripts and census rows. - B2: skill-llm-eval grades browse/sections/command-list.md with one union judge that also carries the baseline score pin; regression-vs-baseline deleted (paid run: pass, c4/c4/a4). - B3: memory-pipeline, ios-qa, ios-qa-swift-build and plan-tune-cathedral make no model calls; renamed out of the paid glob so they run on every PR. Swift builds need GSTACK_TEST_SWIFT=1; device stub deleted. - B4: codex-e2e*, outside-voice, aside and ios-device cannot run in the CI image; excluded from the weekly lane with a tracked re-entry condition. - B6: fold opus-47's negative routing controls into skill-routing-e2e journey-negatives (paid run: 3/3 unrouted) and delete the file. - B7: delete the never-green brain-privacy-gate eval; a free gstack-skill-start test now proves consent precedes artifacts egress.
This commit is contained in:
garrytan committed
1 parent
5d032ef299
commit
53e7f3212f
58 files changed
+273
-2594
No files matched your search
@@ -0,0 +1,424 @@
|
||||
/**
|
||||
* /plan-tune cathedral contract tests (T16) — 5 scenarios, free suite.
|
||||
*
|
||||
* Each scenario verifies that the cathedral's substrate works end-to-end
|
||||
* through local hook and bin invocations. No model is called: these scenarios
|
||||
* exercise the installed-file contracts, using synthetic hook envelopes and
|
||||
* a synthetic Codex session. Unit tests cover the individual components.
|
||||
*
|
||||
* Each scenario uses GSTACK_STATE_ROOT to isolate from the user's real
|
||||
* ~/.gstack (per cathedral T1 + Codex D16 fix). Every attempt gets a fresh fixture.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { ROOT, copyDirSync } from './helpers/e2e-helpers';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
/** Scaffold a fixture project with the bins + scripts the cathedral needs. */
|
||||
function scaffoldFixture(workDir: string): { workDir: string; stateRoot: string; slug: string; env: NodeJS.ProcessEnv } {
|
||||
const stateRoot = path.join(workDir, '.gstack-state');
|
||||
fs.mkdirSync(stateRoot, { recursive: true });
|
||||
|
||||
// git init so gstack-slug resolves a deterministic slug.
|
||||
spawnSync('git', ['init', '-b', 'main'], { cwd: workDir, stdio: 'pipe', timeout: 30_000 });
|
||||
spawnSync('git', ['config', 'user.email', 't@t.com'], { cwd: workDir, stdio: 'pipe', timeout: 30_000 });
|
||||
spawnSync('git', ['config', 'user.name', 'T'], { cwd: workDir, stdio: 'pipe', timeout: 30_000 });
|
||||
fs.writeFileSync(path.join(workDir, 'README.md'), '# cathedral fixture\n');
|
||||
spawnSync('git', ['add', '.'], { cwd: workDir, stdio: 'pipe', timeout: 30_000 });
|
||||
spawnSync('git', ['commit', '-m', 'init'], { cwd: workDir, stdio: 'pipe', timeout: 30_000 });
|
||||
|
||||
// Copy bins.
|
||||
const binDir = path.join(workDir, 'bin');
|
||||
fs.mkdirSync(binDir, { recursive: true });
|
||||
for (const script of [
|
||||
'gstack-slug',
|
||||
'gstack-config',
|
||||
'gstack-paths',
|
||||
'gstack-question-log',
|
||||
'gstack-question-preference',
|
||||
'gstack-developer-profile',
|
||||
'gstack-codex-session-import',
|
||||
'gstack-distill-free-text',
|
||||
'gstack-distill-apply',
|
||||
]) {
|
||||
const src = path.join(ROOT, 'bin', script);
|
||||
if (fs.existsSync(src)) {
|
||||
fs.copyFileSync(src, path.join(binDir, script));
|
||||
fs.chmodSync(path.join(binDir, script), 0o755);
|
||||
}
|
||||
}
|
||||
|
||||
// Copy scripts that the bins import.
|
||||
const scriptsDir = path.join(workDir, 'scripts');
|
||||
fs.mkdirSync(scriptsDir, { recursive: true });
|
||||
for (const f of [
|
||||
'question-registry.ts',
|
||||
'psychographic-signals.ts',
|
||||
'archetypes.ts',
|
||||
'one-way-doors.ts',
|
||||
'declared-annotation.ts',
|
||||
]) {
|
||||
const src = path.join(ROOT, 'scripts', f);
|
||||
if (fs.existsSync(src)) fs.copyFileSync(src, path.join(scriptsDir, f));
|
||||
}
|
||||
|
||||
// Both imports are resolved relative to the copied install, not the source checkout.
|
||||
const libDir = path.join(workDir, 'lib');
|
||||
fs.mkdirSync(libDir, { recursive: true });
|
||||
for (const file of ['jsonl-store.ts', 'is-conductor.ts']) {
|
||||
fs.copyFileSync(path.join(ROOT, 'lib', file), path.join(libDir, file));
|
||||
}
|
||||
|
||||
// Copy hooks dir.
|
||||
copyDirSync(path.join(ROOT, 'hosts', 'claude', 'hooks'), path.join(workDir, 'hosts', 'claude', 'hooks'));
|
||||
|
||||
const slug = path.basename(workDir).replace(/[^a-zA-Z0-9._-]/g, '');
|
||||
// These contracts exercise ordinary Claude Code hook delivery. Conductor's
|
||||
// question redirection has its own tests and must not replace this branch.
|
||||
const env = { ...process.env };
|
||||
delete env.CONDUCTOR_WORKSPACE_PATH;
|
||||
delete env.CONDUCTOR_PORT;
|
||||
delete env.OPENCLAW_SESSION;
|
||||
delete env.GSTACK_SESSION_KIND;
|
||||
return { workDir, stateRoot, slug, env };
|
||||
}
|
||||
|
||||
function cleanupFixture(workDir: string): void {
|
||||
try {
|
||||
fs.rmSync(workDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
|
||||
/** Own setup, assertions and cleanup inside each callback. */
|
||||
function testCathedral(
|
||||
name: string,
|
||||
prefix: string,
|
||||
run: (fixture: ReturnType<typeof scaffoldFixture>) => Promise<void>,
|
||||
): void {
|
||||
test(name, async () => {
|
||||
const workDir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||
try {
|
||||
await run(scaffoldFixture(workDir));
|
||||
} finally {
|
||||
cleanupFixture(workDir);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 1: Hook capture — PostToolUse hook writes to question-log.jsonl
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('PlanTune cathedral E2E: hook capture', () => {
|
||||
testCathedral('plan-tune-hook-capture', 'cathedral-cap-', async (fixture) => {
|
||||
// Direct hook invocation simulates Claude Code's PostToolUse delivery.
|
||||
// E2E verifies the hook + bin chain works against real bins on disk
|
||||
// (the unit test exercises this with mocks).
|
||||
const hookPath = path.join(fixture.workDir, 'hosts', 'claude', 'hooks', 'question-log-hook');
|
||||
const payload = {
|
||||
session_id: 'cathedral-e2e-cap',
|
||||
tool_name: 'AskUserQuestion',
|
||||
tool_use_id: 'tu-cap-1',
|
||||
tool_input: {
|
||||
questions: [
|
||||
{
|
||||
question:
|
||||
'D1 — Cathedral E2E capture <gstack-qid:ship-test-failure-triage>\nRecommendation: A',
|
||||
options: ['A) Fix now (recommended)', 'B) Investigate'],
|
||||
},
|
||||
],
|
||||
},
|
||||
tool_response: { answers: [{ option_label: 'A) Fix now (recommended)' }] },
|
||||
cwd: fixture.workDir,
|
||||
};
|
||||
const res = spawnSync(hookPath, [], {
|
||||
env: {
|
||||
...fixture.env,
|
||||
GSTACK_STATE_ROOT: fixture.stateRoot,
|
||||
GSTACK_QUESTION_LOG_NO_DERIVE: '1',
|
||||
},
|
||||
input: JSON.stringify(payload),
|
||||
encoding: 'utf-8',
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(res.status).toBe(0);
|
||||
const logPath = path.join(fixture.stateRoot, 'projects', fixture.slug, 'question-log.jsonl');
|
||||
expect(fs.existsSync(logPath)).toBe(true);
|
||||
const lines = fs.readFileSync(logPath, 'utf-8').trim().split('\n');
|
||||
expect(lines.length).toBeGreaterThanOrEqual(1);
|
||||
const evt = JSON.parse(lines[0]);
|
||||
expect(evt.source).toBe('hook');
|
||||
expect(evt.question_id).toBe('ship-test-failure-triage');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 2: Enforcement — never-ask preference + marker + 2-way → deny
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('PlanTune cathedral E2E: enforcement', () => {
|
||||
testCathedral('plan-tune-enforcement', 'cathedral-enf-', async (fixture) => {
|
||||
fs.mkdirSync(path.join(fixture.stateRoot, 'projects', fixture.slug), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(fixture.stateRoot, 'projects', fixture.slug, 'question-preferences.json'),
|
||||
JSON.stringify({ 'ship-changelog-voice-polish': 'never-ask' }),
|
||||
);
|
||||
const hookPath = path.join(
|
||||
fixture.workDir,
|
||||
'hosts',
|
||||
'claude',
|
||||
'hooks',
|
||||
'question-preference-hook',
|
||||
);
|
||||
const payload = {
|
||||
session_id: 'cathedral-e2e-enf',
|
||||
tool_name: 'AskUserQuestion',
|
||||
tool_use_id: 'tu-enf-1',
|
||||
tool_input: {
|
||||
questions: [
|
||||
{
|
||||
question:
|
||||
'<gstack-qid:ship-changelog-voice-polish> Polish CHANGELOG entry?',
|
||||
options: ['A) Accept (recommended)', 'B) Skip'],
|
||||
},
|
||||
],
|
||||
},
|
||||
cwd: fixture.workDir,
|
||||
};
|
||||
const res = spawnSync(hookPath, [], {
|
||||
env: {
|
||||
...fixture.env,
|
||||
GSTACK_STATE_ROOT: fixture.stateRoot,
|
||||
GSTACK_QUESTION_LOG_NO_DERIVE: '1',
|
||||
},
|
||||
input: JSON.stringify(payload),
|
||||
encoding: 'utf-8',
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(res.status).toBe(0);
|
||||
const parsed = JSON.parse(res.stdout || '{}');
|
||||
expect(parsed.hookSpecificOutput?.permissionDecision).toBe('deny');
|
||||
expect(parsed.hookSpecificOutput?.permissionDecisionReason).toContain('Accept');
|
||||
|
||||
// Auto-decided event was logged.
|
||||
const logPath = path.join(fixture.stateRoot, 'projects', fixture.slug, 'question-log.jsonl');
|
||||
expect(fs.existsSync(logPath)).toBe(true);
|
||||
const events = fs
|
||||
.readFileSync(logPath, 'utf-8')
|
||||
.trim()
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((l) => JSON.parse(l));
|
||||
const auto = events.filter((e) => e.source === 'auto-decided');
|
||||
expect(auto.length).toBe(1);
|
||||
expect(auto[0].question_id).toBe('ship-changelog-voice-polish');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 3: Annotation — declared profile injected via additionalContext
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('PlanTune cathedral E2E: annotation', () => {
|
||||
testCathedral('plan-tune-annotation', 'cathedral-ann-', async (fixture) => {
|
||||
// Strong declared profile that should annotate any signal_key=detail-preference question.
|
||||
fs.writeFileSync(
|
||||
path.join(fixture.stateRoot, 'developer-profile.json'),
|
||||
JSON.stringify({ declared: { detail_preference: 0.9 } }),
|
||||
);
|
||||
// Seed a memory nugget for the matching signal_key.
|
||||
fs.writeFileSync(
|
||||
path.join(fixture.stateRoot, 'free-text-memory.json'),
|
||||
JSON.stringify({
|
||||
nuggets: [
|
||||
{
|
||||
nugget: 'User prefers verbose explanations with tradeoffs',
|
||||
applies_to_signal_keys: ['detail-preference'],
|
||||
applied_at: new Date().toISOString(),
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const hookPath = path.join(
|
||||
fixture.workDir,
|
||||
'hosts',
|
||||
'claude',
|
||||
'hooks',
|
||||
'question-preference-hook',
|
||||
);
|
||||
const payload = {
|
||||
session_id: 'cathedral-e2e-ann',
|
||||
tool_name: 'AskUserQuestion',
|
||||
tool_use_id: 'tu-ann-1',
|
||||
tool_input: {
|
||||
questions: [
|
||||
{
|
||||
question: '<gstack-qid:ship-todos-reorganize> Reorganize TODOs?',
|
||||
options: ['A) Accept (recommended)', 'B) Skip'],
|
||||
},
|
||||
],
|
||||
},
|
||||
cwd: fixture.workDir,
|
||||
};
|
||||
const res = spawnSync(hookPath, [], {
|
||||
env: {
|
||||
...fixture.env,
|
||||
GSTACK_STATE_ROOT: fixture.stateRoot,
|
||||
GSTACK_QUESTION_LOG_NO_DERIVE: '1',
|
||||
},
|
||||
input: JSON.stringify(payload),
|
||||
encoding: 'utf-8',
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(res.status).toBe(0);
|
||||
const parsed = JSON.parse(res.stdout || '{}');
|
||||
// #2035: memory-nugget delivery is additionalContext-ONLY. Emitting a
|
||||
// permissionDecision here (the old 'defer') pauses the tool call for a
|
||||
// resumption that never comes in interactive sessions.
|
||||
expect('permissionDecision' in (parsed.hookSpecificOutput ?? {})).toBe(false);
|
||||
expect(parsed.hookSpecificOutput?.additionalContext).toContain('verbose explanations');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 4: Codex import — JSONL session → import bin → log fills
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('PlanTune cathedral E2E: codex import', () => {
|
||||
testCathedral('plan-tune-codex-import', 'cathedral-cdx-', async (fixture) => {
|
||||
const sessionFile = path.join(fixture.workDir, 'rollout-cathedral.jsonl');
|
||||
const lines = [
|
||||
JSON.stringify({
|
||||
type: 'session_meta',
|
||||
payload: { id: 'cathedral-sess-1', cwd: fixture.workDir },
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: new Date().toISOString(),
|
||||
type: 'event_msg',
|
||||
payload: {
|
||||
type: 'agent_message',
|
||||
message:
|
||||
'D1 — Cathedral import <gstack-qid:plan-eng-review-scope-reduce>\nRecommendation: A\nA) Reduce (recommended)\nB) Keep',
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: new Date().toISOString(),
|
||||
type: 'event_msg',
|
||||
payload: { type: 'user_message', message: 'A' },
|
||||
}),
|
||||
];
|
||||
fs.writeFileSync(sessionFile, lines.join('\n') + '\n');
|
||||
const bin = path.join(fixture.workDir, 'bin', 'gstack-codex-session-import');
|
||||
const res = spawnSync(bin, [sessionFile], {
|
||||
env: {
|
||||
...fixture.env,
|
||||
GSTACK_STATE_ROOT: fixture.stateRoot,
|
||||
GSTACK_QUESTION_LOG_NO_DERIVE: '1',
|
||||
},
|
||||
encoding: 'utf-8',
|
||||
cwd: fixture.workDir,
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(res.status).toBe(0);
|
||||
expect(res.stdout).toContain('IMPORTED: 1');
|
||||
const logPath = path.join(fixture.stateRoot, 'projects', fixture.slug, 'question-log.jsonl');
|
||||
expect(fs.existsSync(logPath)).toBe(true);
|
||||
const events = fs
|
||||
.readFileSync(logPath, 'utf-8')
|
||||
.trim()
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((l) => JSON.parse(l));
|
||||
expect(events.length).toBe(1);
|
||||
expect(events[0].source).toBe('codex-import-marker');
|
||||
expect(events[0].question_id).toBe('plan-eng-review-scope-reduce');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 5: Dream cycle round-trip — capture → distill (mocked) → apply →
|
||||
// re-fire → memory injection
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('PlanTune cathedral E2E: dream cycle', () => {
|
||||
testCathedral('plan-tune-dream-cycle', 'cathedral-dream-', async (fixture) => {
|
||||
// Seed proposals file directly (the SDK call is exercised by the unit
|
||||
// test; here we verify apply → re-fire round-trip on top of a known
|
||||
// proposal shape).
|
||||
fs.mkdirSync(path.join(fixture.stateRoot, 'projects', fixture.slug), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(fixture.stateRoot, 'projects', fixture.slug, 'distillation-proposals.json'),
|
||||
JSON.stringify({
|
||||
generated_at: new Date().toISOString(),
|
||||
source_event_count: 1,
|
||||
proposals: [
|
||||
{
|
||||
kind: 'memory-nugget',
|
||||
confidence: 0.95,
|
||||
nugget: 'User wants every fix tested before shipping',
|
||||
applies_to_signal_keys: ['test-discipline'],
|
||||
source_quotes: ['always add tests for any fix'],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
// 1. Apply the proposal via gstack-distill-apply.
|
||||
const applyBin = path.join(fixture.workDir, 'bin', 'gstack-distill-apply');
|
||||
const applyRes = spawnSync(applyBin, ['--proposal', '0'], {
|
||||
env: { ...fixture.env, GSTACK_STATE_ROOT: fixture.stateRoot },
|
||||
encoding: 'utf-8',
|
||||
cwd: fixture.workDir,
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(applyRes.status).toBe(0);
|
||||
|
||||
// Memory file should now contain the nugget.
|
||||
const memPath = path.join(fixture.stateRoot, 'free-text-memory.json');
|
||||
expect(fs.existsSync(memPath)).toBe(true);
|
||||
const mem = JSON.parse(fs.readFileSync(memPath, 'utf-8'));
|
||||
expect(mem.nuggets.length).toBe(1);
|
||||
|
||||
// 2. Re-fire a question whose signal_key matches the nugget. PreToolUse
|
||||
// hook should surface the nugget via additionalContext.
|
||||
const hookPath = path.join(
|
||||
fixture.workDir,
|
||||
'hosts',
|
||||
'claude',
|
||||
'hooks',
|
||||
'question-preference-hook',
|
||||
);
|
||||
const payload = {
|
||||
session_id: 'cathedral-e2e-dream',
|
||||
tool_name: 'AskUserQuestion',
|
||||
tool_use_id: 'tu-dream-1',
|
||||
tool_input: {
|
||||
questions: [
|
||||
{
|
||||
question:
|
||||
'<gstack-qid:plan-eng-review-test-gap> Add tests for this gap?',
|
||||
options: ['A) Add (recommended)', 'B) Skip'],
|
||||
},
|
||||
],
|
||||
},
|
||||
cwd: fixture.workDir,
|
||||
};
|
||||
const hookRes = spawnSync(hookPath, [], {
|
||||
env: {
|
||||
...fixture.env,
|
||||
GSTACK_STATE_ROOT: fixture.stateRoot,
|
||||
GSTACK_QUESTION_LOG_NO_DERIVE: '1',
|
||||
},
|
||||
input: JSON.stringify(payload),
|
||||
encoding: 'utf-8',
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(hookRes.status).toBe(0);
|
||||
const parsed = JSON.parse(hookRes.stdout || '{}');
|
||||
expect(parsed.hookSpecificOutput?.additionalContext).toContain('User wants every fix tested');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user