mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 09:56:57 +02:00
180 lines
8.7 KiB
TypeScript
180 lines
8.7 KiB
TypeScript
/**
|
|
* AUTO_DECIDE opt-in preserved under Conductor flags (periodic-tier, paid, real-PTY).
|
|
*
|
|
* Regression test for v1.21+ fix: the new "Tool resolution" preamble
|
|
* (scripts/resolvers/preamble/generate-ask-user-format.ts) tells the model
|
|
* to prefer mcp__*__AskUserQuestion variants and fall back to plan-file
|
|
* decisions when neither is callable. This must NOT break the legitimate
|
|
* `/plan-tune` AUTO_DECIDE path: when the user has explicitly opted into
|
|
* auto-deciding a specific question via `gstack-question-preference --write
|
|
* never-ask`, the model is supposed to honor that — it should still
|
|
* auto-pick the recommended option and emit the AUTO_DECIDE annotation
|
|
* ("Auto-decided <summary> → <option> (your preference). Change with
|
|
* /plan-tune.") instead of opening a question prompt.
|
|
*
|
|
* Periodic tier: AUTO_DECIDE behavior depends on the model adhering to
|
|
* the QUESTION_TUNING preamble injection. Non-deterministic; runs weekly
|
|
* or manually rather than gating CI.
|
|
*
|
|
* Set up:
|
|
* - tmpDir as GSTACK_HOME (isolated state, doesn't touch the user's
|
|
* real ~/.gstack)
|
|
* - question_tuning=true in the tmp config
|
|
* - preference for plan-ceo-review-mode → never-ask (source: plan-tune)
|
|
*
|
|
* Spawn:
|
|
* claude --permission-mode plan --disallowedTools AskUserQuestion
|
|
* /plan-ceo-review
|
|
*
|
|
* Expected:
|
|
* - outcome === 'auto_decided' (the AUTO_DECIDE preamble fired and the
|
|
* "Auto-decided ... (your preference)" text rendered)
|
|
*
|
|
* If outcome is 'asked', the model ignored the user's `/plan-tune`
|
|
* preference — that's a regression against the opt-in feature. If outcome
|
|
* is 'plan_ready' with no AUTO_DECIDE text, the model auto-decided BUT
|
|
* skipped the annotation (acceptable; AUTO_DECIDE annotation is good
|
|
* practice but not the load-bearing behavior).
|
|
*/
|
|
|
|
import { test, expect } from 'bun:test';
|
|
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
|
import { describeE2ETier } from './helpers/e2e-gate';
|
|
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
|
|
import { createPlanCountFixture } from './helpers/plan-count-fixture';
|
|
import { seedHermeticGstackHome } from './helpers/hermetic-env';
|
|
import * as fs from 'fs';
|
|
import * as os from 'os';
|
|
import * as path from 'path';
|
|
import { spawnSync } from 'child_process';
|
|
|
|
const describeE2E = describeE2ETier('periodic');
|
|
|
|
const ROOT = path.resolve(import.meta.dir, '..');
|
|
const PLAN = `# Draft: deterministic skill-list ordering
|
|
|
|
Users compare skill listings in scripts and reviews. Make the existing listing
|
|
path sort registered skill names deterministically before rendering them.
|
|
Keep skill membership, aliases, metadata and text/JSON output formats unchanged.
|
|
Cover mixed-case names and differing directory enumeration order with tests.
|
|
|
|
This draft is the review target, not the current branch. For this invocation,
|
|
I want only the review-mode decision; I will handle optional Office Hours and
|
|
setup separately, and run the substantive review later. No review mode has
|
|
been selected. For this mode-only handoff, use the full selected mode name in
|
|
the normal audit record's user_choice and recommended fields and in the skill's
|
|
normal mode handoff line. A menu letter alone would not identify the chosen mode.`;
|
|
|
|
describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', () => {
|
|
test('user-opted-in question still auto-decides when AskUserQuestion is --disallowedTools', async () => {
|
|
const tmpHome = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-auto-decide-')));
|
|
let fixture: ReturnType<typeof createPlanCountFixture> | undefined;
|
|
try {
|
|
fixture = createPlanCountFixture(PLAN);
|
|
seedHermeticGstackHome(tmpHome);
|
|
// 1. Bootstrap the tmp GSTACK_HOME with question_tuning=true.
|
|
const configBin = path.join(ROOT, 'bin', 'gstack-config');
|
|
const setRes = spawnSync(configBin, ['set', 'question_tuning', 'true'], {
|
|
cwd: fixture.cwd,
|
|
env: { ...process.env, GSTACK_HOME: tmpHome },
|
|
encoding: 'utf-8',
|
|
timeout: 30_000,
|
|
});
|
|
if (setRes.status !== 0) {
|
|
throw new Error(`gstack-config set failed: ${setRes.stderr || setRes.stdout}`);
|
|
}
|
|
const privacyRes = spawnSync(configBin, ['set', 'cross_project_learnings', 'false'], {
|
|
cwd: fixture.cwd, env: { ...process.env, GSTACK_HOME: tmpHome }, encoding: 'utf-8', timeout: 30_000,
|
|
});
|
|
if (privacyRes.status !== 0) throw new Error(`gstack-config privacy baseline failed: ${privacyRes.stderr || privacyRes.stdout}`);
|
|
|
|
// 2. Resolve slug for the project (uses git remote — same as the spawned
|
|
// claude would resolve). The preference file path keys on this slug.
|
|
const slugBin = path.join(ROOT, 'bin', 'gstack-slug');
|
|
const slugRes = spawnSync(slugBin, [], {
|
|
// Preference writer and observer share this attempt's standalone project.
|
|
cwd: fixture.cwd,
|
|
env: { ...process.env, GSTACK_HOME: tmpHome },
|
|
encoding: 'utf-8',
|
|
timeout: 30_000,
|
|
});
|
|
// gstack-slug emits `eval`-able shell exports like `SLUG=garrytan-gstack`.
|
|
const slug = slugRes.stdout.match(/SLUG=([^\s;]+)/)?.[1]?.replace(/['"]/g, '');
|
|
if (slugRes.status !== 0 || !slug) throw new Error('gstack-slug failed to identify the owned project');
|
|
|
|
// 3. Write the preference: plan-ceo-review-mode → never-ask. The
|
|
// 'plan-tune' source bypasses the inline-user origin gate.
|
|
const prefBin = path.join(ROOT, 'bin', 'gstack-question-preference');
|
|
const writeRes = spawnSync(
|
|
prefBin,
|
|
['--write', JSON.stringify({
|
|
question_id: 'plan-ceo-review-mode',
|
|
preference: 'never-ask',
|
|
source: 'plan-tune',
|
|
})],
|
|
{
|
|
cwd: fixture.cwd,
|
|
env: { ...process.env, GSTACK_HOME: tmpHome },
|
|
encoding: 'utf-8',
|
|
timeout: 30_000,
|
|
},
|
|
);
|
|
if (writeRes.status !== 0) {
|
|
throw new Error(`gstack-question-preference --write failed: ${writeRes.stderr || writeRes.stdout}`);
|
|
}
|
|
|
|
// Sanity: the preference file landed where we expect.
|
|
const prefFile = path.join(tmpHome, 'projects', slug, 'question-preferences.json');
|
|
if (!fs.existsSync(prefFile)) {
|
|
throw new Error(`expected preference file at ${prefFile}; not found. slug=${slug}`);
|
|
}
|
|
|
|
// 4. Run /plan-ceo-review with the Conductor flag set + isolated state.
|
|
// GSTACK_HOME=tmpHome is REQUIRED: the preference + question_tuning were
|
|
// seeded there. Without it the spawned claude reads the real ~/.gstack,
|
|
// never sees the never-ask preference, and the test silently exercises
|
|
// the wrong state root (pre-existing bug, Codex #9 / Issue 13).
|
|
// CONDUCTOR_WORKSPACE_PATH additionally proves auto-decide still WINS
|
|
// over the Conductor prose redirect (precedence: settled preference
|
|
// beats transport-avoidance).
|
|
const obs = await runPlanSkillObservation({
|
|
skillName: 'plan-ceo-review',
|
|
cwd: fixture.cwd,
|
|
// Keep the observed question within the single stored preference's scope.
|
|
// An unseeded invocation asks which plan to review, a different question.
|
|
initialPlanContent: PLAN,
|
|
inPlanMode: true,
|
|
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
|
timeoutMs: CAPTURE_LONG_MS,
|
|
// A judge calling an idle session 'waiting' is not an observed question.
|
|
// Missing decision evidence still fails at the unchanged deadline.
|
|
requireProseEvidence: true,
|
|
autoDecisionState: { stateRoot: fs.realpathSync(tmpHome), projectSlug: slug },
|
|
env: { GSTACK_HOME: tmpHome, GSTACK_STATE_ROOT: tmpHome, CONDUCTOR_WORKSPACE_PATH: fixture.cwd,
|
|
DISABLE_AUTOUPDATER: '1', CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1' },
|
|
});
|
|
|
|
// 5. Pass: 'auto_decided' (the strongest signal) or 'plan_ready' with
|
|
// no question rendered. Fail: 'asked' (model ignored the opt-in).
|
|
if (obs.outcome === 'asked') {
|
|
throw new Error(
|
|
`AUTO_DECIDE regression: the model surfaced an AskUserQuestion despite the user's never-ask preference.\n` +
|
|
`summary: ${obs.summary}\n` +
|
|
`--- evidence (last 2KB visible) ---\n${obs.evidence}`,
|
|
);
|
|
}
|
|
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
|
|
throw new Error(
|
|
`AUTO_DECIDE preserve test inconclusive: outcome=${obs.outcome}\n` +
|
|
`summary: ${obs.summary}\n` +
|
|
`--- evidence (last 2KB visible) ---\n${obs.evidence}`,
|
|
);
|
|
}
|
|
expect(['auto_decided', 'plan_ready']).toContain(obs.outcome);
|
|
} finally {
|
|
try { fixture?.cleanup(); }
|
|
finally { fs.rmSync(tmpHome, { recursive: true, force: true }); }
|
|
}
|
|
}, PTY_MS);
|
|
});
|