mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-22 12:50:50 +02:00
v1.87.5.0 perf: remove idle waits from tests and CI planning (#2897)
* v1.87.5.0 perf: remove idle waits from tests and CI planning * fix: settle split PTY redraws before routing input * docs: record final burst-safe test benchmarks * fix: keep cold-setup snapshot metadata dependency-free * fix: avoid early-reader pipe races in artifact URL parsing * fix: preserve safety matches for multiline command payloads * fix: recognize concurrent CSO publication removal * test: preload the UI design-review target before invocation * docs: record validation blocker fixes * fix: bind plan observer rejection to the invoked command * fix: count only native design decisions in the UI gate * docs: clarify UI-positive eval evidence requirements * test: recognize native UI decisions without weakening finding counts * test: decouple native UI evidence from question punctuation * test: recognize concrete native UI decisions independently of prose format * fix: retain failed eval logs under the hidden CI cache * test: await telemetry completion instead of racing disk writes
This commit is contained in:
@@ -0,0 +1,181 @@
|
||||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { buildRunManifest, collectPaidTestFiles, type PaidRunManifest, type SliceResult } from '../scripts/test-paid-shards';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
type Step = { uses?: string; run?: string; if?: string; with?: Record<string, unknown> };
|
||||
type Job = {
|
||||
needs?: string | string[];
|
||||
if?: string;
|
||||
container?: unknown;
|
||||
permissions: Record<string, string>;
|
||||
steps: Step[];
|
||||
};
|
||||
const workflows = ['evals.yml', 'evals-periodic.yml'].map(name => ({
|
||||
name,
|
||||
jobs: (Bun.YAML.parse(fs.readFileSync(path.join(ROOT, '.github/workflows', name), 'utf8')) as {
|
||||
jobs: Record<string, Job>;
|
||||
}).jobs,
|
||||
}));
|
||||
|
||||
describe('paid CI coordination stays off the eval image', () => {
|
||||
for (const { name, jobs } of workflows) {
|
||||
test(`${name}: planning is independent of image startup and has no dependency install`, () => {
|
||||
const planner = jobs['plan-slices'];
|
||||
expect(planner.needs).toBeUndefined();
|
||||
expect(planner.container).toBeUndefined();
|
||||
expect(planner.permissions).toEqual({ contents: 'read' });
|
||||
const checkout = planner.steps.find(step => step.uses?.startsWith('actions/checkout@'))!;
|
||||
expect(checkout.with?.['persist-credentials']).toBe(false);
|
||||
if (name === 'evals.yml') expect(checkout.with?.['fetch-depth']).toBe(0);
|
||||
const setup = planner.steps.find(step => step.uses?.startsWith('oven-sh/setup-bun@'))!;
|
||||
expect(setup.with?.['bun-version']).toBe('1.4.0');
|
||||
expect(JSON.stringify(planner)).not.toMatch(/secrets\.|restore-deps|bun install|bun run build/);
|
||||
expect(planner.steps.find(step => step.run?.includes('--emit-plan'))?.run).toContain('bun --no-install run');
|
||||
});
|
||||
|
||||
test(`${name}: executors still require both prerequisites and consume the image`, () => {
|
||||
const executor = jobs['eval-slices'];
|
||||
expect(executor.needs).toEqual(['build-image', 'plan-slices']);
|
||||
expect(JSON.stringify(executor.container)).toContain('needs.build-image.outputs.image-tag');
|
||||
if (name === 'evals.yml') {
|
||||
expect(executor.if).toBe("always() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success'");
|
||||
} else {
|
||||
expect(executor.if).toBeUndefined();
|
||||
}
|
||||
expect(executor.steps.some(step => step.run === 'bun run build')).toBe(true);
|
||||
expect(executor.steps.some(step => step.uses === './.github/actions/restore-deps')).toBe(true);
|
||||
});
|
||||
|
||||
test(`${name}: report still reconciles failed executors without installing dependencies`, () => {
|
||||
const report = jobs[name === 'evals.yml' ? 'slices-report' : 'report'];
|
||||
expect(report.container).toBeUndefined();
|
||||
expect(report.needs).toContain('plan-slices');
|
||||
expect(report.needs).toContain('eval-slices');
|
||||
expect(report.if).toBe("always() && needs.plan-slices.result == 'success'");
|
||||
expect(JSON.stringify(report.steps)).not.toMatch(/restore-deps|bun install/);
|
||||
expect(report.steps.find(step => step.run?.includes('--report'))?.run).toContain('bun --no-install run');
|
||||
if (name === 'evals.yml') expect(report.permissions).toEqual({ contents: 'read' });
|
||||
});
|
||||
|
||||
test(`${name}: failure logs include the hidden spool directory without uploading the rest of the cache`, () => {
|
||||
const logs = jobs['eval-slices'].steps.find(step => step.with?.name === 'paid-slice-${{ matrix.slice }}-logs');
|
||||
expect(logs?.uses).toStartWith('actions/upload-artifact@');
|
||||
expect(logs?.if).toBe('failure()');
|
||||
expect(logs?.with?.['include-hidden-files']).toBe(true);
|
||||
expect(String(logs?.with?.path).trim().split('\n')).toEqual([
|
||||
'/home/runner/.cache/gstack-paid-shard-*.log',
|
||||
'/tmp/gstack-paid-shard-*.log',
|
||||
]);
|
||||
expect(Object.values(jobs).flatMap(job => job.steps).filter(step => step.with?.['include-hidden-files']))
|
||||
.toEqual([logs]);
|
||||
});
|
||||
}
|
||||
|
||||
test('PR planning preserves the fork and Dependabot trust boundaries without the needs chain', () => {
|
||||
expect(workflows[0].jobs['plan-slices'].if).toBe(
|
||||
"github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('dependency-free CI planner and report execution', () => {
|
||||
let fixture: string;
|
||||
|
||||
beforeAll(() => {
|
||||
fixture = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-paid-coordination-'));
|
||||
fs.cpSync(path.join(ROOT, 'scripts'), path.join(fixture, 'scripts'), { recursive: true });
|
||||
fs.cpSync(path.join(ROOT, 'test/helpers'), path.join(fixture, 'test/helpers'), { recursive: true });
|
||||
for (const file of collectPaidTestFiles()) {
|
||||
fs.copyFileSync(path.join(ROOT, file), path.join(fixture, file));
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
fs.rmSync(fixture, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function run(args: string[], tier: string, env: NodeJS.ProcessEnv = {}) {
|
||||
return spawnSync(process.execPath, ['--no-install', 'run', 'scripts/test-paid-shards.ts', '--tier', tier, ...args], {
|
||||
cwd: fixture,
|
||||
env: { PATH: '', HOME: fixture, EVALS_ALL: '1', EVALS_TIER: tier, ...env },
|
||||
encoding: 'utf8',
|
||||
timeout: 10_000,
|
||||
});
|
||||
}
|
||||
|
||||
test('diff-selected host planning matches the same checkout with no installed dependencies', () => {
|
||||
const git = Bun.which('git')!;
|
||||
const gitDir = spawnSync(git, ['rev-parse', '--absolute-git-dir'], { cwd: ROOT, encoding: 'utf8', timeout: 10_000 });
|
||||
expect(gitDir.status).toBe(0);
|
||||
const manifestPath = path.join(fixture, 'diff-manifest.json');
|
||||
const env = { EVALS_ALL: '', EVALS_BASE: 'HEAD' };
|
||||
const planned = run(['--emit-plan', manifestPath, '--slices', '6'], 'gate', {
|
||||
...env,
|
||||
PATH: path.dirname(git),
|
||||
GIT_DIR: gitDir.stdout.trim(),
|
||||
GIT_WORK_TREE: ROOT,
|
||||
});
|
||||
expect(planned.status, planned.stderr).toBe(0);
|
||||
const manifest: PaidRunManifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
||||
expect(manifest).toEqual(buildRunManifest({ tier: 'gate', sliceCount: 6, evalsAll: false, env }));
|
||||
expect(manifest.evalsAll).toBe(false);
|
||||
});
|
||||
|
||||
for (const tier of ['gate', 'periodic'] as const) {
|
||||
test(`${tier}: host planner preserves the complete manifest and report fails closed`, () => {
|
||||
const sliceCount = tier === 'gate' ? 6 : 7;
|
||||
const dedicatedAutoplanSlice = tier === 'periodic';
|
||||
const reportDir = path.join(fixture, tier);
|
||||
const manifestPath = path.join(reportDir, 'manifest.json');
|
||||
const planned = run([
|
||||
'--emit-plan', manifestPath, '--slices', String(sliceCount),
|
||||
...(dedicatedAutoplanSlice ? ['--autoplan-slice'] : []),
|
||||
], tier);
|
||||
expect(planned.error).toBeUndefined();
|
||||
expect(planned.status, planned.stderr).toBe(0);
|
||||
const manifest: PaidRunManifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
||||
expect(manifest).toEqual(buildRunManifest({
|
||||
tier, sliceCount, dedicatedAutoplanSlice, evalsAll: true, env: { EVALS_ALL: '1' },
|
||||
}));
|
||||
expect(manifest.entries.filter(entry => entry.status === 'planned').length).toBeGreaterThan(0);
|
||||
expect(fs.existsSync(path.join(fixture, 'node_modules'))).toBe(false);
|
||||
for (let sliceIndex = 1; sliceIndex <= sliceCount; sliceIndex++) {
|
||||
const result: SliceResult = {
|
||||
version: 1, tier, sliceIndex, sliceCount,
|
||||
outcomes: manifest.entries.filter(entry => entry.status === 'planned' && entry.slice === sliceIndex).map(entry => ({
|
||||
files: [entry.file], status: 'passed', exitCode: 0, elapsedMs: 1, executedTests: 1, skippedTests: 0,
|
||||
...(entry.budget ? { budget: entry.budget } : {}),
|
||||
})),
|
||||
};
|
||||
fs.writeFileSync(path.join(reportDir, `slice-${sliceIndex}.json`), JSON.stringify(result));
|
||||
}
|
||||
const clean = run(['--report', reportDir], tier);
|
||||
expect(clean.status, clean.stderr).toBe(0);
|
||||
expect(clean.stdout).toContain('every planned shard accounted and passed');
|
||||
|
||||
const lastSlice = path.join(reportDir, `slice-${sliceCount}.json`);
|
||||
const saved = fs.readFileSync(lastSlice, 'utf8');
|
||||
fs.rmSync(lastSlice);
|
||||
const missing = run(['--report', reportDir], tier);
|
||||
expect(missing.status).toBe(1);
|
||||
expect(missing.stderr).toContain(`slice ${sliceCount}/${sliceCount} reported NO result`);
|
||||
|
||||
const failed: SliceResult = JSON.parse(saved);
|
||||
expect(failed.outcomes.length).toBeGreaterThan(0);
|
||||
failed.outcomes[0].status = 'failed';
|
||||
failed.outcomes[0].exitCode = 1;
|
||||
fs.writeFileSync(lastSlice, JSON.stringify(failed));
|
||||
const red = run(['--report', reportDir], tier);
|
||||
expect(red.status).toBe(1);
|
||||
expect(red.stderr).toContain(`${failed.outcomes[0].files[0]}: failed`);
|
||||
|
||||
fs.writeFileSync(manifestPath, '{');
|
||||
const corrupt = run(['--report', reportDir], tier);
|
||||
expect(corrupt.status).toBe(1);
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -20,6 +20,7 @@ async function observe(frames:string[],verdict:'waiting'|'working',required?:boo
|
||||
launchClaudePty:async()=>({send:()=>{},mark:()=>0,exited:()=>false,visibleSince:current,rawOutput:current,currentScreen:async()=>current(),hermeticConfigDir:null,close:async()=>{closed++;}}),
|
||||
createPlanCountSnapshotWriter:()=>()=>({}),logPtySnapshot:()=>{},
|
||||
isProseAUQVisible:predicates.isProseAUQVisible,isPlanReadyVisible:predicates.isPlanReadyVisible,
|
||||
isUnknownSlashCommandVisible:predicates.isUnknownSlashCommandVisible,
|
||||
isScopeGateQuestionVisible:predicates.isScopeGateQuestionVisible,isScopeGateAutoSelectVisible:predicates.isScopeGateAutoSelectVisible,
|
||||
classifyVisible:predicates.classifyVisible,extractPlanFilePath:predicates.extractPlanFilePath,findNativeAutoDecision:()=>null,
|
||||
judgePtyState:()=>{judged++;return {state:verdict,reasoning:'synthetic fixed verdict'};},
|
||||
|
||||
@@ -117,7 +117,7 @@ describe('CSO Git metadata hardening',()=>{
|
||||
test.skipIf(process.platform==='win32')('does not follow a worktree .git pointer swapped between lstat and open',async()=>{
|
||||
const {root,repo,runDir}=fixture(),gitDir=path.join(root,'git-data'),marker=path.join(repo,'.git'),oversized=path.join(root,'oversized-git-pointer');
|
||||
fs.renameSync(marker,gitDir);fs.writeFileSync(marker,'gitdir: ../git-data\n');fs.writeFileSync(oversized,'gitdir: '+'.'.repeat(16*1024));
|
||||
const race=replaceWithSymlinkAfterLstat(marker,oversized,2);
|
||||
const race=replaceWithSymlinkAfterLstat(marker,oversized);
|
||||
try{await expect(capture(repo,runDir)).rejects.toMatchObject({code:'SNAPSHOT_RACE'});}finally{race.patched.mockRestore();}
|
||||
expect(race.wasSwapped()).toBe(true);
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { afterAll, afterEach, describe, expect, spyOn, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';import * as os from 'node:os';import * as path from 'node:path';import { spawn, spawnSync } from 'node:child_process';
|
||||
import { assertSnapshot, capture } from '../lib/cso/snapshot';import { CsoError } from '../lib/cso/contracts';import { runProcess, sanitizeForJson, sanitizeHelperForJson } from '../lib/cso/process';import { discardAtomicNoReplaceTemp, finalizeReplayTemporary, loadReport, newRun, privateRoot, readJson, retention, saveReport, secureDirectory, stateRoot, withLock, writeHelperJson, writeJson } from '../lib/cso/state';
|
||||
import { assertSnapshot, capture } from '../lib/cso/snapshot';import { CsoError } from '../lib/cso/contracts';import { runProcess, sanitizeForJson, sanitizeHelperForJson } from '../lib/cso/process';import { discardAtomicNoReplaceTemp, finalizeReplayTemporary, loadReport, newRun, privateRoot, readJson, recoverAtomicNoReplaceJson, retention, saveReport, secureDirectory, stateRoot, withLock, writeHelperJson, writeJson } from '../lib/cso/state';
|
||||
const roots:string[]=[];const tmp=()=>{const p=fs.mkdtempSync(path.join(os.tmpdir(),'cso-snapshot-'));roots.push(p);return p;};afterEach(()=>{for(const p of roots.splice(0))fs.rmSync(p,{recursive:true,force:true});});
|
||||
const originalState=process.env.GSTACK_HOME,state=fs.mkdtempSync(path.join(os.tmpdir(),'cso-state-'));process.env.GSTACK_HOME=state;afterAll(()=>{if(originalState===undefined)delete process.env.GSTACK_HOME;else process.env.GSTACK_HOME=originalState;fs.rmSync(state,{recursive:true,force:true});});
|
||||
function git(repo:string,...args:string[]){const r=spawnSync('/usr/bin/git',['-C',repo,...args],{encoding:'utf8',env:{PATH:'/usr/bin:/bin',HOME:repo},timeout:30_000});if(r.status)throw new Error(r.stderr);return r.stdout;}
|
||||
@@ -167,6 +167,23 @@ describe('private state and process output',()=>{
|
||||
test('a synchronous exact-release failure is attempted only once',()=>{const dir=tmp(),lstat=fs.lstatSync;let observed=0,reader:any;try{expect(()=>withLock(dir,()=>{const leases=path.join(dir,'.mutation-lock-leases'),names=fs.readdirSync(leases),candidate=path.join(leases,names.find(name=>name.endsWith('.json'))!),active=path.join(leases,names.find(name=>name.includes('.active.'))!),value=fs.readFileSync(active);fs.unlinkSync(active);fs.writeFileSync(active,value,{mode:0o600,flag:'wx'});reader=spyOn(fs,'lstatSync').mockImplementation(((file:any,options?:any)=>{if(String(file)===candidate)observed++;return options===undefined?lstat(file):lstat(file,options);}) as typeof fs.lstatSync);})).toThrow('active phase changed before cleanup');expect(observed).toBe(1);}finally{reader?.mockRestore();}});
|
||||
test('exact release rejects active or decision inode substitution and overrides callback success or failure',()=>{for(const phase of ['active','decision'] as const){const dir=tmp();let replacement='',parked='';expect(()=>withLock(dir,()=>{const leases=path.join(dir,'.mutation-lock-leases'),name=fs.readdirSync(leases).find(value=>phase==='active'?value.includes('.active.'):value.endsWith('.decision'))!,lease=path.join(leases,name),value=fs.readFileSync(lease);parked=`${lease}.replaced`;fs.renameSync(lease,parked);fs.writeFileSync(lease,value,{mode:0o600,flag:'wx'});replacement=lease;if(phase==='active')throw new Error('callback failed');return 1;})).toThrow(/changed before (cleanup|exact release)/);expect(fs.existsSync(replacement)).toBe(true);expect(fs.existsSync(parked)).toBe(true);expect(fs.statSync(replacement).ino).not.toBe(fs.statSync(parked).ino);fs.rmSync(dir,{recursive:true,force:true});}});
|
||||
test('concurrent stale-lock recovery never admits overlapping report writers',async()=>{const dir=tmp(),lock=path.join(dir,'.mutation-lock'),script=path.join(dir,'racer.ts');fs.mkdirSync(lock);fs.writeFileSync(path.join(lock,'owner.json'),JSON.stringify({pid:2147483647,token:'stale',createdAt:0,expiresAt:0}));fs.writeFileSync(script,`import fs from 'node:fs';import path from 'node:path';import {withLock} from ${JSON.stringify(path.resolve(import.meta.dir,'../lib/cso/state.ts'))};const dir=process.env.RACE_DIR!;try{await withLock(dir,async()=>{const active=path.join(dir,'active');try{fs.writeFileSync(active,String(process.pid),{flag:'wx'});}catch{fs.appendFileSync(path.join(dir,'overlap'),'yes\\n');}await Bun.sleep(50);try{if(fs.readFileSync(active,'utf8')===String(process.pid))fs.unlinkSync(active);}catch{}});}catch{}`);const children=Array.from({length:8},()=>spawn(process.execPath,[script],{env:{...process.env,RACE_DIR:dir},stdio:'ignore'}));await Promise.all(children.map(child=>new Promise<void>(resolve=>child.on('close',()=>resolve()))));expect(fs.existsSync(path.join(dir,'overlap'))).toBe(false);});
|
||||
test.each(['removed','replaced'] as const)('publication %s during candidate enumeration stays fail-closed',(change)=>{
|
||||
const dir=tmp(),target=path.join(dir,'artifact.json'),temporary=`${target}.tmp.2147483647.cafebabe`,replacement=path.join(dir,'replacement.json');
|
||||
fs.writeFileSync(target,'{"value":"original"}\n',{mode:0o600});fs.linkSync(target,temporary);
|
||||
fs.writeFileSync(replacement,'{"value":"replacement"}\n',{mode:0o600});
|
||||
const readdir=fs.readdirSync;let changed=false;
|
||||
const reader=spyOn(fs,'readdirSync').mockImplementation(((directory:any,options?:any)=>{
|
||||
const entries=options===undefined?readdir(directory):readdir(directory,options);
|
||||
if(String(directory)===dir&&!changed){changed=true;fs.unlinkSync(temporary);if(change==='removed')fs.unlinkSync(target);else fs.renameSync(replacement,target);}
|
||||
return entries;
|
||||
}) as typeof fs.readdirSync);
|
||||
try{
|
||||
let caught:unknown;try{recoverAtomicNoReplaceJson(target,{label:'Test publication',maxBytes:4096});}catch(error){caught=error;}
|
||||
expect(changed).toBe(true);
|
||||
expect(caught).toMatchObject(change==='removed'?{name:'AtomicPublicationTransition',code:'SNAPSHOT_RACE'}:{code:'UNSAFE_PATH'});
|
||||
if(change==='removed')expect(fs.existsSync(target)).toBe(false);else expect(fs.readFileSync(target,'utf8')).toBe('{"value":"replacement"}\n');
|
||||
}finally{reader.mockRestore();}
|
||||
});
|
||||
test('concurrent recovery of one dead hard-link publication has a winner and no raw race errors',async()=>{
|
||||
const dir=tmp(),barrier=path.join(dir,'barrier'),script=path.join(dir,'recovery-racer.ts');fs.mkdirSync(barrier);expect(withLock(dir,()=>1)).toBe(1);
|
||||
const leases=path.join(dir,'.mutation-lock-leases'),token='c'.repeat(32),lease=path.join(leases,`${token}.json`),temporary=`${lease}.tmp.2147483647.deadbeef`;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { capturePlanCountQuestion, nativePlanCallFingerprint } from './helpers/claude-pty-runner';
|
||||
import { pickDesignCountOutsideVoices } from './helpers/design-count-outside';
|
||||
import { isDesignCountFirstReview } from './helpers/design-count-review';
|
||||
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
|
||||
|
||||
const packet: NativePlanQuestionCall = {
|
||||
@@ -101,4 +102,34 @@ describe('Design count fixture outside-review choice', () => {
|
||||
expect(pickDesignCountOutsideVoices(outside, { ...outside, nativeCall: call })).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
test('the binary outside-voices variant still selects only its explicit opt-out', () => {
|
||||
for (const id of ['outside-voices-design', 'plan-design-review-outside-voices']) {
|
||||
const call = structuredClone(packet);
|
||||
call.questions = [call.questions[1]!];
|
||||
const q = call.questions[0]!;
|
||||
q.question = `D3 — Want outside voices before the detailed review? <gstack-qid:${id}>`;
|
||||
q.options[0]!.label = 'Yes, run outside voices (recommended)';
|
||||
const native = nativePlanCallFingerprint(call, 0, true);
|
||||
expect(pickDesignCountOutsideVoices(native, native)).toBe(2);
|
||||
const visible = capturePlanCountQuestion(screen(0, call), new Set(), 0, true)!;
|
||||
expect(pickDesignCountOutsideVoices(visible, visible)).toBe(2);
|
||||
q.options[1]!.label = 'No, leave the design defect unfixed';
|
||||
const product = nativePlanCallFingerprint(call, 0, true);
|
||||
expect(pickDesignCountOutsideVoices(product, product)).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
test('an outside-review opt-in with a design-review-prefixed ID cannot start a finding', () => {
|
||||
const call = structuredClone(packet);
|
||||
call.questions = [call.questions[1]!];
|
||||
const q = call.questions[0]!;
|
||||
q.question = 'D3 — Want outside voices before the detailed review?\n' +
|
||||
'Project/branch/task: main branch; design review of PLAN.md before the 7 passes. ' +
|
||||
'<gstack-qid:plan-design-review-outside-voices>';
|
||||
call.answered = true;
|
||||
call.unansweredQuestionIndices = [];
|
||||
call.answers = { [q.question]: q.options[0]!.label };
|
||||
expect(isDesignCountFirstReview(nativePlanCallFingerprint(call, 0, true))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import { nativePlanCallFingerprint } from './helpers/claude-pty-runner';
|
||||
import { isDesignUIScopeReview } from './helpers/design-ui-scope';
|
||||
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
|
||||
import captured from './fixtures/plan-design-ui-scope.json';
|
||||
import { E2E_TOUCHFILES } from './helpers/touchfiles-data';
|
||||
|
||||
const calls = captured.calls as NativePlanQuestionCall[];
|
||||
const fingerprint = (call: NativePlanQuestionCall) => nativePlanCallFingerprint(call, 0, true);
|
||||
const recovered = captured.additionalQuestionCaptures[0]!;
|
||||
const recoveredCall: NativePlanQuestionCall = {
|
||||
sessionId: 'ui-scope-replay',
|
||||
toolUseId: 'recovered-question',
|
||||
questions: [recovered.question],
|
||||
answered: true,
|
||||
failed: false,
|
||||
answers: { [recovered.question.question]: recovered.answer },
|
||||
unansweredQuestionIndices: [],
|
||||
};
|
||||
|
||||
test('the captured untagged dashboard decision proves UI review, but its setup questions do not', () => {
|
||||
expect(calls.map(call => isDesignUIScopeReview(fingerprint(call)))).toEqual([false, false, false, true]);
|
||||
expect(calls[3]!.questions[0]!.question).not.toContain('<gstack-qid:');
|
||||
});
|
||||
|
||||
test('the second captured review distinguishes setup from all ten native design decisions', () => {
|
||||
const replay = captured.additionalCaptures[0]!.calls as NativePlanQuestionCall[];
|
||||
expect(replay.map(call => isDesignUIScopeReview(fingerprint(call))))
|
||||
.toEqual([false, false, ...Array(10).fill(true)]);
|
||||
});
|
||||
|
||||
test('issue and pass separators do not change native design evidence', () => {
|
||||
for (const issueSeparator of [':', ' —', ' –', ' -']) {
|
||||
for (const passSeparator of [',', ';', ' —', ' –', ' -', ':', ' (']) {
|
||||
const call = structuredClone(calls[3]!);
|
||||
const q = call.questions[0]!;
|
||||
q.question = q.question.replace('Issue 1:', `Issue 1${issueSeparator}`)
|
||||
.replace(', Pass 1', `${passSeparator} Pass 1`);
|
||||
call.answers = { [q.question]: q.options[0]!.label };
|
||||
expect(isDesignUIScopeReview(fingerprint(call)), `${issueSeparator} / ${passSeparator}`).toBe(true);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('a recovered UI decision replays with fixture-owned metadata without filename, pass, or leading question verb', () => {
|
||||
expect(isDesignUIScopeReview(fingerprint(recoveredCall))).toBe(true);
|
||||
const call = structuredClone(recoveredCall);
|
||||
const q = call.questions[0]!;
|
||||
q.question = q.question.replace(/^Project\/branch\/task:[^\n]*\n/m, '');
|
||||
call.answers = { [q.question]: q.options[0]!.label };
|
||||
expect(isDesignUIScopeReview(fingerprint(call))).toBe(true);
|
||||
});
|
||||
|
||||
test('choice identity does not depend on punctuation after the issue letter', () => {
|
||||
for (const separator of ['', ':', '.', ')', '—', '–', '-']) {
|
||||
const call = structuredClone(recoveredCall);
|
||||
const q = call.questions[0]!;
|
||||
for (const option of q.options) option.label = option.label.replace(/^6([A-Z]) /, `6$1${separator} `);
|
||||
call.answers = { [q.question]: q.options[0]!.label };
|
||||
expect(isDesignUIScopeReview(fingerprint(call)), separator).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('numbered UI language still requires concrete design choices rather than workflow or another target', () => {
|
||||
for (const mutate of [
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.header = 'Scope'; },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.question = q.question.replace('dashboard plan on main', 'OTHER.md dashboard plan on main'); },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.question = q.question.replace('D10 — Issue 6:', 'Example:'); },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.question = q.question.replace('Undo toast?', 'Undo toast.'); },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.options[0]!.label = '7A Immediate + Undo toast'; },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.options = [{ label: '6A Yes' }, { label: '6B No' }]; },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.options[0]!.label = '6A Review the modal later'; },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.question = q.question.replace("'Mark all as read' — confirmation modal (as planned) or immediate action with an Undo toast?", 'Which modal should the outside reviewers discuss?'); },
|
||||
]) {
|
||||
const call = structuredClone(recoveredCall);
|
||||
const q = call.questions[0]!;
|
||||
mutate(q);
|
||||
call.answers = { [q.question]: q.options[0]!.label };
|
||||
expect(isDesignUIScopeReview(fingerprint(call))).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('native ownership and complete offered answers are required for UI evidence', () => {
|
||||
for (const mutate of [
|
||||
(call: NativePlanQuestionCall) => { call.answered = false; },
|
||||
(call: NativePlanQuestionCall) => { call.failed = true; },
|
||||
(call: NativePlanQuestionCall) => { call.unansweredQuestionIndices = [0]; },
|
||||
(call: NativePlanQuestionCall) => { call.answers = {}; },
|
||||
(call: NativePlanQuestionCall) => { call.answers = { [call.questions[0]!.question]: 'Unrelated answer' }; },
|
||||
(call: NativePlanQuestionCall) => { call.questions[0]!.multiSelect = true; },
|
||||
(call: NativePlanQuestionCall) => { call.questions[0]!.options = call.questions[0]!.options.slice(0, 1); },
|
||||
]) {
|
||||
const call = structuredClone(calls[3]!);
|
||||
mutate(call);
|
||||
expect(isDesignUIScopeReview(fingerprint(call))).toBe(false);
|
||||
}
|
||||
expect(isDesignUIScopeReview({ ...fingerprint(calls[3]!), signature: 'another-session:another-call' })).toBe(false);
|
||||
});
|
||||
|
||||
test('issue-like framing cannot promote setup, examples, another plan, or mismatched choices', () => {
|
||||
for (const mutate of [
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.header = 'Outside voices'; },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.question = 'Example:\n' + q.question; },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.question = q.question.replace('PLAN.md', 'OTHER.md'); },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.question = q.question.replace('Pass 1', 'before Pass 1'); },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.question = q.question.replace("Which panel is primary, and what's the order?", 'Which review scope should cover the panels?'); },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.options[1]!.label = '2B: Another issue'; },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.options[1]!.label = '1B: Run outside reviewers'; },
|
||||
(q: NativePlanQuestionCall['questions'][number]) => { q.options[1]!.label = q.options[0]!.label; },
|
||||
]) {
|
||||
const call = structuredClone(calls[3]!);
|
||||
const q = call.questions[0]!;
|
||||
mutate(q);
|
||||
call.answers = { [q.question]: q.options[0]!.label };
|
||||
expect(isDesignUIScopeReview(fingerprint(call))).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('the UI gate owns its classifier, captured evidence, and regression tests', () => {
|
||||
for (const file of ['test/helpers/design-ui-scope.ts', 'test/design-ui-scope.test.ts', 'test/fixtures/plan-design-ui-scope.json']) {
|
||||
expect(Object.entries(E2E_TOUCHFILES).filter(([, files]) => files.includes(file)).map(([owner]) => owner))
|
||||
.toEqual(['plan-design-with-ui-scope']);
|
||||
}
|
||||
});
|
||||
@@ -136,6 +136,7 @@ async function mockedObservation(frames: string[], verdict: 'waiting' | 'working
|
||||
close: async () => { closed++; } }),
|
||||
createPlanCountSnapshotWriter: () => () => ({}), logPtySnapshot: () => {},
|
||||
isProseAUQVisible: predicates.isProseAUQVisible, isPlanReadyVisible: predicates.isPlanReadyVisible,
|
||||
isUnknownSlashCommandVisible: predicates.isUnknownSlashCommandVisible,
|
||||
isScopeGateQuestionVisible: predicates.isScopeGateQuestionVisible,
|
||||
isScopeGateAutoSelectVisible: predicates.isScopeGateAutoSelectVisible,
|
||||
classifyVisible, extractPlanFilePath, findNativeAutoDecision: () => null,
|
||||
|
||||
@@ -61,9 +61,9 @@ describe('evals.yml sliced-lane wiring (post-matrix)', () => {
|
||||
});
|
||||
|
||||
test('planner, executors, and report all run tier=gate on the shared runner', () => {
|
||||
expect(evalsYml).toMatch(/EVALS_TIER=gate bun run scripts\/test-paid-shards\.ts --tier gate --emit-plan/);
|
||||
expect(evalsYml).toMatch(/EVALS_TIER=gate bun --no-install run scripts\/test-paid-shards\.ts --tier gate --emit-plan/);
|
||||
expect(evalsYml).toMatch(/EVALS_TIER=gate bun run scripts\/test-paid-shards\.ts --tier gate --plan .* --slice /);
|
||||
expect(evalsYml).toMatch(/EVALS_TIER=gate bun run scripts\/test-paid-shards\.ts --tier gate --report /);
|
||||
expect(evalsYml).toMatch(/EVALS_TIER=gate bun --no-install run scripts\/test-paid-shards\.ts --tier gate --report /);
|
||||
});
|
||||
|
||||
test('executor matrix slice list matches the planner --slices count', () => {
|
||||
@@ -124,9 +124,9 @@ describe('evals.yml sliced-lane wiring (post-matrix)', () => {
|
||||
|
||||
describe('evals-periodic.yml sliced-lane wiring', () => {
|
||||
test('planner/executor/report tier=periodic and slice counts agree', () => {
|
||||
expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --emit-plan/);
|
||||
expect(periodicYml).toMatch(/EVALS_TIER=periodic bun --no-install run scripts\/test-paid-shards\.ts --tier periodic --emit-plan/);
|
||||
expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --plan .* --slice /);
|
||||
expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --report /);
|
||||
expect(periodicYml).toMatch(/EVALS_TIER=periodic bun --no-install run scripts\/test-paid-shards\.ts --tier periodic --report /);
|
||||
const planned = plannedSlices(periodicYml);
|
||||
const matrices = matrixSlices(periodicYml);
|
||||
expect(planned).toHaveLength(1);
|
||||
|
||||
+540
@@ -0,0 +1,540 @@
|
||||
{
|
||||
"source": {
|
||||
"commit": "980abc21",
|
||||
"workflowRun": 35533668862,
|
||||
"attempt": 2,
|
||||
"retainedRange": "focus through first design issue"
|
||||
},
|
||||
"calls": [
|
||||
{
|
||||
"sessionId": "75bfb2cc-901f-445b-84dc-6b1c483fe9eb",
|
||||
"toolUseId": "toolu_01Sh1T4EU8cmj9rokTaS8pKR",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 — Review all 7 design dimensions, or focus?\nProject/branch/task: `main`, reviewing PLAN.md (User Dashboard Page).\nELI10: I've rated the plan 4/10 on design completeness. The biggest gaps are: no content hierarchy (which panel wins after login?), undesigned first-time-user empty state (a new user lands on three empty boxes), and no tokens (fonts, spacing, colors are all TBD with no DESIGN.md). Next I'll generate visual mockups with the gstack designer, then walk the 7 review passes (Interaction States, AI Slop Risk, Information Architecture, User Journey, Responsive, Accessibility, Specificity), asking you a decision per gap.\nStakes if we pick wrong: a narrow focus is faster but leaves the skipped dimensions at their current ~3/10 and the implementer guesses; the full pass takes longer but every gap gets an explicit decision.\nRecommendation: A because with no DESIGN.md and a greenfield page, every dimension is under-specified; skipping any leaves guesswork in the plan.\nCompleteness: A=10/10, B=6/10, C=4/10\nNet: full rigor now vs. speed now and rework later.",
|
||||
"header": "Focus",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "All 7 dimensions (recommended)",
|
||||
"description": "✅ Every gap gets a decision; the plan becomes buildable without designer guesses (human: ~half a day / CC: ~20 min). ✅ Catches cross-cutting issues like partial failure and toast/mobile collisions. ❌ Longer session with more decision prompts to answer."
|
||||
},
|
||||
{
|
||||
"label": "Hierarchy + states + journey only",
|
||||
"description": "✅ Hits the three highest-impact gaps: what's first, empty/error states, post-login arc. ✅ Roughly half the questions of the full pass. ❌ Responsive, a11y, and token specificity stay unspecified and get improvised in code."
|
||||
},
|
||||
{
|
||||
"label": "Mockups + hierarchy only",
|
||||
"description": "✅ Fastest route to a visual direction you can react to. ✅ Good if a DESIGN.md is coming separately via /design-consultation. ❌ States, journey, a11y, and toast/modal patterns remain at 3/10; implementer decides them."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 — Review all 7 design dimensions, or focus?\nProject/branch/task: `main`, reviewing PLAN.md (User Dashboard Page).\nELI10: I've rated the plan 4/10 on design completeness. The biggest gaps are: no content hierarchy (which panel wins after login?), undesigned first-time-user empty state (a new user lands on three empty boxes), and no tokens (fonts, spacing, colors are all TBD with no DESIGN.md). Next I'll generate visual mockups with the gstack designer, then walk the 7 review passes (Interaction States, AI Slop Risk, Information Architecture, User Journey, Responsive, Accessibility, Specificity), asking you a decision per gap.\nStakes if we pick wrong: a narrow focus is faster but leaves the skipped dimensions at their current ~3/10 and the implementer guesses; the full pass takes longer but every gap gets an explicit decision.\nRecommendation: A because with no DESIGN.md and a greenfield page, every dimension is under-specified; skipping any leaves guesswork in the plan.\nCompleteness: A=10/10, B=6/10, C=4/10\nNet: full rigor now vs. speed now and rework later.": "All 7 dimensions (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:04:52.389Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "75bfb2cc-901f-445b-84dc-6b1c483fe9eb",
|
||||
"toolUseId": "toolu_017khLgsWGurNVwJXUyRcR3G",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 — Run outside design voices before the detailed review?\nProject/branch/task: `main`, reviewing PLAN.md (User Dashboard Page), 7-dimension pass approved.\nELI10: Before I do my own 7-pass review, I can pull in two independent opinions: Codex (OpenAI's model) checks the plan against a set of design hard rules and litmus checks (brand clarity, one visual anchor, cards earning their existence, etc.), and a fresh Claude subagent with no context does a completeness review. Where they agree with me, findings get pre-loaded; where they disagree, you see both sides.\nStakes if we pick wrong: skipping means single-model blind spots go unchallenged; running costs a few minutes and, if Codex isn't installed or authed, only the Claude subagent contributes (tagged single-model).\nRecommendation: A because this plan is greenfield with no DESIGN.md, so independent eyes on hierarchy and slop risk are cheap insurance.\nNote: options differ in kind, not coverage — no completeness score.\nNet: a few minutes of wall time for cross-model disagreement you can actually see.",
|
||||
"header": "Outside voices",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Yes, run outside voices (recommended)",
|
||||
"description": "✅ Codex litmus scorecard plus an independent Claude completeness review, synthesized into the passes. ✅ Disagreements surface explicitly instead of hiding in one model's taste. ❌ Adds a few minutes; Codex may be unavailable in this environment."
|
||||
},
|
||||
{
|
||||
"label": "No, proceed without",
|
||||
"description": "✅ Straight into the 7-pass review, fastest path to decisions. ✅ No dependency on Codex install or auth state. ❌ Single-model review; slop and hierarchy calls rest on one opinion."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 — Run outside design voices before the detailed review?\nProject/branch/task: `main`, reviewing PLAN.md (User Dashboard Page), 7-dimension pass approved.\nELI10: Before I do my own 7-pass review, I can pull in two independent opinions: Codex (OpenAI's model) checks the plan against a set of design hard rules and litmus checks (brand clarity, one visual anchor, cards earning their existence, etc.), and a fresh Claude subagent with no context does a completeness review. Where they agree with me, findings get pre-loaded; where they disagree, you see both sides.\nStakes if we pick wrong: skipping means single-model blind spots go unchallenged; running costs a few minutes and, if Codex isn't installed or authed, only the Claude subagent contributes (tagged single-model).\nRecommendation: A because this plan is greenfield with no DESIGN.md, so independent eyes on hierarchy and slop risk are cheap insurance.\nNote: options differ in kind, not coverage — no completeness score.\nNet: a few minutes of wall time for cross-model disagreement you can actually see.": "No, proceed without"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:05:46.775Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "75bfb2cc-901f-445b-84dc-6b1c483fe9eb",
|
||||
"toolUseId": "toolu_01EtwDxVuEn7oDvi26nmncsC",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 — Enable cross-project learnings search?\nProject/branch/task: `main`, reviewing PLAN.md; about to start Pass 1 (Information Architecture).\nELI10: gstack saves small \"learnings\" (pitfalls, patterns) from each session. It can search learnings from your other projects on this machine to find ones that apply here. This stays local; nothing leaves your machine. Useful for solo developers; skip if you work across client codebases where cross-contamination is a concern.\nStakes if we pick wrong: enabling on a multi-client machine could surface one client's patterns while working on another; disabling just means each project learns on its own. Both are one-line config flips.\nRecommendation: A because this is a one-time local setting and the fixture repo has no learnings yet, so the wider net is the only way to find any.\nNote: options differ in kind, not coverage — no completeness score.\nNet: broader recall vs. strict per-project isolation.",
|
||||
"header": "Learnings",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Enable cross-project (recommended)",
|
||||
"description": "✅ Past pitfalls from any project on this machine surface in reviews here. ✅ Fully local; a single gstack-config flag you can flip back anytime. ❌ On shared or multi-client machines, patterns from unrelated codebases may show up."
|
||||
},
|
||||
{
|
||||
"label": "Keep project-scoped only",
|
||||
"description": "✅ Strict isolation; nothing from other codebases influences this review. ✅ Safer default when you work on multiple client projects. ❌ This fresh repo has zero learnings, so the search returns nothing for now."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 — Enable cross-project learnings search?\nProject/branch/task: `main`, reviewing PLAN.md; about to start Pass 1 (Information Architecture).\nELI10: gstack saves small \"learnings\" (pitfalls, patterns) from each session. It can search learnings from your other projects on this machine to find ones that apply here. This stays local; nothing leaves your machine. Useful for solo developers; skip if you work across client codebases where cross-contamination is a concern.\nStakes if we pick wrong: enabling on a multi-client machine could surface one client's patterns while working on another; disabling just means each project learns on its own. Both are one-line config flips.\nRecommendation: A because this is a one-time local setting and the fixture repo has no learnings yet, so the wider net is the only way to find any.\nNote: options differ in kind, not coverage — no completeness score.\nNet: broader recall vs. strict per-project isolation.": "Enable cross-project (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:06:16.432Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "75bfb2cc-901f-445b-84dc-6b1c483fe9eb",
|
||||
"toolUseId": "toolu_01PK7hitKrKsZrY1izfnAkrs",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D5 — Issue 1: Which panel is primary, and what's the order?\nProject/branch/task: `main`, PLAN.md User Dashboard, Pass 1 Information Architecture (3/10).\nELI10: The plan lists three panels with no order. Whatever the implementer puts top-left on desktop and first in the mobile stack becomes the thing every user sees first after login. Right now that's an accident. Quick Actions is the only panel that always has content (actions exist even with zero data), so it's the safest thing to anchor above the fold on a phone.\nStakes if we pick wrong: a first-time user's first screen after login is either an empty feed or an empty inbox, and a returning user hunts for the button they came to press.\nRecommendation: 1A because actions are never empty, activity is the scan target, and notifications are context; this also matches conventional dashboard wayfinding.\nCompleteness: 1A=10/10, 1B=8/10, 1C=8/10 (all define an order; A also handles the empty-first-login case)\nNet: always-useful anchor at top vs. feed-first convention vs. inbox-first urgency.",
|
||||
"header": "Hierarchy",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "1A: Actions row → Activity primary → Notifications side (recommended)",
|
||||
"description": "✅ Quick Actions as a compact row under the greeting is never empty, so the fold is never blank on first login. ✅ Activity as the wide primary column, Notifications as a narrower side column on lg, stacked second on sm/md. ❌ Unread notifications sit below actions and to the side; urgency relies on the badge count."
|
||||
},
|
||||
{
|
||||
"label": "1B: Activity primary, Notifications side, Actions in side rail",
|
||||
"description": "✅ Classic feed-first dashboard; returning users get scannable content immediately. ✅ Actions and notifications share a side rail, keeping the main column pure. ❌ First-time user's top-left region is an empty feed; actions buried in the rail on mobile."
|
||||
},
|
||||
{
|
||||
"label": "1C: Notifications primary, Activity secondary, Actions row",
|
||||
"description": "✅ Unread items are the most actionable content, so they lead. ✅ Read-all flow becomes the hero interaction. ❌ A quiet user (zero unread) sees an empty inbox first; activity feed loses its role as the scan target."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D5 — Issue 1: Which panel is primary, and what's the order?\nProject/branch/task: `main`, PLAN.md User Dashboard, Pass 1 Information Architecture (3/10).\nELI10: The plan lists three panels with no order. Whatever the implementer puts top-left on desktop and first in the mobile stack becomes the thing every user sees first after login. Right now that's an accident. Quick Actions is the only panel that always has content (actions exist even with zero data), so it's the safest thing to anchor above the fold on a phone.\nStakes if we pick wrong: a first-time user's first screen after login is either an empty feed or an empty inbox, and a returning user hunts for the button they came to press.\nRecommendation: 1A because actions are never empty, activity is the scan target, and notifications are context; this also matches conventional dashboard wayfinding.\nCompleteness: 1A=10/10, 1B=8/10, 1C=8/10 (all define an order; A also handles the empty-first-login case)\nNet: always-useful anchor at top vs. feed-first convention vs. inbox-first urgency.": "1A: Actions row → Activity primary → Notifications side (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:06:45.808Z"
|
||||
}
|
||||
],
|
||||
"additionalCaptures": [
|
||||
{
|
||||
"source": {
|
||||
"commit": "f1928bee",
|
||||
"workflowRun": 35535549200,
|
||||
"attempt": 2,
|
||||
"retainedRange": "outside voices through tenth design issue (available job-log tail)"
|
||||
},
|
||||
"calls": [
|
||||
{
|
||||
"sessionId": "03736bff-d38b-4f3e-a849-84f17064a75b",
|
||||
"toolUseId": "toolu_01VKZeF87WqJiAedWw2JiLFF",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 — Run outside design voices before the detailed review?\nProject/branch/task: main, reviewing PLAN.md (User Dashboard Page); mockups unavailable (no OpenAI key), so this is the remaining independent-perspective step.\nELI10: Before I do the 7-pass review myself, I can pull in two independent reviewers: Codex (OpenAI's CLI) grades the plan against a set of design hard rules and litmus checks, and a fresh Claude subagent that has seen none of my notes does an independent completeness review. Their findings get folded into the passes so I'm not the only opinion in the room.\nStakes if we pick wrong: skip it and the review is single-voice; run it and it costs a couple of minutes, and Codex may be unavailable in this environment (then only the Claude subagent runs, tagged single-model).\nRecommendation: A because greenfield UI with no design system benefits most from a second and third set of eyes on hierarchy and missing states.\nNote: options differ in kind, not coverage — no completeness score.",
|
||||
"header": "Outside voices",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Yes, run outside voices (recommended)",
|
||||
"description": "✅ Independent hard-rejection and litmus checks catch generic-SaaS-card-grid drift before code exists\n✅ Findings pre-load into the 7 passes so we skip discovery and go straight to fixes (human: ~1 hr / CC: ~3 min)\n❌ Codex may be missing or unauthenticated here; then coverage is Claude-subagent only"
|
||||
},
|
||||
{
|
||||
"label": "No, proceed without",
|
||||
"description": "✅ Faster path straight into the 7 design passes and per-issue approvals\n✅ No dependency on external CLI availability\n❌ Single reviewer; blind spots in my own taste go unchallenged"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 — Run outside design voices before the detailed review?\nProject/branch/task: main, reviewing PLAN.md (User Dashboard Page); mockups unavailable (no OpenAI key), so this is the remaining independent-perspective step.\nELI10: Before I do the 7-pass review myself, I can pull in two independent reviewers: Codex (OpenAI's CLI) grades the plan against a set of design hard rules and litmus checks, and a fresh Claude subagent that has seen none of my notes does an independent completeness review. Their findings get folded into the passes so I'm not the only opinion in the room.\nStakes if we pick wrong: skip it and the review is single-voice; run it and it costs a couple of minutes, and Codex may be unavailable in this environment (then only the Claude subagent runs, tagged single-model).\nRecommendation: A because greenfield UI with no design system benefits most from a second and third set of eyes on hierarchy and missing states.\nNote: options differ in kind, not coverage — no completeness score.": "No, proceed without"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:40:29.058Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "03736bff-d38b-4f3e-a849-84f17064a75b",
|
||||
"toolUseId": "toolu_01FdQL1KN9LerFiMj3MnrnnY",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 — Let gstack search learnings from your other projects on this machine?\nProject/branch/task: main, PLAN.md design review; one-time gstack setting before the review passes.\nELI10: gstack saves small lessons per project (pitfalls, patterns, preferences). It can also look across your other projects on this machine to spot patterns that apply here. Everything stays local; no data leaves the machine. Good for solo developers; skip if you work on multiple client codebases where cross-contamination is a concern.\nStakes if we pick wrong: enable on a multi-client machine and one client's patterns may color another's review; disable and each project learns from scratch.\nRecommendation: A because this is a fixture repo with zero learnings so far, and cross-project recall is the only way it gets smarter across sessions.\nNote: options differ in kind, not coverage — no completeness score.",
|
||||
"header": "Learnings",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Enable cross-project (recommended)",
|
||||
"description": "✅ Design pitfalls learned elsewhere (toast a11y, modal focus) surface here automatically\n✅ Stays on your machine; one config flag you can flip back any time\n❌ Patterns from unrelated codebases may show up where they don't apply"
|
||||
},
|
||||
{
|
||||
"label": "Keep project-scoped only",
|
||||
"description": "✅ Strict isolation between codebases; nothing bleeds across clients\n✅ Learnings still accumulate for this project on its own\n❌ Every new project starts cold, including this one right now"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 — Let gstack search learnings from your other projects on this machine?\nProject/branch/task: main, PLAN.md design review; one-time gstack setting before the review passes.\nELI10: gstack saves small lessons per project (pitfalls, patterns, preferences). It can also look across your other projects on this machine to spot patterns that apply here. Everything stays local; no data leaves the machine. Good for solo developers; skip if you work on multiple client codebases where cross-contamination is a concern.\nStakes if we pick wrong: enable on a multi-client machine and one client's patterns may color another's review; disable and each project learns from scratch.\nRecommendation: A because this is a fixture repo with zero learnings so far, and cross-project recall is the only way it gets smarter across sessions.\nNote: options differ in kind, not coverage — no completeness score.": "Enable cross-project (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:41:00.666Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "03736bff-d38b-4f3e-a849-84f17064a75b",
|
||||
"toolUseId": "toolu_017YabtVfvEXTzC6wAuVDVin",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 1 — Which panel is primary on the dashboard, and what is the reading order?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 1 Information Architecture, currently 3/10.\nELI10: Three panels are listed as equals. A user who just logged in has one question: \"what happened while I was away?\" One panel has to own the first glance, and the other two have to visibly step back. This decision drives layout at every breakpoint, what goes above the fold on a phone, and where the loading skeleton draws attention. Principle: hierarchy as service; if everything competes, nothing wins.\nStakes if we pick wrong: on a 375px phone the user scrolls past two panels of stuff they didn't ask for before seeing the one that matters; the page reads as a widget mosaic.\nRecommendation: 1A because notifications are the \"while you were away\" answer, activity is the browse-able context, and quick actions are a launcher that should be reachable, not read.\nNote: options differ in kind, not coverage — no completeness score.",
|
||||
"header": "Issue 1",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "1A: Notifications first (recommended)",
|
||||
"description": "✅ Order: Notifications (unread-first) > Activity feed > Quick Actions as a compact action bar; answers \"what changed\" in the first glance\n✅ Unread count becomes the page's single visual anchor, satisfying the one-anchor litmus\n❌ Users with zero notifications see the primary slot empty on every visit; the empty state must carry the page (handled in Pass 2)"
|
||||
},
|
||||
{
|
||||
"label": "1B: Activity feed first",
|
||||
"description": "✅ Activity is always populated for active accounts, so the primary slot rarely reads empty\n✅ Familiar feed pattern; users know how to scroll it\n❌ Buries unread notifications below a feed the user may not care about today; the actionable thing loses to the ambient thing"
|
||||
},
|
||||
{
|
||||
"label": "1C: Quick Actions first (launcher)",
|
||||
"description": "✅ Treats the dashboard as a launchpad: the user came to do something, so put the doing first\n✅ Works well when the product has 2-4 dominant tasks and the feed is secondary\n❌ Ignores the \"users land here after login\" context: a launcher doesn't tell them what happened; notifications and activity become an afterthought"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 1 — Which panel is primary on the dashboard, and what is the reading order?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 1 Information Architecture, currently 3/10.\nELI10: Three panels are listed as equals. A user who just logged in has one question: \"what happened while I was away?\" One panel has to own the first glance, and the other two have to visibly step back. This decision drives layout at every breakpoint, what goes above the fold on a phone, and where the loading skeleton draws attention. Principle: hierarchy as service; if everything competes, nothing wins.\nStakes if we pick wrong: on a 375px phone the user scrolls past two panels of stuff they didn't ask for before seeing the one that matters; the page reads as a widget mosaic.\nRecommendation: 1A because notifications are the \"while you were away\" answer, activity is the browse-able context, and quick actions are a launcher that should be reachable, not read.\nNote: options differ in kind, not coverage — no completeness score.": "1A: Notifications first (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:41:45.355Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "03736bff-d38b-4f3e-a849-84f17064a75b",
|
||||
"toolUseId": "toolu_01JNoKum3CkUH1TidSdGtexa",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 2 — Add a screen-structure diagram (page frame + panel placement) to the plan?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 1 Information Architecture, now 6/10 after Issue 1.\nELI10: The plan says nothing about the page frame: is there a top nav, a page title, where the three panels sit on a wide screen versus a phone. Without this, the implementer picks a layout on the fly and the trunk test fails (cover everything but the nav: can you tell what site and page you're on?). I'd add an ASCII structure diagram for desktop (lg) and mobile (sm) that fixes panel placement per the approved order. Principle: users scan, they don't read; clearly defined areas are how they scan.\nStakes if we pick wrong: three equal-width columns or a stacked card mosaic, the hard-rejection pattern for app UI.\nRecommendation: 2A because the two-zone layout gives Notifications a real anchor position and keeps Quick Actions out of the reading flow.\nCompleteness: 2A=10/10, 2B=7/10, 2C=3/10",
|
||||
"header": "Issue 2",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "2A: Two-zone layout, diagram at lg + sm (recommended)",
|
||||
"description": "✅ lg: sticky top nav; page header row with title + Quick Actions bar right-aligned; body is 2 columns, Notifications left (7/12) and Activity right (5/12)\n✅ sm: single column, Quick Actions as a horizontal scroll strip under the header, then Notifications, then Activity; diagram fixes both\n❌ Notifications-left at 7/12 gives the feed less room; long activity rows will truncate harder on lg"
|
||||
},
|
||||
{
|
||||
"label": "2B: Three-column body, diagram at lg only",
|
||||
"description": "✅ Each panel gets a column; simple grid, simple to build (human: ~1h / CC: ~5min)\n✅ Quick Actions visible as a full column on desktop\n❌ Three equal peers contradicts the approved Notifications-first hierarchy; mobile behavior stays unspecified"
|
||||
},
|
||||
{
|
||||
"label": "2C: Skip the diagram, prose only",
|
||||
"description": "✅ Zero extra planning time; implementer decides layout in code\n✅ Leaves room for the implementer's judgment\n❌ Layout by omission; the first review of a real screen becomes the layout discussion we could have had now"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 2 — Add a screen-structure diagram (page frame + panel placement) to the plan?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 1 Information Architecture, now 6/10 after Issue 1.\nELI10: The plan says nothing about the page frame: is there a top nav, a page title, where the three panels sit on a wide screen versus a phone. Without this, the implementer picks a layout on the fly and the trunk test fails (cover everything but the nav: can you tell what site and page you're on?). I'd add an ASCII structure diagram for desktop (lg) and mobile (sm) that fixes panel placement per the approved order. Principle: users scan, they don't read; clearly defined areas are how they scan.\nStakes if we pick wrong: three equal-width columns or a stacked card mosaic, the hard-rejection pattern for app UI.\nRecommendation: 2A because the two-zone layout gives Notifications a real anchor position and keeps Quick Actions out of the reading flow.\nCompleteness: 2A=10/10, 2B=7/10, 2C=3/10": "2A: Two-zone layout, diagram at lg + sm (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:42:15.329Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "03736bff-d38b-4f3e-a849-84f17064a75b",
|
||||
"toolUseId": "toolu_01D6hkqEL4BpbLKWUECMsBYE",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 3 — Specify what the user sees in every state of every panel (state table with copy and actions)?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 2 Interaction States, currently 3/10.\nELI10: The plan lists \"empty, loading, error\" per panel but never says what any of them look like or say. Left alone, an engineer ships \"No notifications.\" in gray text, and that's the first thing a brand-new user sees in the page's primary slot. I'd add a table covering loading / empty / error / success / partial for Notifications, Activity, and Quick Actions, with the exact copy, the primary action, and skeleton shape. Principle: empty states are features; every one needs warmth, a primary action, and context.\nStakes if we pick wrong: first-login dashboard reads as broken or abandoned; error states offer no recovery path.\nRecommendation: 3A because the first-run and zero-data cases are the ones most users hit first, and writing the copy now costs minutes.\nCompleteness: 3A=10/10, 3B=6/10, 3C=3/10",
|
||||
"header": "Issue 3",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "3A: Full state table, copy + actions + skeletons (recommended)",
|
||||
"description": "✅ Every cell written: e.g. Notifications empty = \"You're all caught up\" + last-checked time; Activity empty on first login = \"Your activity will show up here\" + the top quick action as CTA\n✅ Error cells get a retry button scoped to that panel plus a plain-language reason; skeletons match row anatomy so layout doesn't jump (human: ~2h / CC: ~10min)\n❌ Copy is a proposal; product voice may want to rewrite it later"
|
||||
},
|
||||
{
|
||||
"label": "3B: Table with placeholders, copy written during implementation",
|
||||
"description": "✅ Locks the structure (which states exist per panel) without committing to words now\n✅ Faster to approve; leaves copy to whoever owns voice\n❌ Placeholders become the copy; \"No items found\" ships because nobody circles back"
|
||||
},
|
||||
{
|
||||
"label": "3C: Keep the one-line checklist as is",
|
||||
"description": "✅ No planning time spent\n✅ Implementer has freedom\n❌ Guaranteed \"No notifications.\" in the primary slot; error states with no retry"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 3 — Specify what the user sees in every state of every panel (state table with copy and actions)?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 2 Interaction States, currently 3/10.\nELI10: The plan lists \"empty, loading, error\" per panel but never says what any of them look like or say. Left alone, an engineer ships \"No notifications.\" in gray text, and that's the first thing a brand-new user sees in the page's primary slot. I'd add a table covering loading / empty / error / success / partial for Notifications, Activity, and Quick Actions, with the exact copy, the primary action, and skeleton shape. Principle: empty states are features; every one needs warmth, a primary action, and context.\nStakes if we pick wrong: first-login dashboard reads as broken or abandoned; error states offer no recovery path.\nRecommendation: 3A because the first-run and zero-data cases are the ones most users hit first, and writing the copy now costs minutes.\nCompleteness: 3A=10/10, 3B=6/10, 3C=3/10": "3A: Full state table, copy + actions + skeletons (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:43:02.148Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "03736bff-d38b-4f3e-a849-84f17064a75b",
|
||||
"toolUseId": "toolu_015dYhiwCXPuYiXYwwthqfoB",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 4 — Shape the API response so each panel can fail independently?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 2 Interaction States, now 6/10.\nELI10: The plan wants per-panel error states but fetches everything in one GET /api/dashboard call. Those two goals conflict unless the response itself can say \"activity failed, notifications are fine.\" Today's shape ({ activity, notifications, quickActions }) can't express that, so any single slow or broken query takes down all three panels. This is a design decision because it determines whether the user ever sees a panel-level error or only whole-page failure. Principle: seeing the system, not the screen.\nStakes if we pick wrong: one slow activity query blanks the notifications the user came for; or three separate requests triple the latency on a cold phone connection.\nRecommendation: 4A because it keeps one round-trip (fast first paint on mobile) while letting each panel degrade on its own.\nCompleteness: 4A=10/10, 4B=8/10, 4C=4/10",
|
||||
"header": "Issue 4",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "4A: One call, per-key result envelopes (recommended)",
|
||||
"description": "✅ Response becomes { notifications: {ok, data|error}, activity: {ok, data|error}, quickActions: {...} }; server runs the three queries in parallel with per-query timeouts and never fails the whole response for one key\n✅ One round trip preserves fast first paint on slow mobile; per-panel Retry hits the same endpoint with ?only=activity (human: ~3h / CC: ~15min)\n❌ Slightly more server code and a non-trivial response contract to document"
|
||||
},
|
||||
{
|
||||
"label": "4B: Three endpoints, one per panel",
|
||||
"description": "✅ Simplest failure isolation; each panel owns its fetch, retry, and cache\n✅ Panels can stream in independently as each resolves\n❌ Three round trips on login; on a bad connection the primary panel is no faster than before and the page flickers in three stages"
|
||||
},
|
||||
{
|
||||
"label": "4C: Keep single flat response, whole-page error only",
|
||||
"description": "✅ Zero contract change; simplest to build\n✅ One error state to design\n❌ Contradicts the plan's own per-panel error requirement; one slow query blanks everything the user logged in to see"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 4 — Shape the API response so each panel can fail independently?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 2 Interaction States, now 6/10.\nELI10: The plan wants per-panel error states but fetches everything in one GET /api/dashboard call. Those two goals conflict unless the response itself can say \"activity failed, notifications are fine.\" Today's shape ({ activity, notifications, quickActions }) can't express that, so any single slow or broken query takes down all three panels. This is a design decision because it determines whether the user ever sees a panel-level error or only whole-page failure. Principle: seeing the system, not the screen.\nStakes if we pick wrong: one slow activity query blanks the notifications the user came for; or three separate requests triple the latency on a cold phone connection.\nRecommendation: 4A because it keeps one round-trip (fast first paint on mobile) while letting each panel degrade on its own.\nCompleteness: 4A=10/10, 4B=8/10, 4C=4/10": "4A: One call, per-key result envelopes (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:43:55.568Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "03736bff-d38b-4f3e-a849-84f17064a75b",
|
||||
"toolUseId": "toolu_01LnFZiSt49gf7W5mJjVdG7R",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 5 — Replace the \"Mark all as read\" confirmation modal with instant action + undo toast?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 2 Interaction States, now 7/10.\nELI10: The plan puts a confirmation modal in front of \"Mark all as read.\" Modals are for one-way doors (delete, pay, send). Marking read is low-stakes and reversible, so the convention (Gmail, GitHub, Slack) is: do it immediately, show a toast with Undo for a few seconds. A modal here makes the user answer a question they didn't ask, every time. Principle: the goodwill reservoir; punishing users with an extra step for a safe action depletes it.\nStakes if we pick wrong: keep the modal and the most-used action on the primary panel gains a click and a read; drop undo and a mis-tap wipes the unread list with no recovery.\nRecommendation: 5A because it removes a step from the page's most frequent action while keeping recovery.\nNote: options differ in kind, not coverage — no completeness score.",
|
||||
"header": "Issue 5",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "5A: Instant + Undo toast, drop the modal (recommended)",
|
||||
"description": "✅ One click: unread dots clear optimistically, badge goes to 0, toast \"Marked 12 as read. [Undo]\" for 6s; Undo restores client state and calls the server\n✅ Removes the Modal component from this plan entirely (one less primitive to build and make accessible)\n❌ The 6-second undo window needs a live-region announcement and pause-on-hover, which the toast spec must cover (Issue 6)"
|
||||
},
|
||||
{
|
||||
"label": "5B: Keep modal, but only when unread > N",
|
||||
"description": "✅ Guards the rare large-clear case (say > 50 unread) where an accident costs more\n✅ Small clears stay one-click\n❌ Two behaviors for one button confuses users (\"why did it ask this time?\"); modal still has to be built and made accessible"
|
||||
},
|
||||
{
|
||||
"label": "5C: Keep the confirmation modal as planned",
|
||||
"description": "✅ Zero chance of accidental mass-mark; explicit intent\n✅ Modal primitive may be needed elsewhere later anyway\n❌ Adds friction to the primary panel's main action; mark-read is recoverable, so the modal solves a problem that doesn't exist"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 5 — Replace the \"Mark all as read\" confirmation modal with instant action + undo toast?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 2 Interaction States, now 7/10.\nELI10: The plan puts a confirmation modal in front of \"Mark all as read.\" Modals are for one-way doors (delete, pay, send). Marking read is low-stakes and reversible, so the convention (Gmail, GitHub, Slack) is: do it immediately, show a toast with Undo for a few seconds. A modal here makes the user answer a question they didn't ask, every time. Principle: the goodwill reservoir; punishing users with an extra step for a safe action depletes it.\nStakes if we pick wrong: keep the modal and the most-used action on the primary panel gains a click and a read; drop undo and a mis-tap wipes the unread list with no recovery.\nRecommendation: 5A because it removes a step from the page's most frequent action while keeping recovery.\nNote: options differ in kind, not coverage — no completeness score.": "5A: Instant + Undo toast, drop the modal (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:44:29.271Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "03736bff-d38b-4f3e-a849-84f17064a75b",
|
||||
"toolUseId": "toolu_01An1yzjWxro5Sx3VBjP4fHJ",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 6 — Specify the toast system (position, timing, stacking, dismiss, screen reader behavior)?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 2 Interaction States, now 8/10.\nELI10: \"Toast notification system for action feedback\" is a component name, not a spec. And after Issue 5 the toast carries Undo, so its timing and accessibility now decide whether a user can recover from a mis-click. I'd pin down: where it appears, how long it stays, what happens with several at once, how to dismiss, and how screen readers hear it (a live region, so the Undo offer is announced and reachable by keyboard). Principle: accessibility is not optional; specify it in the plan or it won't exist.\nStakes if we pick wrong: a screen-reader user never hears \"Undo\"; toasts stack over the Quick Actions bar on mobile; a 3-second toast makes Undo a race.\nRecommendation: 6A because the toast is now the recovery mechanism for the primary panel's main action.\nCompleteness: 6A=10/10, 6B=6/10",
|
||||
"header": "Issue 6",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "6A: Full toast spec (recommended)",
|
||||
"description": "✅ Bottom-center on sm (above safe-area, never over the action strip), bottom-right on md+; 6s default, 10s when it carries an action, pause on hover/focus; max 3 stacked, oldest drops\n✅ role=status live region for info, role=alert for errors; action button is a real <button> reachable by Tab; Escape dismisses the focused toast; swipe-down dismisses on touch (human: ~3h / CC: ~15min)\n❌ More component surface to build and test than a fire-and-forget snackbar"
|
||||
},
|
||||
{
|
||||
"label": "6B: Minimal spec: position + duration only",
|
||||
"description": "✅ Enough to stop toasts landing over the action bar; quick to write\n✅ Leaves implementation freedom on stacking and a11y\n❌ No live region means Undo is invisible to assistive tech; stacking and dismiss get decided in code"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 6 — Specify the toast system (position, timing, stacking, dismiss, screen reader behavior)?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 2 Interaction States, now 8/10.\nELI10: \"Toast notification system for action feedback\" is a component name, not a spec. And after Issue 5 the toast carries Undo, so its timing and accessibility now decide whether a user can recover from a mis-click. I'd pin down: where it appears, how long it stays, what happens with several at once, how to dismiss, and how screen readers hear it (a live region, so the Undo offer is announced and reachable by keyboard). Principle: accessibility is not optional; specify it in the plan or it won't exist.\nStakes if we pick wrong: a screen-reader user never hears \"Undo\"; toasts stack over the Quick Actions bar on mobile; a 3-second toast makes Undo a race.\nRecommendation: 6A because the toast is now the recovery mechanism for the primary panel's main action.\nCompleteness: 6A=10/10, 6B=6/10": "6A: Full toast spec (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:45:09.113Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "03736bff-d38b-4f3e-a849-84f17064a75b",
|
||||
"toolUseId": "toolu_01YTEyoxMNs3gNr8P34hzqpP",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 7 — What happens when the user clicks a notification row or an activity row?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 3 User Journey, currently 4/10.\nELI10: The primary panel is a list of notifications, and the plan never says what a row does when clicked. Is it a link to the thing being notified about? Does clicking mark it read? Same for activity rows. Without this, an engineer either makes rows inert (dead-end dashboard) or invents a destination. Principle: seeing the system, not the screen; the dashboard exists to route the user onward.\nStakes if we pick wrong: inert rows make the primary panel read-only wallpaper; rows that navigate without marking read leave the unread badge stuck.\nRecommendation: 7A because it matches the convention users already know from every inbox, and it keeps the badge honest.\nCompleteness: 7A=10/10, 7B=7/10, 7C=3/10",
|
||||
"header": "Issue 7",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "7A: Whole row is a link; click marks read then navigates (recommended)",
|
||||
"description": "✅ Notification row = <a href={targetUrl}> covering the full row (44px min height); click optimistically marks that item read, then navigates; unread dot fades before route change\n✅ Activity row links to its object (e.g. the document, the comment); rows with no target render as plain text, not fake links (human: ~2h / CC: ~10min)\n❌ Requires each notification and activity item to carry a targetUrl from the API; items without one need the plain-text fallback"
|
||||
},
|
||||
{
|
||||
"label": "7B: Row expands inline; explicit \"Open\" link inside",
|
||||
"description": "✅ User previews the full message without leaving the dashboard\n✅ Mark-read happens on expand, so badge stays honest\n❌ Two clicks to reach the object; expand/collapse adds state and a11y (aria-expanded) that the inbox convention doesn't need"
|
||||
},
|
||||
{
|
||||
"label": "7C: Rows are static; only \"Mark all as read\" is interactive",
|
||||
"description": "✅ Simplest build; no per-item endpoints\n✅ No risk of mis-navigation\n❌ Primary panel becomes a read-only log; users have to hunt elsewhere for the thing they were notified about"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 7 — What happens when the user clicks a notification row or an activity row?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 3 User Journey, currently 4/10.\nELI10: The primary panel is a list of notifications, and the plan never says what a row does when clicked. Is it a link to the thing being notified about? Does clicking mark it read? Same for activity rows. Without this, an engineer either makes rows inert (dead-end dashboard) or invents a destination. Principle: seeing the system, not the screen; the dashboard exists to route the user onward.\nStakes if we pick wrong: inert rows make the primary panel read-only wallpaper; rows that navigate without marking read leave the unread badge stuck.\nRecommendation: 7A because it matches the convention users already know from every inbox, and it keeps the badge honest.\nCompleteness: 7A=10/10, 7B=7/10, 7C=3/10": "7A: Whole row is a link; click marks read then navigates (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:45:58.362Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "03736bff-d38b-4f3e-a849-84f17064a75b",
|
||||
"toolUseId": "toolu_015ZD7y9iw62qs78kvuc7hoY",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 8 — Does the page header carry a greeting/summary line, and what does it say?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 3 User Journey, now 8/10.\nELI10: The structure diagram shows \"Good morning, Sam. 3 unread.\" under the Dashboard title as a placeholder. That line can do real work (a one-sentence status the user reads before scanning panels) or it can be happy talk that wastes the most valuable line on the page. On visit #500 a time-of-day greeting is noise; a status sentence still earns its place. Principle: omit, then omit again; every word must carry information.\nStakes if we pick wrong: a greeting that feels generated and gets ignored, or a bare title with no summary, so the user has to scan the panels to learn there's nothing new.\nRecommendation: 8A because a status sentence reinforces the Notifications anchor and reads true on the first visit and the thousandth.\nNote: options differ in kind, not coverage — no completeness score.",
|
||||
"header": "Issue 8",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "8A: Status sentence only, no greeting (recommended)",
|
||||
"description": "✅ Copy variants: \"3 unread notifications.\" / \"You're all caught up.\" / \"2 unread, 5 new activity items since yesterday.\"; no name, no time-of-day\n✅ Doubles as the page-level live summary for screen readers (aria-describedby on the h1)\n❌ Feels less personal than a greeting; if that warmth matters to the brand this is the wrong call"
|
||||
},
|
||||
{
|
||||
"label": "8B: Greeting + status (\"Good morning, Sam. 3 unread.\")",
|
||||
"description": "✅ Warmer first impression for new users; name confirms which account is logged in\n✅ Still carries the status count\n❌ Time-of-day greetings are the most common generated-dashboard tell; on daily visits it's read as filler"
|
||||
},
|
||||
{
|
||||
"label": "8C: Title only, no second line",
|
||||
"description": "✅ Maximum subtraction; the panels speak for themselves\n✅ One less string to localize\n❌ User must scan panels to learn there's nothing new; loses the cheap page-level screen-reader summary"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 8 — Does the page header carry a greeting/summary line, and what does it say?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 3 User Journey, now 8/10.\nELI10: The structure diagram shows \"Good morning, Sam. 3 unread.\" under the Dashboard title as a placeholder. That line can do real work (a one-sentence status the user reads before scanning panels) or it can be happy talk that wastes the most valuable line on the page. On visit #500 a time-of-day greeting is noise; a status sentence still earns its place. Principle: omit, then omit again; every word must carry information.\nStakes if we pick wrong: a greeting that feels generated and gets ignored, or a bare title with no summary, so the user has to scan the panels to learn there's nothing new.\nRecommendation: 8A because a status sentence reinforces the Notifications anchor and reads true on the first visit and the thousandth.\nNote: options differ in kind, not coverage — no completeness score.": "8A: Status sentence only, no greeting (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:46:30.592Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "03736bff-d38b-4f3e-a849-84f17064a75b",
|
||||
"toolUseId": "toolu_01JySFYtMJF5AxpGUDe3WTtf",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 9 — Specify the row anatomy for notifications and activity items (fields, truncation, timestamps, unread indicator, pagination)?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 4 AI Slop Risk, currently 4/10.\nELI10: The plan says \"rows\" and nothing more. Without a row spec, every generated dashboard produces the same thing: avatar circle, bold name, gray sentence, timestamp on the right, all the same weight. A designed row decides what the eye hits first (the object, not the actor), how a 47-character name behaves, whether the time reads \"3m\" or \"Sep 20, 2:14 PM\", and how unread is marked. Principle: specificity over vibes; edge cases (long names, zero results) are user experiences.\nStakes if we pick wrong: rows overflow on long names, timestamps wrap, unread is a colored left border (blacklist item #8), and the page reads as template output.\nRecommendation: 9A because the row is the unit the user reads 20 times per visit; it's where care is most visible.\nCompleteness: 9A=10/10, 9B=6/10",
|
||||
"header": "Issue 9",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "9A: Full row spec, both panels + pagination (recommended)",
|
||||
"description": "✅ Notification: 8px unread dot (accent) in a fixed 16px gutter, title 16px medium 1-line truncate, summary 14px muted 1-line truncate, time right-aligned tabular relative (\"3m\", \"2h\", \"Tue\", \"Sep 3\") with full datetime in title attr\n✅ Activity: 24px avatar, sentence \"<Actor> <verb> <Object>\" where Object is medium-weight and actor truncates at 24ch with ellipsis; time same format; page size 20 / 10 with \"Load more\" (human: ~2h / CC: ~10min)\n❌ Fixes v1 rows to single-line truncation; multi-line notification bodies need a later revision"
|
||||
},
|
||||
{
|
||||
"label": "9B: Field list only, visual treatment left to implementation",
|
||||
"description": "✅ Locks the data each row needs from the API without dictating pixels\n✅ Faster to approve\n❌ Truncation, timestamp format, and unread indicator get invented in code; the colored-left-border default is likely"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 9 — Specify the row anatomy for notifications and activity items (fields, truncation, timestamps, unread indicator, pagination)?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 4 AI Slop Risk, currently 4/10.\nELI10: The plan says \"rows\" and nothing more. Without a row spec, every generated dashboard produces the same thing: avatar circle, bold name, gray sentence, timestamp on the right, all the same weight. A designed row decides what the eye hits first (the object, not the actor), how a 47-character name behaves, whether the time reads \"3m\" or \"Sep 20, 2:14 PM\", and how unread is marked. Principle: specificity over vibes; edge cases (long names, zero results) are user experiences.\nStakes if we pick wrong: rows overflow on long names, timestamps wrap, unread is a colored left border (blacklist item #8), and the page reads as template output.\nRecommendation: 9A because the row is the unit the user reads 20 times per visit; it's where care is most visible.\nCompleteness: 9A=10/10, 9B=6/10": "9A: Full row spec, both panels + pagination (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:47:23.064Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "03736bff-d38b-4f3e-a849-84f17064a75b",
|
||||
"toolUseId": "toolu_017iCDG42LbcVXfXdtAsDXSg",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 10 — Define the plan's design tokens (typeface, type scale, color roles as CSS variables, spacing, radius)?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 5 Design System, currently 2/10.\nELI10: Decisions 1-9 use names like text-muted, accent, surface-hover, 16px/14px, but nothing defines them. Tailwind's defaults will fill the gaps: system-ui font, gray-500 text, blue-600 accent. That's the exact \"assembled, not designed\" look. I'd add a small token block: one typeface (a real one, not the system stack), a 4-step type scale, ~8 color roles as CSS variables mapped into Tailwind's theme, a spacing scale, and one radius. These are proposals to be replaced by DESIGN.md if you run /design-consultation later. Principle: specificity over vibes; name the font, the spacing scale, the interaction pattern.\nStakes if we pick wrong: the dashboard ships in Tailwind default blue on gray with system-ui, and every later screen inherits it.\nRecommendation: 10A because the tokens are cheap to write now and every component in this plan is blocked on them.\nCompleteness: 10A=10/10, 10B=6/10, 10C=2/10",
|
||||
"header": "Issue 10",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "10A: Full token block, CSS variables into Tailwind theme (recommended)",
|
||||
"description": "✅ Typeface: Instrument Sans (body/UI, Operate-surface approved), tabular-nums enabled; scale 12/14/16/20/28 with named roles; 8 color roles as --color-* vars (bg, surface, surface-hover, border, text, text-muted, accent, danger) with light values now and dark slots reserved\n✅ Spacing 4/8/12/16/24/32, one radius (6px) for buttons and toasts only, no radius on rows or panels; focus ring, selection color, and scrollbar themed from the palette (human: ~3h / CC: ~15min)\n❌ Specific picks (typeface, accent hue) are my taste until a DESIGN.md exists; you may want to swap them"
|
||||
},
|
||||
{
|
||||
"label": "10B: Roles only, values TBD",
|
||||
"description": "✅ Names the variables so components reference roles, not raw Tailwind colors\n✅ Defers taste calls (font, hue) to /design-consultation\n❌ Values default to Tailwind's until someone fills them; the first shipped screen is still default-blue"
|
||||
},
|
||||
{
|
||||
"label": "10C: Use Tailwind defaults, no tokens",
|
||||
"description": "✅ Zero setup; fastest to build\n✅ Familiar to any Tailwind dev\n❌ Fails universal rules (no color variables, default font stack); generated-dashboard look guaranteed"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 10 — Define the plan's design tokens (typeface, type scale, color roles as CSS variables, spacing, radius)?\nProject/branch/task: main, PLAN.md User Dashboard; Pass 5 Design System, currently 2/10.\nELI10: Decisions 1-9 use names like text-muted, accent, surface-hover, 16px/14px, but nothing defines them. Tailwind's defaults will fill the gaps: system-ui font, gray-500 text, blue-600 accent. That's the exact \"assembled, not designed\" look. I'd add a small token block: one typeface (a real one, not the system stack), a 4-step type scale, ~8 color roles as CSS variables mapped into Tailwind's theme, a spacing scale, and one radius. These are proposals to be replaced by DESIGN.md if you run /design-consultation later. Principle: specificity over vibes; name the font, the spacing scale, the interaction pattern.\nStakes if we pick wrong: the dashboard ships in Tailwind default blue on gray with system-ui, and every later screen inherits it.\nRecommendation: 10A because the tokens are cheap to write now and every component in this plan is blocked on them.\nCompleteness: 10A=10/10, 10B=6/10, 10C=2/10": "10A: Full token block, CSS variables into Tailwind theme (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-20T20:48:12.161Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"additionalQuestionCaptures": [
|
||||
{
|
||||
"source": {
|
||||
"commit": "269b5747",
|
||||
"workflowRun": 35537130656,
|
||||
"attempt": 2,
|
||||
"retainedRange": "final question, options and answer; owning IDs precede the retained log tail"
|
||||
},
|
||||
"question": {
|
||||
"question": "D10 — Issue 6: 'Mark all as read' — confirmation modal (as planned) or immediate action with an Undo toast?\nProject/branch/task: dashboard plan on main; plan line 13 specifies a modal dialog.\nELI10: A confirmation dialog asks 'are you sure?' before doing something. It's right for deleting data. For marking notifications read, it's an extra click every time, and the user learns to dismiss it without reading. The alternative is: do it instantly, show a small toast 'Marked 3 as read. Undo' for 5 seconds, and flip them back if they tap Undo. Same safety, zero friction.\nStakes if we pick wrong: Modal: a daily annoyance that trains users to click through dialogs (which then makes real destructive dialogs less safe). Undo without a backend path: a toast that lies.\nRecommendation: 6A because the action is reversible and low-stakes; modals should be reserved for one-way doors. Principle: clicks don't matter, thinking does; goodwill reservoir.\nCompleteness: 6A=10/10, 6B=7/10, 6C=5/10\nNet: frictionless with a small backend addition vs. friction with no backend change.",
|
||||
"header": "Mark read",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "6A Immediate + Undo toast, drop the modal (recommended)",
|
||||
"description": "✅ Click -> rows fade to read state optimistically, badge goes to 0, toast 'Marked {n} as read. Undo' for 6s; Undo restores the exact set. ✅ Removes the modal component from scope entirely; toast system already in plan. ❌ Needs a backend 'mark unread by id list' (or a single reversible batch endpoint) so Undo is real, not cosmetic. (human: ~3h / CC: ~20 min)"
|
||||
},
|
||||
{
|
||||
"label": "6B Keep the modal, make it a one-line dialog",
|
||||
"description": "✅ No backend change; matches the plan as written. ✅ Dialog: 'Mark 3 notifications as read?' [Cancel] [Mark read], focus on Cancel, Esc closes. ❌ Adds a click to a daily action; trains users to dismiss dialogs; modal + focus trap must still be built and tested. (human: ~2h / CC: ~15 min)"
|
||||
},
|
||||
{
|
||||
"label": "6C Immediate, plain success toast, no Undo",
|
||||
"description": "✅ Simplest: click, rows update, toast 'Marked as read'. ✅ No modal, no undo endpoint. ❌ Accidental click has no recovery; unread state is lost for that session."
|
||||
}
|
||||
]
|
||||
},
|
||||
"answer": "6A Immediate + Undo toast, drop the modal (recommended)"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -22,6 +22,16 @@ function run(args: string[]): { code: number; stdout: string; stderr: string } {
|
||||
}
|
||||
|
||||
describe('gstack-artifacts-url', () => {
|
||||
test('normalization does not depend on external line readers', () => {
|
||||
for (const url of ['git@github.com:team/repo.git', 'https://github.com/team/repo.git']) {
|
||||
const result = spawnSync(Bun.which('bash')!, [URL_BIN, '--to', 'https', url], {
|
||||
env: { PATH: '' }, encoding: 'utf8', timeout: 30_000,
|
||||
});
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
expect(result.stdout.trim()).toBe('https://github.com/team/repo');
|
||||
}
|
||||
});
|
||||
|
||||
test('--to ssh from canonical https', () => {
|
||||
const r = run(['--to', 'ssh', 'https://github.com/garrytan/gstack-artifacts-garrytan']);
|
||||
expect(r.code).toBe(0);
|
||||
|
||||
@@ -143,6 +143,7 @@ export interface ClaudePtySession {
|
||||
* dialog or boot banner residue. Returns a marker handle.
|
||||
*/
|
||||
mark(): number;
|
||||
waitForOutput(since: number, timeoutMs: number): Promise<void>;
|
||||
/** Visible text since the most recent (or specific) mark. */
|
||||
visibleSince(marker?: number): string;
|
||||
/**
|
||||
@@ -3637,7 +3638,7 @@ export const designFirstReviewAUQ: Step0BoundaryPredicate = (fp) => {
|
||||
// question ID as well, and exclude its scope/focus/onboarding identities.
|
||||
const id = /<gstack-qid:\s*plan-design-review-([a-z0-9-]+)/i.exec(fp.promptSnippet)?.[1];
|
||||
if (id && /(?:^|[│\s])D\s*\d+\s*[—–-]/i.test(fp.promptSnippet) &&
|
||||
!/(?:^|-)(?:scope|focus|setup|routing|onboarding|posture|mockups?|target)(?:-|$)/i.test(id) &&
|
||||
!/(?:^|-)(?:scope|focus|setup|routing|onboarding|posture|mockups?|target|outside(?:-design)?-voices)(?:-|$)/i.test(id) &&
|
||||
!designStep0Boundary(fp)) return true;
|
||||
// Explicit pass headings are also review evidence; an initial assessment
|
||||
// that merely mentions reviewing seven passes does not match this shape.
|
||||
@@ -3674,7 +3675,10 @@ export async function launchClaudePty(
|
||||
|
||||
let buffer = '';
|
||||
let exited = false;
|
||||
let closing = false;
|
||||
let exitCodeCaptured: number | null = null;
|
||||
const outputWaiters = new Set<() => void>();
|
||||
const notifyOutput = () => { for (const done of outputWaiters) done(); };
|
||||
|
||||
const args: string[] = [];
|
||||
// Pin the model so smokes don't inherit the operator's settings.json model
|
||||
@@ -3757,6 +3761,7 @@ export async function launchClaudePty(
|
||||
const text = chunk.toString('utf-8');
|
||||
buffer += text;
|
||||
if (screen && !screenClosing) screen.write(text);
|
||||
notifyOutput();
|
||||
},
|
||||
},
|
||||
cwd,
|
||||
@@ -3770,10 +3775,12 @@ export async function launchClaudePty(
|
||||
.then(async (code: number | null) => {
|
||||
exitCodeCaptured = code;
|
||||
exited = true;
|
||||
notifyOutput();
|
||||
await disposeScreen();
|
||||
})
|
||||
.catch(async () => {
|
||||
exited = true;
|
||||
notifyOutput();
|
||||
await disposeScreen();
|
||||
});
|
||||
}
|
||||
@@ -3848,6 +3855,19 @@ export async function launchClaudePty(
|
||||
return stripAnsi(buffer.slice(offset));
|
||||
}
|
||||
|
||||
async function waitForOutput(since: number, timeoutMs: number): Promise<void> {
|
||||
if (buffer.length > since || exited || closing) return;
|
||||
await new Promise<void>((resolve) => {
|
||||
const done = () => {
|
||||
clearTimeout(timer);
|
||||
outputWaiters.delete(done);
|
||||
resolve();
|
||||
};
|
||||
const timer = setTimeout(done, timeoutMs);
|
||||
outputWaiters.add(done);
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForAny(
|
||||
patterns: Array<RegExp | string>,
|
||||
waitOpts?: { timeoutMs?: number; pollMs?: number; since?: number },
|
||||
@@ -3890,25 +3910,28 @@ export async function launchClaudePty(
|
||||
}
|
||||
|
||||
async function close(): Promise<void> {
|
||||
closing = true;
|
||||
notifyOutput();
|
||||
clearTimeout(wallTimer);
|
||||
clearTimeout(trustWatcherStop);
|
||||
clearInterval(trustWatcher);
|
||||
for (const timer of trustInputTimers) clearTimeout(timer);
|
||||
if (exited) { pendingFiles.forEach(({ recorder }) => recorder.dispose()); pendingExit?.dispose(); pendingQuestion?.dispose(); pendingArtifact?.dispose(); await disposeScreen(); return; }
|
||||
try {
|
||||
proc.kill?.('SIGINT');
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
// Wait up to 2s for graceful exit.
|
||||
await Promise.race([exitedPromise, Bun.sleep(2000)]);
|
||||
if (!exited) {
|
||||
for (const [signal, timeout] of [['SIGINT', 2000], ['SIGKILL', 1000]] as const) {
|
||||
if (exited) break;
|
||||
try {
|
||||
proc.kill?.('SIGKILL');
|
||||
proc.kill?.(signal);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
await Promise.race([exitedPromise, Bun.sleep(1000)]);
|
||||
let deadline!: ReturnType<typeof setTimeout>;
|
||||
try {
|
||||
await Promise.race([exitedPromise, new Promise<void>((resolve) => {
|
||||
deadline = setTimeout(resolve, timeout);
|
||||
})]);
|
||||
} finally {
|
||||
clearTimeout(deadline);
|
||||
}
|
||||
}
|
||||
pendingFiles.forEach(({ recorder }) => recorder.dispose());
|
||||
pendingExit?.dispose();
|
||||
@@ -3928,6 +3951,7 @@ export async function launchClaudePty(
|
||||
return screen.read();
|
||||
},
|
||||
mark,
|
||||
waitForOutput,
|
||||
visibleSince,
|
||||
waitForAny,
|
||||
waitFor,
|
||||
@@ -4256,7 +4280,7 @@ export async function runPlanSkillObservation(opts: {
|
||||
...highWaterFlags(),
|
||||
};
|
||||
}
|
||||
if (visible.includes('Unknown command:')) {
|
||||
if (isUnknownSlashCommandVisible(visible, `/${opts.skillName}`)) {
|
||||
return {
|
||||
outcome: 'exited',
|
||||
summary: `claude rejected /${opts.skillName} as unknown command (skill not registered in this cwd)`,
|
||||
@@ -4440,6 +4464,12 @@ export interface PlanSkillCountObservation {
|
||||
administrativeCount: number;
|
||||
}
|
||||
|
||||
export function isUnknownSlashCommandVisible(visible: string, slashCommand: string): boolean {
|
||||
const command = slashCommand.trim().split(/\s+/)[0];
|
||||
return [...visible.matchAll(/Unknown command:\s*(\/[\w-]+)(?=\s|$)/g)]
|
||||
.some(match => match[1] === command);
|
||||
}
|
||||
|
||||
/**
|
||||
* Drive a plan-* skill in plan mode and count distinct native review-phase
|
||||
* AskUserQuestions until a terminal signal fires. Each run disables the
|
||||
@@ -4536,6 +4566,7 @@ export async function runPlanSkillCounting(opts: {
|
||||
firstAUQPick?: (fp: AskUserQuestionFingerprint) => number;
|
||||
/** Total budget including startup and cleanup. Must exceed the 5s cleanup reserve. Default 1_500_000. */
|
||||
timeoutMs?: number;
|
||||
startupReadyMarker?: string;
|
||||
/** Extra env merged into the spawned `claude` process. */
|
||||
env?: Record<string, string>;
|
||||
/** Override the spawned model. Defaults via launchClaudePty's chain. */
|
||||
@@ -4545,6 +4576,9 @@ export async function runPlanSkillCounting(opts: {
|
||||
const startedAt = Date.now();
|
||||
const defaultPick = opts.defaultPick ?? 1;
|
||||
const timeoutMs = opts.timeoutMs ?? 1_500_000;
|
||||
if (opts.startupReadyMarker !== undefined && !opts.startupReadyMarker.length) {
|
||||
throw new RangeError('Plan counting startup-ready marker must not be empty');
|
||||
}
|
||||
// The caller may use this same limit as its Bun timeout. Leave room for
|
||||
// close()'s 2s graceful + 1s forced exit waits and artifact/fixture cleanup.
|
||||
// A second work window after boot lets Bun retry while this body is alive.
|
||||
@@ -4642,14 +4676,29 @@ export async function runPlanSkillCounting(opts: {
|
||||
return observation;
|
||||
}
|
||||
|
||||
let observedOutput = session.mark();
|
||||
let lastObservationAt = -Infinity;
|
||||
try {
|
||||
if (await waitForWork(8000)) { // boot grace is part of the total budget
|
||||
session.mark();
|
||||
let startupReady: boolean;
|
||||
if (opts.startupReadyMarker !== undefined) {
|
||||
await session.waitFor(opts.startupReadyMarker, { timeoutMs: Math.min(8000, remainingWork()) });
|
||||
startupReady = remainingWork() > 0;
|
||||
} else {
|
||||
startupReady = await waitForWork(8000);
|
||||
}
|
||||
if (startupReady) {
|
||||
observedOutput = session.mark();
|
||||
session.send(`${opts.slashCommand}\r`);
|
||||
}
|
||||
|
||||
while (remainingWork() > 0) {
|
||||
if (!await waitForWork(2000)) break;
|
||||
await session.waitForOutput(observedOutput, Math.min(2000, remainingWork()));
|
||||
if (remainingWork() <= 0) break;
|
||||
const coalesceMs = session.rawOutput().length > observedOutput
|
||||
? 250 : 250 - (performance.now() - lastObservationAt);
|
||||
if (coalesceMs > 0 && !await waitForWork(coalesceMs)) break;
|
||||
observedOutput = session.mark();
|
||||
lastObservationAt = performance.now();
|
||||
const visible = viewport = await session.currentScreen();
|
||||
if (remainingWork() <= 0) break;
|
||||
transcript = session.hermeticConfigDir
|
||||
@@ -4703,7 +4752,7 @@ export async function runPlanSkillCounting(opts: {
|
||||
);
|
||||
}
|
||||
|
||||
if (visible.includes('Unknown command:')) {
|
||||
if (isUnknownSlashCommandVisible(visible, opts.slashCommand)) {
|
||||
return snapshot(
|
||||
'exited',
|
||||
`claude rejected ${opts.slashCommand} as unknown command (skill not registered in this cwd)`,
|
||||
@@ -5010,7 +5059,7 @@ export async function runPlanSkillFloorCheck(opts: {
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
});
|
||||
}
|
||||
if (visible.includes('Unknown command:')) {
|
||||
if (isUnknownSlashCommandVisible(visible, opts.slashCommand)) {
|
||||
return finish({
|
||||
auqObserved: false,
|
||||
outcome: 'exited',
|
||||
|
||||
@@ -17,7 +17,7 @@ export function pickDesignCountOutsideVoices(
|
||||
if (active.signature !== identity) return null;
|
||||
const q = call.questions[index]!;
|
||||
if (q.multiSelect || !/^outside(?: design)? voices$/i.test(q.header.trim()) ||
|
||||
!/<gstack-qid:outside-voices-design>/.test(q.question)) return null;
|
||||
!/<gstack-qid:(?:outside-voices-design|plan-design-review-outside-voices)>/.test(q.question)) return null;
|
||||
question = q.question;
|
||||
labels = q.options.map(option => option.label);
|
||||
} else {
|
||||
@@ -31,11 +31,11 @@ export function pickDesignCountOutsideVoices(
|
||||
labels = active.options.map(option => option.label);
|
||||
while (labels.length > 2 && /^(?:Type something\.?|Chat about this)$/i.test(labels.at(-1)!.trim())) labels.pop();
|
||||
}
|
||||
if (!/\b(?:want|run|include|enable)\b[^?]{0,90}\boutside design voices\b/i.test(question) ||
|
||||
if (!/\b(?:want|run|include|enable)\b[^?]{0,90}\boutside(?: design)? voices\b/i.test(question) ||
|
||||
!/\b(?:before|for)\s+(?:the\s+)?(?:detailed\s+)?(?:design\s+)?review\b/i.test(question)) return null;
|
||||
labels = labels.map(label => label.trim().replace(/\s*\(recommended\)\s*$/i, ''));
|
||||
if (labels.length !== 2) return null;
|
||||
const yes = labels.map(label => /^Yes,?\s+run outside design voices$/i.test(label));
|
||||
const yes = labels.map(label => /^Yes,?\s+run outside(?: design)? voices$/i.test(label));
|
||||
const no = labels.map(label => /^No,?\s+proceed without$/i.test(label));
|
||||
if (yes.filter(Boolean).length !== 1 || no.filter(Boolean).length !== 1) return null;
|
||||
return no.findIndex(Boolean) + 1;
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import type { AskUserQuestionFingerprint } from './claude-pty-runner';
|
||||
import { isDesignCountFirstReview } from './design-count-review';
|
||||
|
||||
export function isDesignUIScopeReview(fp: AskUserQuestionFingerprint): boolean {
|
||||
const call = fp.nativeCall;
|
||||
if (!call?.answered || call.failed || !Array.isArray(call.unansweredQuestionIndices) ||
|
||||
call.unansweredQuestionIndices.length || !call.questions.length ||
|
||||
fp.signature !== `${call.sessionId}:${call.toolUseId}`) return false;
|
||||
if (call.questions.some(q => q.multiSelect || q.options.length < 2 ||
|
||||
new Set(q.options.map(option => option.label)).size !== q.options.length ||
|
||||
!q.options.some(option => option.label === call.answers?.[q.question]))) return false;
|
||||
if (isDesignCountFirstReview(fp)) return true;
|
||||
const workflow = /\b(?:review(?:s|ers?)?|scope|setup|learnings|routing|mockups?|permissions?|codex|claude|outside)\b/i;
|
||||
const ui = /\b(?:dashboard|hierarchy|panels?|layout|headers?|buttons?|navigation|notifications?|activity|actions?|spacing|colou?rs?|fonts?|typography|loading|errors?|focus|contrast|keyboard|mobile|responsive|toasts?|modals?|empty)\b/i;
|
||||
return call.questions.some(q => {
|
||||
if (/^(?:scope|focus|learnings|routing|next steps?|outside(?: design)? voices)$/i.test(q.header.trim())) return false;
|
||||
const issue = /^(?:D\d+\s*[—–:-]\s*)?Issue ([1-9]\d*)\s*[:—–-]\s*([^\n]+\?)$/i.exec(q.question.split('\n')[0]!.trim());
|
||||
if (!issue || workflow.test(issue[2]!) || !ui.test(issue[2]!) ||
|
||||
!q.options.some(option => ui.test(`${option.label} ${option.description ?? ''}`))) return false;
|
||||
const context = /^Project\/branch\/task:([^\n]*)/mi.exec(q.question)?.[1] ?? '';
|
||||
const namedPlans = context.match(/\b[\w.-]+\.md\b/gi) ?? [];
|
||||
if ((namedPlans.length && !namedPlans.some(plan => /^PLAN\.md$/i.test(plan))) ||
|
||||
/\b(?:before|prior to)\s+Pass\b/i.test(context)) return false;
|
||||
const choice = new RegExp(`^${issue[1]}[A-Z](?:[).:—–-]\\s*|\\s+)\\S`);
|
||||
return q.options.every(option => choice.test(option.label) && !workflow.test(option.label));
|
||||
});
|
||||
}
|
||||
@@ -228,6 +228,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
"test/ceo-hold-commitment-ar.test.ts", "test/fixtures/ceo-hold-commitment-ar.json",
|
||||
],
|
||||
'plan-design-with-ui-scope': [
|
||||
'test/helpers/design-ui-scope.ts', 'test/design-ui-scope.test.ts', 'test/fixtures/plan-design-ui-scope.json',
|
||||
"test/plan-scope-recovery-av.test.ts",
|
||||
"test/fixtures/plan-scope-recovery-av.json",
|
||||
"test/fixtures/design-scope-checkpoint-at.json",'plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-with-ui.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt',
|
||||
|
||||
@@ -287,6 +287,23 @@ describe('check-careful.sh', () => {
|
||||
});
|
||||
});
|
||||
|
||||
test.each([
|
||||
['rm -rf node_modules\nrm -rf /', 'recursive delete'],
|
||||
['rm${IFS}-rf${IFS}/', 'obfuscation'],
|
||||
['psql -c "DROP DATABASE production"', 'SQL DROP'],
|
||||
['psql -c "TRUNCATE users"', 'SQL TRUNCATE'],
|
||||
['git push --force origin feature', 'force-push'],
|
||||
['git reset --hard', 'reset --hard'],
|
||||
['git restore .', 'uncommitted changes'],
|
||||
['kubectl delete pod app', 'kubectl delete'],
|
||||
['docker system prune', 'Docker'],
|
||||
])('keeps %s visible before large multiline content', (command, reason) => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput(`${command}\n# ${'x'.repeat(100_000)}`));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.hookSpecificOutput?.permissionDecision).toBe('ask');
|
||||
expect(output.hookSpecificOutput?.permissionDecisionReason).toContain(reason);
|
||||
});
|
||||
|
||||
// --- Shell obfuscation ---
|
||||
|
||||
describe('shell obfuscation', () => {
|
||||
@@ -681,6 +698,16 @@ describe('check-careful.sh', () => {
|
||||
});
|
||||
});
|
||||
|
||||
test('a project pattern matches before large multiline content', () => {
|
||||
withPatternFile('terraform\\s+destroy\n', (gstackHome) => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT,
|
||||
carefulInput(`terraform destroy\n# ${'x'.repeat(100_000)}`), { GSTACK_HOME: gstackHome });
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.hookSpecificOutput?.permissionDecision).toBe('ask');
|
||||
expect(output.hookSpecificOutput?.permissionDecisionReason).toContain('Project rule');
|
||||
});
|
||||
});
|
||||
|
||||
test('a garbage pattern file cannot suppress a baseline match (additive invariant)', () => {
|
||||
withPatternFile('# override: allow everything\nallow-everything\nignore baseline\n', (gstackHome) => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('rm -rf /var/data'), { GSTACK_HOME: gstackHome });
|
||||
|
||||
@@ -98,11 +98,12 @@ process.stdin.on('data', async data => {
|
||||
});
|
||||
process.on('SIGINT',()=>process.exit(0));
|
||||
process.stdin.resume();
|
||||
process.stdout.write('PTY_READY:'+item.log+'\x1b[2J\x1b[H');
|
||||
`);
|
||||
fs.chmodSync(fake, 0o755);
|
||||
fs.writeFileSync(worker, `import {runPlanSkillCounting} from ${JSON.stringify(pathToFileURL(path.resolve(import.meta.dir, 'helpers/claude-pty-runner.ts')).href)};
|
||||
const cases=${JSON.stringify(cases)};
|
||||
const results=await Promise.all(cases.map(async item=>({late:item.late,observation:await runPlanSkillCounting({skillName:'plan-eng-review',slashCommand:'/plan-eng-review',followUpPrompt:'Review this fixture.',isLastStep0AUQ:()=>false,isReviewAUQ:()=>true,reviewCountCeiling:8,timeoutMs:48000,expectedPlanPath:item.report,env:{CHECKBOX_CASE:JSON.stringify(item)}})})));
|
||||
const results=await Promise.all(cases.map(async item=>({late:item.late,observation:await runPlanSkillCounting({skillName:'plan-eng-review',slashCommand:'/plan-eng-review',followUpPrompt:'Review this fixture.',isLastStep0AUQ:()=>false,isReviewAUQ:()=>true,reviewCountCeiling:8,timeoutMs:48000,startupReadyMarker:'PTY_READY:'+item.log,expectedPlanPath:item.report,env:{CHECKBOX_CASE:JSON.stringify(item)}})})));
|
||||
await Bun.write(${JSON.stringify(output)},JSON.stringify(results));`);
|
||||
const child = Bun.spawn([process.execPath, worker], {
|
||||
env: { ...process.env, BROWSE_TERMINAL_BINARY: fake, EVALS_HERMETIC: '1' }, stdout: 'pipe', stderr: 'pipe',
|
||||
|
||||
@@ -343,10 +343,11 @@ process.stdin.on('data', data => {
|
||||
process.stdout.write('407 +NO UNRESOLVED DECISIONS\n●' + text.replace(/ /g, '') + '\nCrunched for 10m 9s ·done 5:49PM\n❯ ');
|
||||
});
|
||||
process.stdin.resume();
|
||||
process.stdout.write('PTY_READY:'+process.env.PROBE_INPUTS+'\x1b[2J\x1b[H');
|
||||
`);
|
||||
fs.chmodSync(fake, 0o755);
|
||||
fs.writeFileSync(worker, `import { runPlanSkillCounting } from ${JSON.stringify(runner)};\n` +
|
||||
`const result = await runPlanSkillCounting({skillName:'plan-devex-review',slashCommand:'/plan-devex-review',followUpPrompt:'# Native completion fixture',expectedPlanPath:${JSON.stringify(output)},isLastStep0AUQ:()=>false,isReviewAUQ:()=>true,reviewCountCeiling:8,timeoutMs:33000,env:${JSON.stringify({PROBE_PLAN:output,PROBE_INPUTS:record,PROBE_REPORT:REPORT})}});\n` +
|
||||
`const result = await runPlanSkillCounting({skillName:'plan-devex-review',slashCommand:'/plan-devex-review',followUpPrompt:'# Native completion fixture',expectedPlanPath:${JSON.stringify(output)},isLastStep0AUQ:()=>false,isReviewAUQ:()=>true,reviewCountCeiling:8,timeoutMs:33000,startupReadyMarker:${JSON.stringify('PTY_READY:'+record)},env:${JSON.stringify({PROBE_PLAN:output,PROBE_INPUTS:record,PROBE_REPORT:REPORT})}});\n` +
|
||||
`await Bun.write(${JSON.stringify(result)},JSON.stringify(result));\n`);
|
||||
const child = Bun.spawn([process.execPath, worker], {
|
||||
env: { ...process.env, EVALS_HERMETIC: '1', EVALS_RUN_ID: '', BROWSE_TERMINAL_BINARY: fake },
|
||||
@@ -595,10 +596,11 @@ process.stdin.on('data', data => {
|
||||
} else if (stage === 2) event('unexpected-plan-approval-input');
|
||||
});
|
||||
process.stdin.resume();
|
||||
process.stdout.write('PTY_READY:'+process.env.PROBE_INPUTS+'\x1b[2J\x1b[H');
|
||||
`);
|
||||
fs.chmodSync(fake, 0o755);
|
||||
fs.writeFileSync(worker, `import { runPlanSkillCounting } from ${JSON.stringify(runner)};\n` +
|
||||
`const result = await runPlanSkillCounting({skillName:'plan-eng-review',slashCommand:'/plan-eng-review',followUpPrompt:'# Completion fixture',expectedPlanPath:${JSON.stringify(output)},isLastStep0AUQ:()=>false,isReviewAUQ:()=>true,reviewCountCeiling:8,timeoutMs:43000,env:${JSON.stringify({PROBE_PLAN:output,PROBE_INPUTS:record,PROBE_EVENTS:events,PROBE_REPORT:REPORT,PROBE_TERMINAL:terminal})}});\n` +
|
||||
`const result = await runPlanSkillCounting({skillName:'plan-eng-review',slashCommand:'/plan-eng-review',followUpPrompt:'# Completion fixture',expectedPlanPath:${JSON.stringify(output)},isLastStep0AUQ:()=>false,isReviewAUQ:()=>true,reviewCountCeiling:8,timeoutMs:43000,startupReadyMarker:${JSON.stringify('PTY_READY:'+record)},env:${JSON.stringify({PROBE_PLAN:output,PROBE_INPUTS:record,PROBE_EVENTS:events,PROBE_REPORT:REPORT,PROBE_TERMINAL:terminal})}});\n` +
|
||||
`await Bun.write(${JSON.stringify(result)},JSON.stringify(result));\n`);
|
||||
const child = Bun.spawn([process.execPath, worker], { env: { ...process.env,
|
||||
EVALS_HERMETIC: '1', EVALS_RUN_ID: '', BROWSE_TERMINAL_BINARY: fake }, stdout: 'pipe', stderr: 'pipe' });
|
||||
|
||||
@@ -103,8 +103,9 @@ process.stdin.setRawMode?.(true);process.stdin.on('data',async data=>{
|
||||
native('assistant',[{type:'tool_use',name:'AskUserQuestion',id:'finding',input:{questions:[q]}}]);native('user',[{type:'tool_result',tool_use_id:'finding',content:'Answered'}],{toolUseResult:{answers:{[q.question]:'Fix'}}});
|
||||
process.stdout.write('\x1b[2J\x1b[HDone.\r\n');
|
||||
});process.on('SIGINT',()=>process.exit(0));process.stdin.resume();
|
||||
process.stdout.write('PTY_READY:'+item.events+'\x1b[2J\x1b[H');
|
||||
`);fs.chmodSync(fake,0o755);
|
||||
fs.writeFileSync(worker,`import {runPlanSkillCounting} from ${JSON.stringify(pathToFileURL(path.join(import.meta.dir,'helpers/claude-pty-runner.ts')).href)};const o=await runPlanSkillCounting({skillName:'plan-ceo-review',slashCommand:'/plan-ceo-review',followUpPrompt:'Review the disposable fixture.',expectedPlanPath:${JSON.stringify(expected)},isLastStep0AUQ:()=>false,isReviewAUQ:()=>true,reviewCountCeiling:1,timeoutMs:28000,env:{FILE_EPOCH_CASE:${JSON.stringify(JSON.stringify({events,expected,screen,intervening,activePlan:variant==='same-basename'}))}}});await Bun.write(${JSON.stringify(output)},JSON.stringify(o));`);
|
||||
fs.writeFileSync(worker,`import {runPlanSkillCounting} from ${JSON.stringify(pathToFileURL(path.join(import.meta.dir,'helpers/claude-pty-runner.ts')).href)};const o=await runPlanSkillCounting({skillName:'plan-ceo-review',slashCommand:'/plan-ceo-review',followUpPrompt:'Review the disposable fixture.',expectedPlanPath:${JSON.stringify(expected)},isLastStep0AUQ:()=>false,isReviewAUQ:()=>true,reviewCountCeiling:1,timeoutMs:28000,startupReadyMarker:${JSON.stringify('PTY_READY:'+events)},env:{FILE_EPOCH_CASE:${JSON.stringify(JSON.stringify({events,expected,screen,intervening,activePlan:variant==='same-basename'}))}}});await Bun.write(${JSON.stringify(output)},JSON.stringify(o));`);
|
||||
const child=Bun.spawn([process.execPath,worker],{env:{...process.env,BROWSE_TERMINAL_BINARY:fake,EVALS_HERMETIC:'1'},stdout:'pipe',stderr:'pipe'});const killer=setTimeout(()=>child.kill('SIGKILL'),33000);
|
||||
try{const [code,out,err]=await Promise.all([child.exited,new Response(child.stdout).text(),new Response(child.stderr).text()]);expect(code,out+err).toBe(0);
|
||||
const o=JSON.parse(fs.readFileSync(output,'utf8'));expect(o.outcome,JSON.stringify(o)).toBe('ceiling_reached');expect(o.reviewCount).toBe(1);
|
||||
|
||||
@@ -7,6 +7,10 @@ import * as path from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { createPlanCountFixture } from './helpers/plan-count-fixture';
|
||||
import { getHermeticDirs } from './helpers/hermetic-env';
|
||||
import { nativePlanCallFingerprint } from './helpers/claude-pty-runner';
|
||||
import { isDesignCountFirstReview } from './helpers/design-count-review';
|
||||
import { isDesignUIScopeReview } from './helpers/design-ui-scope';
|
||||
import designUICapture from './fixtures/plan-design-ui-scope.json';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const PROMPT = '# Seeded settings plan\n\nReview each issue separately.\n' +
|
||||
@@ -166,6 +170,12 @@ try {
|
||||
const cases = [
|
||||
{ name: 'design', skillName: 'plan-design-review', prompt: PROMPT, mode: 'complete', files: { 'DESIGN.md': '# Approved design\nKeep the existing layout.\n' } },
|
||||
{ name: 'design-direct', skillName: 'plan-design-review', prompt: PROMPT, mode: 'direct-finding' },
|
||||
{ name: 'design-named-target', skillName: 'plan-design-review', prompt: fs.readFileSync(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'), 'utf8'), mode: 'direct-finding', namedTarget: true },
|
||||
{ name: 'design-tool-diagnostic', skillName: 'plan-design-review', prompt: PROMPT, mode: 'direct-finding', toolDiagnostic: true },
|
||||
{ name: 'design-gate-positive', skillName: 'plan-design-review', prompt: PROMPT, mode: 'direct-finding', gateFilter: true },
|
||||
{ name: 'design-ui-captured', skillName: 'plan-design-review', prompt: fs.readFileSync(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'), 'utf8'), mode: 'direct-finding', gateFilter: true, namedTarget: true, capturedQuestions: designUICapture.calls[3]!.questions },
|
||||
{ name: 'design-ui-captured-separators', skillName: 'plan-design-review', prompt: fs.readFileSync(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'), 'utf8'), mode: 'direct-finding', gateFilter: true, namedTarget: true, capturedQuestions: designUICapture.additionalCaptures[0]!.calls[2]!.questions },
|
||||
{ name: 'design-ui-captured-decision', skillName: 'plan-design-review', prompt: fs.readFileSync(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'), 'utf8'), mode: 'direct-finding', gateFilter: true, namedTarget: true, capturedQuestions: [designUICapture.additionalQuestionCaptures[0]!.question] },
|
||||
{ name: 'design-batched', skillName: 'plan-design-review', prompt: PROMPT, mode: 'batched-finding' },
|
||||
{ name: 'failed-native', skillName: 'plan-design-review', prompt: PROMPT, mode: 'failed-call' },
|
||||
{ name: 'native-permission-policy', skillName: 'plan-eng-review', prompt: PROMPT, mode: 'native-permission-policy', report: path.join(dir, 'native-policy-report.md') },
|
||||
@@ -414,6 +424,13 @@ process.stdin.on('data', (data) => {
|
||||
}
|
||||
firstInput = false;
|
||||
if (process.env.FIXTURE_MODE === 'exit') process.exit(7);
|
||||
if (process.env.FIXTURE_TOOL_DIAGNOSTIC === 'true') render('Unknown command: --help\n');
|
||||
if (process.env.FIXTURE_GATE_FILTER === 'true') {
|
||||
ask([questionMetadata('Focus', 'D2 — Review all 7 design dimensions, or focus on specific areas?', ['All 7 dimensions', 'Choose areas'])]);
|
||||
answer();
|
||||
ask([questionMetadata('Outside voices', 'D3 — Want outside design voices before the detailed review?\nA fresh reviewer checks completeness. <gstack-qid:outside-voices-design>', ['Yes, run outside voices (recommended)', 'No, proceed without'])]);
|
||||
answer();
|
||||
}
|
||||
if (process.env.FIXTURE_MODE === 'damaged-menu') {
|
||||
render('☐Stripe event types\nWhich event should the handler accept?\n❯1.Specify one canonical event\n2.Accept all events\n' +
|
||||
'·'.repeat(4200) + '\nMinimum required test cases:\n1.Happy path\n2.Email failure\n3.DB timeout\n4.Unknown event\n5.Unknown user\n❯1\n');
|
||||
@@ -443,9 +460,16 @@ process.stdin.on('data', (data) => {
|
||||
ask([questionMetadata('Missing answer', 'Should the save retry be idempotent?', ['Yes', 'No'])]);
|
||||
native('user', [{ type: 'tool_result', tool_use_id: 'question-' + callId, is_error: true, content: 'Question rejected' }]);
|
||||
}
|
||||
const questions = [questionMetadata('Button style', 'D1 — How should the four header buttons differ? <gstack-qid:plan-design-review-button-hierarchy>', ['Filled primary', 'Ghost buttons'])];
|
||||
const detail = process.env.FIXTURE_GATE_FILTER === 'true' ? ' Specify the primary button treatment.'.repeat(30) : '';
|
||||
const questions = process.env.FIXTURE_CAPTURED_QUESTIONS ? JSON.parse(process.env.FIXTURE_CAPTURED_QUESTIONS)
|
||||
: [questionMetadata('Button style', 'D1 — How should the four header buttons differ?' + detail + ' <gstack-qid:plan-design-review-button-hierarchy>', ['Filled primary', 'Ghost buttons'])];
|
||||
if (process.env.FIXTURE_MODE === 'batched-finding') questions.push(questionMetadata('Loading', 'D2 — Define the loading state <gstack-qid:plan-design-review-loading>', ['Add spinner', 'Keep blank']));
|
||||
ask(questions);
|
||||
if (process.env.FIXTURE_CAPTURED_QUESTIONS) {
|
||||
const q = questions[0];
|
||||
render('\r☐' + q.header + '\r' + q.question.split('\n')[0] + '\r' + q.options.map((o, i) => (i === 0 ? '❯' : '') + (i + 1) + '.' + o.label).join('\r') + '\r');
|
||||
return;
|
||||
}
|
||||
render('\r☐Buttonstyle\r│D1—Howshouldthe4headerbuttonsbedifferentiated?<gstack-qid:plan-design-review-butn-hierarchy>\r❯1.Filledprimary\r2.Ghostbuttons\r');
|
||||
return;
|
||||
}
|
||||
@@ -485,10 +509,14 @@ process.stdin.resume();
|
||||
const runnerUrl = pathToFileURL(path.join(ROOT, 'test/helpers/claude-pty-runner.ts')).href;
|
||||
const hermeticUrl = pathToFileURL(path.join(ROOT, 'test/helpers/hermetic-env.ts')).href;
|
||||
const devexUrl = pathToFileURL(path.join(ROOT, 'test/helpers/devex-count-fixture.ts')).href;
|
||||
const designUrl = pathToFileURL(path.join(ROOT, 'test/helpers/design-count-review.ts')).href;
|
||||
const designUIUrl = pathToFileURL(path.join(ROOT, 'test/helpers/design-ui-scope.ts')).href;
|
||||
fs.writeFileSync(workerPath, `
|
||||
import { runPlanSkillCounting, designFirstReviewAUQ } from ${JSON.stringify(runnerUrl)};
|
||||
import { getHermeticDirs } from ${JSON.stringify(hermeticUrl)};
|
||||
import { devexReviewModePick } from ${JSON.stringify(devexUrl)};
|
||||
import { isDesignCountFirstReview } from ${JSON.stringify(designUrl)};
|
||||
import { isDesignUIScopeReview } from ${JSON.stringify(designUIUrl)};
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
const shared = getHermeticDirs().gstackHome;
|
||||
@@ -500,23 +528,25 @@ const results = await Promise.all(cases.map(async (item) => ({
|
||||
name: item.name,
|
||||
observation: await runPlanSkillCounting({
|
||||
skillName: item.skillName,
|
||||
slashCommand: '/' + item.skillName,
|
||||
slashCommand: '/' + item.skillName + (item.namedTarget ? ' PLAN.md' : ''),
|
||||
followUpPrompt: item.prompt,
|
||||
fixtureFiles: item.files,
|
||||
expectedPlanPath: item.report,
|
||||
isLastStep0AUQ: () => false,
|
||||
isLastStep0AUQ: item.gateFilter ? fp => fp.nativeCall?.questions[0]?.header === 'Focus' : () => false,
|
||||
isFirstReviewAUQ: ['direct-finding', 'batched-finding', 'failed-call'].includes(item.mode) ? designFirstReviewAUQ : undefined,
|
||||
isReviewAUQ: item.custom ? fp => fp.promptSnippet.includes('routing-proof-after-240') : undefined,
|
||||
isReviewAUQ: item.capturedQuestions ? isDesignUIScopeReview : item.gateFilter ? isDesignCountFirstReview : item.custom ? fp => fp.promptSnippet.includes('routing-proof-after-240') : undefined,
|
||||
pickAUQ: item.mode === 'native-permission-policy' ? () => 2
|
||||
: ['late-mode', 'batched-mode'].includes(item.mode) ? devexReviewModePick
|
||||
: item.custom ? fp => fp.promptSnippet.includes('routing-proof-after-240') ? 1 : null : undefined,
|
||||
reviewCountCeiling: 8,
|
||||
reviewCountCeiling: item.gateFilter ? 1 : 8,
|
||||
timeoutMs: item.mode === 'permission-lifecycle' ? 35000 : 28000,
|
||||
firstAUQPick: () => ['late-mode', 'batched-mode'].includes(item.mode) ? 1 : 2,
|
||||
env: {
|
||||
FIXTURE_RECORD: item.record, FIXTURE_SKILL: item.skillName, FIXTURE_MODE: item.mode,
|
||||
FIXTURE_EXPECTED_REPORT: item.report ?? '',
|
||||
FIXTURE_CUSTOM: String(item.custom ?? false),
|
||||
FIXTURE_TOOL_DIAGNOSTIC: String(item.toolDiagnostic ?? false), FIXTURE_GATE_FILTER: String(item.gateFilter ?? false),
|
||||
FIXTURE_CAPTURED_QUESTIONS: item.capturedQuestions ? JSON.stringify(item.capturedQuestions) : '',
|
||||
FIXTURE_SKIP_INDEX: String(item.skipIndex ?? ''), FIXTURE_CONFIG_BIN: ${JSON.stringify(path.join(ROOT, 'bin/gstack-config'))},
|
||||
FIXTURE_SKIP_LABEL: item.skipLabel ?? '',
|
||||
GSTACK_HOME: ${JSON.stringify(hostState)}, GSTACK_STATE_ROOT: ${JSON.stringify(hostState)},
|
||||
@@ -575,7 +605,7 @@ await Bun.write(${JSON.stringify(resultPath)}, JSON.stringify({ results, onboard
|
||||
expect(startup.skill).toContain(`name: ${item.skillName}`);
|
||||
expect(startup.sections).toBe(fs.readFileSync(path.join(ROOT, item.skillName, 'sections/review-sections.md'), 'utf8'));
|
||||
expect(events.filter((event) => event.type === 'input').map((event) => event.data).join(''))
|
||||
.toBe(`/${item.skillName}\r` + (item.mode === 'prerequisite' ? `${item.custom ? '1\r' : '2'}${item.skipIndex}`
|
||||
.toBe(`/${item.skillName}${item.namedTarget ? ' PLAN.md' : ''}\r` + (item.mode === 'prerequisite' ? `${item.custom ? '1\r' : '2'}${item.skipIndex}`
|
||||
: item.mode === 'permission-lifecycle' ? '1\r1\r2'
|
||||
: item.mode === 'damaged-submit' ? '\x1b[Z2\r\r'
|
||||
: item.mode === 'batched-finding' ? '2\r1\r' : item.mode === 'batched-mode' ? '1\r'
|
||||
@@ -586,7 +616,7 @@ await Bun.write(${JSON.stringify(resultPath)}, JSON.stringify({ results, onboard
|
||||
expect(() => process.kill(startup.pid, 0)).toThrow();
|
||||
const result = results.find((result) => result.name === item.name);
|
||||
expect(result.observation.outcome, `${item.name}: ${JSON.stringify(result.observation)}`).toBe(item.mode === 'exit' ? 'exited'
|
||||
: ['missing-transcript', 'failed-call'].includes(item.mode) ? 'transcript_unavailable' : 'completion_summary');
|
||||
: ['missing-transcript', 'failed-call'].includes(item.mode) ? 'transcript_unavailable' : item.gateFilter ? 'ceiling_reached' : 'completion_summary');
|
||||
const artifacts = result.observation.artifactDir;
|
||||
expect(result.observation.artifactError).toBeUndefined();
|
||||
expect(fs.existsSync(artifacts)).toBe(true); // Survives the temporary fixture's cleanup.
|
||||
@@ -595,12 +625,21 @@ await Bun.write(${JSON.stringify(resultPath)}, JSON.stringify({ results, onboard
|
||||
expect(captured.capture.cwd).toBe(startup.cwd);
|
||||
expect(fs.readFileSync(path.join(artifacts, 'terminal.raw.log'), 'utf8')).toContain(item.mode === 'exit' ? 'STARTUP_DIAGNOSTIC' : 'GSTACK REVIEW REPORT');
|
||||
expect(fs.readFileSync(path.join(artifacts, 'terminal.visible.log'), 'utf8')).toContain(item.mode === 'exit' ? 'STARTUP_DIAGNOSTIC' : 'GSTACK REVIEW REPORT');
|
||||
if (['direct-finding', 'batched-finding', 'failed-call'].includes(item.mode)) {
|
||||
if (['direct-finding', 'batched-finding', 'failed-call'].includes(item.mode) && !item.gateFilter) {
|
||||
expect(result.observation.reviewCount).toBe(1);
|
||||
expect(result.observation.step0Count).toBe(0);
|
||||
expect(result.observation.fingerprints).toHaveLength(1);
|
||||
expect(result.observation.fingerprints[0].nativeCall.questions).toHaveLength(item.mode === 'batched-finding' ? 2 : 1);
|
||||
}
|
||||
if (item.gateFilter) {
|
||||
expect(result.observation.reviewCount).toBe(1);
|
||||
expect(result.observation.step0Count).toBe(2);
|
||||
expect(result.observation.fingerprints.map(fp => fp.preReview)).toEqual([true, true, false]);
|
||||
expect(result.observation.fingerprints.at(-1).nativeCall.questions[0].header).toBe(item.capturedQuestions?.[0].header ?? 'Button style');
|
||||
const finding = result.observation.fingerprints.at(-1);
|
||||
expect(finding.promptSnippet.length).toBe(240);
|
||||
expect((item.capturedQuestions ? isDesignUIScopeReview : isDesignCountFirstReview)(nativePlanCallFingerprint(finding.nativeCall, finding.observedAtMs, finding.preReview))).toBe(true);
|
||||
}
|
||||
if (item.mode === 'damaged-menu') {
|
||||
expect(events.filter(event => event.type === 'input-during-prose')).toEqual([]);
|
||||
expect(events.filter(event => event.type === 'damaged-menu-answer').map(event => event.input)).toEqual(['2\r']);
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
import { expect, spyOn, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { isUnknownSlashCommandVisible, launchClaudePty, runPlanSkillCounting, type ClaudePtySession } from './helpers/claude-pty-runner';
|
||||
|
||||
test('unknown-command diagnostics identify the invoked slash command, not child tools', () => {
|
||||
for (const command of ['/plan-design-review', '/plan-design-review PLAN.md']) {
|
||||
expect(isUnknownSlashCommandVisible('Unknown command: /plan-design-review\n', command)).toBe(true);
|
||||
expect(isUnknownSlashCommandVisible('Unknown command: /other\nUnknown command: /plan-design-review', command)).toBe(true);
|
||||
expect(isUnknownSlashCommandVisible('Unknown command: --help\n', command)).toBe(false);
|
||||
expect(isUnknownSlashCommandVisible('Unknown command: /plan-design-review-other\n', command)).toBe(false);
|
||||
expect(isUnknownSlashCommandVisible('Unknown command: /other\n', command)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test.skipIf(process.platform === 'win32')('PTY output and exit wake observers without leaving deadline timers', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-pty-output-'));
|
||||
const fake = path.join(dir, 'fake-claude');
|
||||
fs.writeFileSync(fake, `#!${process.execPath}
|
||||
process.stdin.setRawMode(true);
|
||||
process.stdin.on('data', bytes => process.stdout.write(bytes));
|
||||
process.on('SIGINT', () => process.exit(0));
|
||||
process.stdin.resume();
|
||||
process.stdout.write('READY');
|
||||
`, { mode: 0o755 });
|
||||
const originalBinary = process.env.BROWSE_TERMINAL_BINARY;
|
||||
let session: ClaudePtySession | undefined;
|
||||
try {
|
||||
process.env.BROWSE_TERMINAL_BINARY = fake;
|
||||
session = await launchClaudePty({ cwd: dir, timeoutMs: 10_000 });
|
||||
} finally {
|
||||
if (originalBinary === undefined) delete process.env.BROWSE_TERMINAL_BINARY;
|
||||
else process.env.BROWSE_TERMINAL_BINARY = originalBinary;
|
||||
if (!session) fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
const schedules: Array<{ timer: ReturnType<typeof setTimeout>; delay: number; fired: boolean }> = [];
|
||||
const originalTimeout = globalThis.setTimeout;
|
||||
let scheduleSpy: ReturnType<typeof spyOn> | undefined;
|
||||
let clearSpy: ReturnType<typeof spyOn> | undefined;
|
||||
try {
|
||||
await session.waitFor('READY', { timeoutMs: 5000 });
|
||||
scheduleSpy = spyOn(globalThis, 'setTimeout').mockImplementation(((fn: (...args: any[]) => void, delay: number, ...args: any[]) => {
|
||||
const entry = { timer: undefined as unknown as ReturnType<typeof setTimeout>, delay, fired: false };
|
||||
entry.timer = originalTimeout(() => { entry.fired = true; fn(...args); }, delay);
|
||||
schedules.push(entry);
|
||||
return entry.timer;
|
||||
}) as typeof setTimeout);
|
||||
clearSpy = spyOn(globalThis, 'clearTimeout');
|
||||
|
||||
const marker = session.mark();
|
||||
const update = session.waitForOutput(marker, 1000);
|
||||
const updateTimer = schedules.at(-1)!;
|
||||
session.send('changed');
|
||||
await update;
|
||||
expect(session.visibleSince(marker)).toContain('changed');
|
||||
expect(updateTimer.fired).toBe(false);
|
||||
expect(clearSpy).toHaveBeenCalledWith(updateTimer.timer);
|
||||
|
||||
const scheduledBeforeBufferedRead = schedules.length;
|
||||
await session.waitForOutput(marker, 1000);
|
||||
expect(schedules).toHaveLength(scheduledBeforeBufferedRead);
|
||||
|
||||
const unchanged = session.mark();
|
||||
await session.waitForOutput(unchanged, 10);
|
||||
expect(schedules.at(-1)!.fired).toBe(true);
|
||||
expect(session.visibleSince(unchanged)).toBe('');
|
||||
|
||||
const exited = session.waitForOutput(session.mark(), 1000);
|
||||
const exitTimer = schedules.at(-1)!;
|
||||
await Promise.all([exited, session.close()]);
|
||||
expect(session.exited()).toBe(true);
|
||||
expect(exitTimer.fired).toBe(false);
|
||||
const closeTimer = schedules.find(entry => entry.delay === 2000)!;
|
||||
expect(closeTimer).toBeDefined();
|
||||
expect(closeTimer.fired).toBe(false);
|
||||
for (const entry of schedules) expect(clearSpy).toHaveBeenCalledWith(entry.timer);
|
||||
|
||||
const scheduledBeforeExitedRead = schedules.length;
|
||||
await session.waitForOutput(session.mark(), 1000);
|
||||
expect(schedules).toHaveLength(scheduledBeforeExitedRead);
|
||||
} finally {
|
||||
try { await session.close(); }
|
||||
finally {
|
||||
scheduleSpy?.mockRestore();
|
||||
clearSpy?.mockRestore();
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
test.skipIf(process.platform === 'win32')('split terminal redraws settle before routing question input', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-pty-split-redraw-'));
|
||||
const fake = path.join(dir, 'fake-claude');
|
||||
const input = path.join(dir, 'input.txt');
|
||||
const ready = `PTY_READY:${dir}`;
|
||||
fs.writeFileSync(fake, `#!${process.execPath}
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
const project = path.join(process.env.CLAUDE_CONFIG_DIR, 'projects', 'split-redraw');
|
||||
fs.mkdirSync(project, {recursive:true});
|
||||
fs.writeFileSync(path.join(project, 'split-redraw.jsonl'), JSON.stringify({
|
||||
cwd:process.cwd(), sessionId:'split-redraw', isSidechain:false,
|
||||
message:{role:'assistant',content:[{type:'text',text:'Fixture CLI started.'}]},
|
||||
}) + '\\n');
|
||||
let started = false;
|
||||
process.stdin.setRawMode(true);
|
||||
process.stdin.on('data', bytes => {
|
||||
fs.appendFileSync(${JSON.stringify(input)}, bytes);
|
||||
if (started) return;
|
||||
started = true;
|
||||
process.stdout.write('☐Stripe event types\\nWhich event should the handler accept?\\n❯1.Specify one canonical event\\n2.Accept all events\\n');
|
||||
setTimeout(() => process.stdout.write('·'.repeat(4200) + '\\nMinimum required test cases:\\n1.Happy path\\n2.Email failure\\n3.DB timeout\\n4.Unknown event\\n5.Unknown user\\n❯1\\n'), 30);
|
||||
setTimeout(() => process.stdout.write('\\x1b[2J\\x1b[HGSTACK REVIEW REPORT\\n'), 700);
|
||||
});
|
||||
process.on('SIGINT', () => process.exit(0));
|
||||
process.stdin.resume();
|
||||
process.stdout.write(${JSON.stringify(ready)} + '\\x1b[2J\\x1b[H');
|
||||
`, { mode: 0o755 });
|
||||
const originalBinary = process.env.BROWSE_TERMINAL_BINARY;
|
||||
try {
|
||||
process.env.BROWSE_TERMINAL_BINARY = fake;
|
||||
const result = await runPlanSkillCounting({
|
||||
skillName: 'plan-ceo-review', slashCommand: '/plan-ceo-review', followUpPrompt: '# Split redraw fixture',
|
||||
isLastStep0AUQ: () => false, reviewCountCeiling: 1, timeoutMs: 9000, startupReadyMarker: ready,
|
||||
});
|
||||
expect(result.outcome).toBe('completion_summary');
|
||||
expect(fs.readFileSync(input, 'utf8')).toBe('/plan-ceo-review\r');
|
||||
} finally {
|
||||
if (originalBinary === undefined) delete process.env.BROWSE_TERMINAL_BINARY;
|
||||
else process.env.BROWSE_TERMINAL_BINARY = originalBinary;
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}, 12_000);
|
||||
|
||||
test.skipIf(process.platform === 'win32')('a missing startup-ready marker sends no command and cleans the owned PTY fixture', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-pty-not-ready-'));
|
||||
const fake = path.join(dir, 'fake-claude');
|
||||
const record = path.join(dir, 'started.json');
|
||||
const input = path.join(dir, 'input.txt');
|
||||
fs.writeFileSync(fake, `#!${process.execPath}
|
||||
import * as fs from 'node:fs';
|
||||
fs.writeFileSync(${JSON.stringify(record)}, JSON.stringify({pid:process.pid,cwd:process.cwd()}));
|
||||
process.stdin.setRawMode(true);
|
||||
process.stdin.on('data', bytes => fs.appendFileSync(${JSON.stringify(input)}, bytes));
|
||||
process.on('SIGINT', () => process.exit(0));
|
||||
process.stdin.resume();
|
||||
process.stdout.write('BOOTING, NOT READY');
|
||||
`, { mode: 0o755 });
|
||||
const originalBinary = process.env.BROWSE_TERMINAL_BINARY;
|
||||
try {
|
||||
process.env.BROWSE_TERMINAL_BINARY = fake;
|
||||
await expect(runPlanSkillCounting({
|
||||
skillName: 'plan-ceo-review', slashCommand: '/plan-ceo-review', followUpPrompt: '# Owned startup fixture',
|
||||
isLastStep0AUQ: () => false, reviewCountCeiling: 1, timeoutMs: 6500,
|
||||
startupReadyMarker: `PTY_READY:${dir}`,
|
||||
})).rejects.toThrow();
|
||||
expect(fs.existsSync(input)).toBe(false);
|
||||
const started = JSON.parse(fs.readFileSync(record, 'utf8'));
|
||||
expect(() => process.kill(started.pid, 0)).toThrow();
|
||||
expect(fs.existsSync(started.cwd)).toBe(false);
|
||||
} finally {
|
||||
if (originalBinary === undefined) delete process.env.BROWSE_TERMINAL_BINARY;
|
||||
else process.env.BROWSE_TERMINAL_BINARY = originalBinary;
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}, 10_000);
|
||||
|
||||
test('close releases observers even when the child never reports exit', async () => {
|
||||
const originalBinary = process.env.BROWSE_TERMINAL_BINARY;
|
||||
const signals: string[] = [];
|
||||
const spawnSpy = spyOn(Bun, 'spawn').mockImplementation((() => ({
|
||||
exited: new Promise(() => {}),
|
||||
kill: (signal: string) => { signals.push(signal); },
|
||||
terminal: { write() {} },
|
||||
})) as typeof Bun.spawn);
|
||||
let session: ClaudePtySession | undefined;
|
||||
let closed: Promise<void> | undefined;
|
||||
let deadline: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
process.env.BROWSE_TERMINAL_BINARY = process.execPath;
|
||||
session = await launchClaudePty({ timeoutMs: 10_000 });
|
||||
const output = session.waitForOutput(session.mark(), 10_000);
|
||||
closed = session.close();
|
||||
await Promise.race([output, new Promise<void>((_, reject) => {
|
||||
deadline = setTimeout(() => reject(new Error('close did not release the output waiter')), 500);
|
||||
})]);
|
||||
clearTimeout(deadline);
|
||||
await closed;
|
||||
expect(signals).toEqual(['SIGINT', 'SIGKILL']);
|
||||
expect(session.exited()).toBe(false);
|
||||
await session.waitForOutput(session.mark(), 10_000);
|
||||
} finally {
|
||||
clearTimeout(deadline);
|
||||
try { await (closed ?? session?.close()); }
|
||||
finally {
|
||||
spawnSpy.mockRestore();
|
||||
if (originalBinary === undefined) delete process.env.BROWSE_TERMINAL_BINARY;
|
||||
else process.env.BROWSE_TERMINAL_BINARY = originalBinary;
|
||||
}
|
||||
}
|
||||
}, 10_000);
|
||||
|
||||
test.skipIf(process.platform === 'win32')('continuous PTY redraws coalesce expensive observations', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-pty-redraw-'));
|
||||
const fake = path.join(dir, 'fake-claude');
|
||||
const worker = path.join(dir, 'worker.ts');
|
||||
const ready = `PTY_READY:${dir}`;
|
||||
fs.writeFileSync(fake, `#!${process.execPath}
|
||||
let spinner;
|
||||
process.stdin.setRawMode(true);
|
||||
process.stdin.on('data', () => {
|
||||
spinner ??= setInterval(() => process.stdout.write('\\rWorking ' + performance.now()), 5);
|
||||
});
|
||||
process.on('SIGINT', () => process.exit(0));
|
||||
process.stdin.resume();
|
||||
process.stdout.write(${JSON.stringify(ready)} + '\\x1b[2J\\x1b[H');
|
||||
`, { mode: 0o755 });
|
||||
const screenUrl = pathToFileURL(path.join(import.meta.dir, 'helpers/pty-screen.ts')).href;
|
||||
const runnerUrl = pathToFileURL(path.join(import.meta.dir, 'helpers/claude-pty-runner.ts')).href;
|
||||
fs.writeFileSync(worker, `import {mock} from 'bun:test';
|
||||
const {createPtyScreen} = await import(${JSON.stringify(screenUrl)});
|
||||
const observations = [];
|
||||
mock.module(${JSON.stringify(screenUrl)}, () => ({createPtyScreen: async (...args) => {
|
||||
const screen = await createPtyScreen(...args);
|
||||
return {...screen, read: async () => { observations.push(performance.now()); return screen.read(); }};
|
||||
}}));
|
||||
const {runPlanSkillCounting} = await import(${JSON.stringify(runnerUrl)});
|
||||
const start = performance.now();
|
||||
const result = await runPlanSkillCounting({skillName:'plan-ceo-review',slashCommand:'/plan-ceo-review',
|
||||
followUpPrompt:'# Continuous redraw fixture',isLastStep0AUQ:()=>false,reviewCountCeiling:1,
|
||||
timeoutMs:9000,startupReadyMarker:${JSON.stringify(ready)}});
|
||||
console.log(JSON.stringify({outcome:result.outcome,elapsedMs:performance.now()-start,reads:observations.length}));
|
||||
`);
|
||||
const child = Bun.spawn([process.execPath, worker], {
|
||||
env: { ...process.env, BROWSE_TERMINAL_BINARY: fake, EVALS_HERMETIC: '1', EVALS_RUN_ID: '' },
|
||||
stdout: 'pipe', stderr: 'pipe',
|
||||
});
|
||||
const deadline = setTimeout(() => child.kill('SIGKILL'), 12_000);
|
||||
try {
|
||||
const [code, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]);
|
||||
expect(code, stdout + stderr).toBe(0);
|
||||
const result = JSON.parse(stdout.trim().split('\n').at(-1)!);
|
||||
expect(result.outcome).toBe('timeout');
|
||||
expect(result.reads).toBeGreaterThanOrEqual(3);
|
||||
expect(result.reads).toBeLessThanOrEqual(Math.ceil(result.elapsedMs / 250) + 1);
|
||||
} finally {
|
||||
clearTimeout(deadline);
|
||||
if (child.exitCode === null) { child.kill('SIGKILL'); await child.exited; }
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}, 15_000);
|
||||
@@ -11,25 +11,21 @@
|
||||
* exit — would pass the no-UI test (vacuously) and ship undetected. This
|
||||
* test is the positive coverage.
|
||||
*
|
||||
* How: launch claude in plan mode in the gstack repo cwd (so the skill
|
||||
* registry is loaded). Send /plan-design-review with the fixture path
|
||||
* inline so the skill reviews the UI-heavy plan rather than git diff or
|
||||
* .claude/plans/. Drive past permission dialogs. Wait for a numbered-
|
||||
* option list that is NOT a permission dialog. Assert evidence does NOT
|
||||
* contain "no UI scope".
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { PTY_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'path';
|
||||
import {
|
||||
launchClaudePty,
|
||||
isNumberedOptionListVisible,
|
||||
isPermissionDialogVisible,
|
||||
parseNumberedOptions,
|
||||
isPlanReadyVisible,
|
||||
runPlanSkillCounting,
|
||||
designStep0Boundary,
|
||||
nativePlanCallFingerprint,
|
||||
} from './helpers/claude-pty-runner';
|
||||
import { isDesignCountSetup, isDesignCompletionHandoff, pickDesignCountQuestion } from './helpers/design-count-review';
|
||||
import { isDesignArtifactGeneration } from './helpers/design-artifact-question';
|
||||
import { isDesignUIScopeReview } from './helpers/design-ui-scope';
|
||||
|
||||
const describeE2E = describeE2ETier('gate');
|
||||
|
||||
@@ -40,116 +36,37 @@ describeE2E('/plan-design-review with UI scope (gate)', () => {
|
||||
test(
|
||||
'reaches a real skill AskUserQuestion (or plan_ready) without echoing the no-UI early-exit phrase',
|
||||
async () => {
|
||||
const fixtureRelPath = path.relative(ROOT, FIXTURE);
|
||||
|
||||
const session = await launchClaudePty({
|
||||
permissionMode: 'plan',
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — skill registry,
|
||||
// hermetic pre-trusted dir, and the repo-relative fixture path above.
|
||||
cwd: ROOT,
|
||||
timeoutMs: PTY_MS,
|
||||
seedSkills: true,
|
||||
const observation = await runPlanSkillCounting({
|
||||
skillName: 'plan-design-review',
|
||||
slashCommand: '/plan-design-review PLAN.md',
|
||||
followUpPrompt: fs.readFileSync(FIXTURE, 'utf8'),
|
||||
isLastStep0AUQ: designStep0Boundary,
|
||||
isFirstReviewAUQ: isDesignUIScopeReview,
|
||||
isReviewAUQ: isDesignUIScopeReview,
|
||||
isSetupAUQ: isDesignCountSetup,
|
||||
isCompletionHandoffAUQ: isDesignCompletionHandoff,
|
||||
isArtifactGenerationAUQ: isDesignArtifactGeneration,
|
||||
pickAUQ: pickDesignCountQuestion,
|
||||
reviewCountCeiling: 1,
|
||||
timeoutMs: 600_000,
|
||||
});
|
||||
|
||||
let outcome: 'real_question' | 'plan_ready' | 'timeout' | 'exited' = 'timeout';
|
||||
let evidence = '';
|
||||
let debugBuffer = ''; // captured at end so timeout error has data
|
||||
|
||||
try {
|
||||
await Bun.sleep(8000);
|
||||
const since = session.mark();
|
||||
// Send the slash command alone first; then provide the UI-heavy
|
||||
// plan content as a follow-up message. Claude Code rejects slash
|
||||
// commands with trailing arguments unless the skill defines them.
|
||||
session.send('/plan-design-review\r');
|
||||
await Bun.sleep(3000);
|
||||
session.send(
|
||||
`Please review this plan for UI scope:\n\n` +
|
||||
`Title: User Dashboard Page\n` +
|
||||
`New React page UserDashboard.tsx with three subcomponents: ` +
|
||||
`ActivityFeed, NotificationsPanel, QuickActions. ` +
|
||||
`Tailwind CSS responsive layout (mobile/desktop breakpoints), ` +
|
||||
`loading skeletons, empty states, hover states on every interactive element, ` +
|
||||
`modal dialog for "mark all read", toast notifications for action feedback. ` +
|
||||
`Reference plan file: ${fixtureRelPath}\r`
|
||||
);
|
||||
|
||||
// 600s, not 360s: the skill preamble (update-check, session bookkeeping,
|
||||
// learnings) plus extended model thinking can take ~6 minutes before the
|
||||
// scope-gate AskUserQuestion renders — a 360s budget expired seconds
|
||||
// before the (correct) AUQ appeared in the observed failure transcript.
|
||||
const budgetMs = 600_000;
|
||||
const start = Date.now();
|
||||
let lastPermSig = '';
|
||||
while (Date.now() - start < budgetMs) {
|
||||
await Bun.sleep(2500);
|
||||
if (session.exited()) {
|
||||
outcome = 'exited';
|
||||
evidence = session.visibleSince(since).slice(-3000);
|
||||
break;
|
||||
}
|
||||
const visible = session.visibleSince(since);
|
||||
|
||||
// Classify the recent tail only — old permission text persists
|
||||
// in visibleSince(since) and would otherwise re-trigger forever.
|
||||
// 5KB window: plan-design-review Step 0 renders a numbered AUQ with
|
||||
// box dividers + per-option descriptions + footer prompt. The full
|
||||
// rendering frequently exceeds 2.5KB, especially after TTY cursor-
|
||||
// positioning escapes resolve through stripAnsi. A 2.5KB tail can
|
||||
// capture the cursor `❯1.` line without capturing the line that has
|
||||
// `2.`, defeating isNumberedOptionListVisible. 5KB comfortably
|
||||
// covers the full AUQ block without including stale scrollback.
|
||||
const recentTail = visible.slice(-5000);
|
||||
|
||||
// Real skill AskUserQuestion visible (not a permission dialog)?
|
||||
if (
|
||||
isNumberedOptionListVisible(recentTail) &&
|
||||
parseNumberedOptions(recentTail).length >= 2 &&
|
||||
!isPermissionDialogVisible(recentTail)
|
||||
) {
|
||||
outcome = 'real_question';
|
||||
evidence = visible.slice(-3000);
|
||||
break;
|
||||
}
|
||||
|
||||
// Permission dialog: grant once per unique rendering.
|
||||
if (isPermissionDialogVisible(recentTail)) {
|
||||
const sig = visible.slice(-500);
|
||||
if (sig !== lastPermSig) {
|
||||
lastPermSig = sig;
|
||||
session.send('1\r');
|
||||
await Bun.sleep(1500);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Plan-ready terminal — also acceptable (skill ran end-to-end
|
||||
// and surfaced claude's "Ready to execute" prompt).
|
||||
if (isPlanReadyVisible(visible)) {
|
||||
outcome = 'plan_ready';
|
||||
evidence = visible.slice(-3000);
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Capture buffer state at end so a timeout error has diagnostic data.
|
||||
debugBuffer = session.visibleSince(since).slice(-4000);
|
||||
} finally {
|
||||
await session.close();
|
||||
}
|
||||
|
||||
// PASS: real_question or plan_ready, AND evidence does NOT echo the
|
||||
// early-exit phrase.
|
||||
if (outcome === 'exited' || outcome === 'timeout') {
|
||||
const designQuestionObserved = observation.fingerprints.some(fp =>
|
||||
!fp.preReview && !fp.administrative && fp.nativeCall &&
|
||||
isDesignUIScopeReview(nativePlanCallFingerprint(fp.nativeCall, fp.observedAtMs, fp.preReview)));
|
||||
if ((observation.outcome !== 'ceiling_reached' && observation.outcome !== 'plan_ready') ||
|
||||
observation.reviewCount < 1 || !designQuestionObserved) {
|
||||
throw new Error(
|
||||
`plan-design-review with UI scope FAILED: outcome=${outcome}\n` +
|
||||
`--- buffer at timeout (last 4KB) ---\n${debugBuffer || evidence}`,
|
||||
`plan-design-review with UI scope FAILED: outcome=${observation.outcome}\n` +
|
||||
`step0=${observation.step0Count} review=${observation.reviewCount}\n` +
|
||||
`${observation.summary}\n--- questions ---\n${JSON.stringify(observation.fingerprints, null, 2)}\n` +
|
||||
`--- evidence ---\n${observation.evidence}`,
|
||||
);
|
||||
}
|
||||
const NO_UI_PHRASE = /no\s+UI\s+scope|isn'?t\s+applicable/i;
|
||||
if (NO_UI_PHRASE.test(evidence)) {
|
||||
if (NO_UI_PHRASE.test(observation.evidence)) {
|
||||
throw new Error(
|
||||
`plan-design-review early-exited despite UI-heavy fixture.\n` +
|
||||
`--- evidence (last 3KB) ---\n${evidence}`,
|
||||
`--- evidence (last 3KB) ---\n${observation.evidence}`,
|
||||
);
|
||||
}
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user