mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-09 14:38:59 +02:00
test: touchfiles, tiers, shim E2E, real-engine fixture
Touchfiles: the catalog, contract, detector bin, checklist resolver, review- army resolver, and DESIGN.md lib join the dep lists of review-design-lite, design-review-fix, the design-consultation cases, and plan-design-review-no- ui-scope, so editing any of them re-selects the tests that read their output. Three new E2E keys: design-review-detector-shim (gate; source mode on a feature-branch diff), design-review-detector-shim-dom (gate; DOM mode: the slop fixture served on loopback, dumped through the browse binary with lib/dom-dump.js, persisted under a GSTACK_HOME-scoped REPORT_DIR, scanned once; self-skips when browse/dist/browse is absent), and design-html-slop-gate (periodic; one fix pass, at most two scans, remaining findings accepted with reason). Every case reaches the engine through test/fixtures/fake-impeccable.ts via IMPECCABLE_BIN from outside the temp repo, reads extracted skill sections (never a whole SKILL.md) with the installed bin path rewritten to this checkout, and asserts the probe ran, the right scan verb ran, `npx impeccable` never did, and the output carries FINDING rows tagged [ai-color-palette] and [low-contrast]. review-design-lite gets the fake engine and an eighth tally signal for a detector row; its 4-hit threshold is unchanged. test/gstack-design-detect.test.ts evaluates design-review's REPORT_DIR expression with GSTACK_HOME set and proves a dump under it is accepted by the wrapper's allow-list. The sample fixtures were real captures from commit 1 (engine 0.1.3), so there is nothing hand-written left to swap. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
f1ce46eb8b
commit
9a03aa5e4a
@@ -19,7 +19,7 @@ const ROOT = path.join(import.meta.dir, '..');
|
||||
const TOKEN = /\b(IMPECCABLE_[A-Z_]+|DETECT_[A-Z_]+|DESIGN_MD_[A-Z_]+|DOM_DUMP_[A-Z_]+|DESIGN_DETECTOR_[A-Z_]+|DESIGN_DETECT_[A-Z_]+)\b/g;
|
||||
// Things that look like sentinels but are env vars / flags the prose legitimately names.
|
||||
// Env vars, flags, and resolver placeholder names the prose legitimately names.
|
||||
const NOT_SENTINELS = new Set(['IMPECCABLE_BIN', 'IMPECCABLE_HOME', 'IMPECCABLE_HOOK_DISABLED', 'DESIGN_DETECT_TIMEOUT_MS', 'DESIGN_MD_CHECK', 'DESIGN_DETECTOR']);
|
||||
const NOT_SENTINELS = new Set(['IMPECCABLE_BIN', 'IMPECCABLE_HOME', 'IMPECCABLE_HOOK_DISABLED', 'DESIGN_DETECT_TIMEOUT_MS', 'DESIGN_MD_CHECK', 'DESIGN_DETECTOR', 'IMPECCABLE_INTEROP' /* docs/designs/IMPECCABLE_INTEROP.md */]);
|
||||
|
||||
function* agentReadableFiles(): Generator<string> {
|
||||
const skip = new Set(['node_modules', '.git', 'dist', 'build', 'test', 'docs', '.context', '.claude', '.agents', '.factory', '.cursor', '.kiro', '.opencode', '.openclaw', '.hermes', '.slate', '.gstack', '.gbrain', '.conductor']);
|
||||
|
||||
@@ -57,7 +57,7 @@ describe.skipIf(!POSIX)('parity with bin/gstack-diff-scope', () => {
|
||||
test('SCOPE_FRONTEND agrees with isFrontendPath for every sample, one file per diff', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-scope-parity-'));
|
||||
const git = (...a: string[]) => {
|
||||
const r = spawnSync('git', a, { cwd: dir, encoding: 'utf-8' });
|
||||
const r = spawnSync('git', a, { cwd: dir, encoding: 'utf-8', timeout: 30_000 });
|
||||
if (r.status !== 0) throw new Error(r.stderr);
|
||||
};
|
||||
try {
|
||||
@@ -73,7 +73,7 @@ describe.skipIf(!POSIX)('parity with bin/gstack-diff-scope', () => {
|
||||
fs.mkdirSync(path.dirname(full), { recursive: true });
|
||||
fs.writeFileSync(full, '/* x */\n');
|
||||
git('add', '-A'); git('commit', '-q', '-m', rel);
|
||||
const r = spawnSync('bash', [path.join(ROOT, 'bin', 'gstack-diff-scope'), 'main'], { cwd: dir, encoding: 'utf-8' });
|
||||
const r = spawnSync('bash', [path.join(ROOT, 'bin', 'gstack-diff-scope'), 'main'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 });
|
||||
const bashSays = /SCOPE_FRONTEND=true/.test(r.stdout);
|
||||
if (bashSays !== expected) mismatches.push(`${rel}: bash=${bashSays} ts=${expected}`);
|
||||
git('checkout', '-q', 'main'); git('branch', '-q', '-D', 'probe');
|
||||
|
||||
@@ -453,6 +453,32 @@ describe('scan', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('design-review REPORT_DIR agrees with the allow-list', () => {
|
||||
test.skipIf(!POSIX)('the template expression, evaluated with GSTACK_HOME set, lands under <gstack home>/projects/<slug>/designs/ and a dump there is scanned', () => {
|
||||
const tmpl = fs.readFileSync(path.join(ROOT, 'design-review', 'SKILL.md.tmpl'), 'utf-8');
|
||||
const m = tmpl.match(/^REPORT_DIR="(.+)"$/m);
|
||||
expect(m).not.toBeNull();
|
||||
const expr = m![1];
|
||||
expect(expr.startsWith('${GSTACK_HOME:-$HOME/.gstack}/projects/$SLUG/designs/')).toBe(true);
|
||||
const r = spawnSync('bash', ['-c', `SLUG=my-repo; echo "${expr}"`], { encoding: 'utf-8', timeout: 30_000, env: { PATH: process.env.PATH!, HOME: path.join(SANDBOX, 'fake-home'), GSTACK_HOME } });
|
||||
const reportDir = r.stdout.trim();
|
||||
expect(reportDir.startsWith(path.join(GSTACK_HOME, 'projects', 'my-repo', 'designs', 'design-audit-'))).toBe(true);
|
||||
const dom = path.join(reportDir, 'dom', '120000-1');
|
||||
fs.mkdirSync(dom, { recursive: true });
|
||||
fs.writeFileSync(path.join(dom, 'home.dom.html'), '<html></html>');
|
||||
const log = path.join(SANDBOX, 'argv-report.log');
|
||||
fs.rmSync(log, { force: true });
|
||||
try {
|
||||
const s = run(['scan', '--format', 'gstack', dom + '/home.dom.html'], { env: { IMPECCABLE_BIN: FAKE, FAKE_IMPECCABLE_LOG: log } });
|
||||
expect(s.err).not.toContain(SENTINEL.DETECT_REFUSED);
|
||||
const argv = JSON.parse(fs.readFileSync(log, 'utf-8').trim().split('\n')[0]).argv as string[];
|
||||
expect(argv.slice(2)).toEqual([fs.realpathSync(path.join(dom, 'home.dom.html'))]);
|
||||
} finally {
|
||||
fs.rmSync(path.join(GSTACK_HOME, 'projects'), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('rules', () => {
|
||||
test('prints every mapped id with kind/impact/tier/handoff and the tested engine versions', () => {
|
||||
const r = run(['rules']);
|
||||
|
||||
@@ -66,7 +66,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'review-sql-injection': ['review/**', 'test/fixtures/review-eval-vuln.rb', 'test/skill-e2e-review.test.ts'],
|
||||
'review-enum-completeness': ['review/**', 'test/fixtures/review-eval-enum*.rb', 'test/skill-e2e-review.test.ts'],
|
||||
'review-base-branch': ['review/**', 'test/skill-e2e-review-attribution.test.ts'],
|
||||
'review-design-lite': ['review/**', 'test/fixtures/review-eval-design-slop.*', 'test/skill-e2e-review.test.ts'],
|
||||
'review-design-lite': ['review/**', 'test/fixtures/review-eval-design-slop.*', 'test/fixtures/fake-impeccable.ts', 'test/fixtures/impeccable-detect-sample.json', 'lib/design-catalog.ts', 'lib/design-detect-contract.ts', 'bin/gstack-design-detect.ts', 'scripts/resolvers/design-checklist.ts', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review.test.ts'],
|
||||
|
||||
// Review Army (specialist dispatch)
|
||||
'review-army-migration-safety': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts'],
|
||||
@@ -305,12 +305,16 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
],
|
||||
|
||||
// Design
|
||||
'design-consultation-core': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'design-consultation-existing': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'design-consultation-core': ['design-consultation/**', 'lib/design-catalog.ts', 'lib/design-md.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'design-consultation-existing': ['design-consultation/**', 'lib/design-md.ts', 'bin/gstack-design-md.ts', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'design-consultation-research': ['design-consultation/**', 'scripts/resolvers/aside.ts', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'design-consultation-preview': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'plan-design-review-no-ui-scope': ['plan-design-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'design-review-fix': ['design-review/**', 'scripts/resolvers/aside.ts', 'scripts/resolvers/design.ts', 'browse/src/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'plan-design-review-no-ui-scope': ['plan-design-review/**', 'lib/design-catalog.ts', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'design-review-fix': ['design-review/**', 'scripts/resolvers/aside.ts', 'scripts/resolvers/design.ts', 'lib/design-catalog.ts', 'browse/src/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
|
||||
// Design detector (user-installed impeccable engine) through the fake engine shim: source mode on a diff and DOM mode on a served page.
|
||||
'design-review-detector-shim': ['design-review/**', 'scripts/resolvers/design.ts', 'lib/design-catalog.ts', 'lib/design-detect-contract.ts', 'lib/dom-dump-script.ts', 'lib/dom-dump.js', 'bin/gstack-design-detect.ts', 'test/fixtures/fake-impeccable.ts', 'test/fixtures/impeccable-detect-sample.json', 'test/fixtures/review-eval-design-slop.*', 'test/skill-e2e-design.test.ts'],
|
||||
'design-review-detector-shim-dom': ['design-review/**', 'scripts/resolvers/design.ts', 'lib/design-detect-contract.ts', 'lib/dom-dump-script.ts', 'lib/dom-dump.js', 'bin/gstack-design-detect.ts', 'browse/src/**', 'test/fixtures/fake-impeccable.ts', 'test/fixtures/impeccable-detect-sample.json', 'test/fixtures/review-eval-design-slop.*', 'test/skill-e2e-design.test.ts'],
|
||||
'design-html-slop-gate': ['design-html/**', 'scripts/resolvers/design.ts', 'lib/design-detect-contract.ts', 'bin/gstack-design-detect.ts', 'test/fixtures/fake-impeccable.ts', 'test/fixtures/impeccable-detect-sample.json', 'test/skill-e2e-design.test.ts'],
|
||||
|
||||
// /diagram (diagram-render bundle consumers). Triplet = deterministic
|
||||
// functional (gate); authoring quality = LLM-judged benchmark (periodic).
|
||||
@@ -738,6 +742,9 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
'design-consultation-preview': 'periodic', // D2a demotion 2026-08 ($0.89/481s)
|
||||
'plan-design-review-no-ui-scope': 'gate',
|
||||
'design-review-fix': 'periodic',
|
||||
'design-review-detector-shim': 'gate', // deterministic sentinels from the fake engine (source mode on a diff)
|
||||
'design-review-detector-shim-dom': 'gate', // same shim, DOM mode through the browse binary's dump; self-skips when the binary is absent
|
||||
'design-html-slop-gate': 'periodic', // one-pass gate behavior is a judgment call on a fake engine's fixed output
|
||||
|
||||
// /diagram — triplet is deterministic functional (gstack-render falls back
|
||||
// to the browse daemon, so CI runs it); judge is a quality benchmark
|
||||
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
ROOT, runId, evalsEnabled, selectedTests,
|
||||
describeIfSelected, testConcurrentIfSelected,
|
||||
copyDirSync, logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
createEvalCollector, finalizeEvalCollector, browseBin,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { asideAvailable } from './helpers/aside-available';
|
||||
import { spawnSync } from 'child_process';
|
||||
@@ -701,3 +701,231 @@ Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion call
|
||||
afterAll(async () => {
|
||||
await finalizeEvalCollector(evalCollector);
|
||||
});
|
||||
|
||||
// --- Design detector (impeccable engine shim) E2E ---
|
||||
//
|
||||
// The user-installed impeccable engine is stood in for by test/fixtures/
|
||||
// fake-impeccable.ts (prints the captured detect --json sample, exit 2),
|
||||
// reached through IMPECCABLE_BIN from OUTSIDE the temp repo (the wrapper
|
||||
// ignores an in-repo IMPECCABLE_BIN by design). The skill text the agent reads
|
||||
// is the extracted Setup detector block + Phase 0 (+ the Phase 3 DOM-dump
|
||||
// section for the DOM case), never the 1,500-line SKILL.md, with the installed
|
||||
// bin path pointed at THIS checkout so the test does not depend on ~/.claude.
|
||||
|
||||
const FAKE_ENGINE_SRC = path.join(ROOT, 'test', 'fixtures', 'fake-impeccable.ts');
|
||||
const DETECT_SAMPLE = path.join(ROOT, 'test', 'fixtures', 'impeccable-detect-sample.json');
|
||||
|
||||
function sliceBetween(text: string, start: string, end: string): string {
|
||||
const i = text.indexOf(start);
|
||||
if (i < 0) throw new Error(`marker not found: ${start}`);
|
||||
const j = text.indexOf(end, i + start.length);
|
||||
return text.slice(i, j > i ? j : undefined);
|
||||
}
|
||||
|
||||
/** design-review's detector prose with the installed bin/lib paths rewritten to this checkout. */
|
||||
function detectorSkillText(sections: Array<[string, string]>): string {
|
||||
const full = fs.readFileSync(path.join(ROOT, 'design-review', 'SKILL.md'), 'utf-8');
|
||||
return sections.map(([a, b]) => sliceBetween(full, a, b)).join('\n\n---\n\n')
|
||||
.replaceAll('$HOME/.claude/skills/gstack', ROOT)
|
||||
.replaceAll('~/.claude/skills/gstack', ROOT);
|
||||
}
|
||||
|
||||
function makeFakeEngine(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-fake-impeccable-'));
|
||||
fs.copyFileSync(FAKE_ENGINE_SRC, path.join(dir, 'impeccable'));
|
||||
fs.chmodSync(path.join(dir, 'impeccable'), 0o755);
|
||||
return dir;
|
||||
}
|
||||
|
||||
describeIfSelected('Design review detector shim E2E', ['design-review-detector-shim', 'design-review-detector-shim-dom'], () => {
|
||||
let repoDir: string;
|
||||
let engineDir: string;
|
||||
let server: ReturnType<typeof Bun.serve> | null = null;
|
||||
|
||||
beforeAll(() => {
|
||||
repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-detector-shim-'));
|
||||
const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: repoDir, stdio: 'pipe', timeout: 5000 });
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
fs.writeFileSync(path.join(repoDir, 'index.html'), '<h1>Clean</h1>\n');
|
||||
fs.writeFileSync(path.join(repoDir, 'styles.css'), 'body { font-size: 16px; }\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial']);
|
||||
run('git', ['checkout', '-b', 'feature/landing']);
|
||||
fs.writeFileSync(path.join(repoDir, 'index.html'), fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-design-slop.html'), 'utf-8'));
|
||||
fs.writeFileSync(path.join(repoDir, 'styles.css'), fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-design-slop.css'), 'utf-8'));
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'add landing page']);
|
||||
engineDir = makeFakeEngine();
|
||||
fs.writeFileSync(
|
||||
path.join(repoDir, 'design-review-detector.md'),
|
||||
detectorSkillText([
|
||||
['**Design detector (optional, deterministic):**', '**Create output directories:**'],
|
||||
['**Phase 0: mechanical scan**', '## Phases 1-6'],
|
||||
]),
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(repoDir, 'design-review-dom-dump.md'),
|
||||
detectorSkillText([['### DOM dump (DOM mode only', '### Auth Detection']]),
|
||||
);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
server?.stop(true);
|
||||
try { fs.rmSync(repoDir, { recursive: true, force: true }); } catch {}
|
||||
try { fs.rmSync(engineDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('design-review-detector-shim', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are in a git repo on branch feature/landing with changes against main (the base branch).
|
||||
Read design-review-detector.md: it is the Setup "Design detector" block and "Phase 0: mechanical scan" from /design-review.
|
||||
This is a diff-aware run with no URL, so it is SOURCE mode. Run the probe, then the Phase 0 source-mode scan with base main, exactly as written (use --host claude).
|
||||
Do not run any browser step, do not fix anything, do not run npx.
|
||||
Then write ${repoDir}/detector-output.md: one FINDING-NNN row per rule in the DETECT_TOP block, each tagged with its [rule-id] and the printed impact, plus the first line the probe printed.`,
|
||||
workingDirectory: repoDir,
|
||||
maxTurns: 15,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'design-review-detector-shim',
|
||||
runId,
|
||||
env: { IMPECCABLE_BIN: path.join(engineDir, 'impeccable'), FAKE_IMPECCABLE_OUTPUT: DETECT_SAMPLE },
|
||||
});
|
||||
|
||||
logCost('/design-review detector shim (source)', result);
|
||||
recordE2E(evalCollector, '/design-review detector shim', 'Design review detector shim E2E (source mode)', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
const bash = result.toolCalls.filter(c => c.tool === 'Bash').map(c => String(c.input?.command ?? ''));
|
||||
expect(bash.some(c => c.includes('gstack-design-detect.ts probe'))).toBe(true);
|
||||
expect(bash.some(c => /gstack-design-detect\.ts scan --changed main/.test(c))).toBe(true);
|
||||
expect(bash.some(c => c.includes('npx impeccable'))).toBe(false);
|
||||
expect(result.output).toContain('IMPECCABLE_READY');
|
||||
|
||||
const outPath = path.join(repoDir, 'detector-output.md');
|
||||
expect(fs.existsSync(outPath)).toBe(true);
|
||||
const out = fs.readFileSync(outPath, 'utf-8');
|
||||
expect(out).toContain('FINDING-001');
|
||||
expect(out).toContain('[ai-color-palette]');
|
||||
expect(out).toContain('[low-contrast]');
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// DOM mode needs a browser engine for the dump: gstack's own browse binary
|
||||
// (CI builds it with build:gates). Self-skips when it is absent, like the
|
||||
// other render gates.
|
||||
testConcurrentIfSelected(
|
||||
'design-review-detector-shim-dom',
|
||||
async () => {
|
||||
if (!fs.existsSync(browseBin)) {
|
||||
console.log('design-review-detector-shim (dom mode): browse binary absent, skipping (build it with bun run build:gates)');
|
||||
return;
|
||||
}
|
||||
const site = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-detector-site-'));
|
||||
fs.copyFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-design-slop.html'), path.join(site, 'index.html'));
|
||||
fs.copyFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-design-slop.css'), path.join(site, 'styles.css'));
|
||||
server = Bun.serve({
|
||||
hostname: '127.0.0.1', port: 0,
|
||||
fetch(req) {
|
||||
const p = new URL(req.url).pathname.replace(/^\//, '') || 'index.html';
|
||||
const f = path.join(site, p);
|
||||
return fs.existsSync(f) ? new Response(Bun.file(f)) : new Response('not found', { status: 404 });
|
||||
},
|
||||
});
|
||||
const url = `http://127.0.0.1:${server.port}/index.html`;
|
||||
const reportDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-detector-report-'));
|
||||
const gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-detector-home-'));
|
||||
// REPORT_DIR must sit under <gstack home>/projects/<slug>/designs/ for the wrapper's allow-list.
|
||||
const allowed = path.join(gstackHome, 'projects', 'shim', 'designs', 'design-audit-20260908');
|
||||
fs.mkdirSync(path.join(allowed, 'dom', 'run1'), { recursive: true });
|
||||
try {
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read design-review-detector.md (the /design-review detector block + Phase 0) and design-review-dom-dump.md (the Phase 3 DOM dump section).
|
||||
The target is the URL ${url}, so this is DOM mode: never scan source files.
|
||||
Aside is NOT available; use the fallback browser engine: $B is ${browseBin}. Run "$B goto ${url}" first, then follow the fallback-engine DOM dump steps exactly as written, with {page} = home, REPORT_DIR=${allowed}, RUN_ID=run1, and --host claude. Then run the single scan over ${allowed}/dom/run1 and write ${allowed}/detector-output.md with one FINDING-NNN row per rule in the DETECT_TOP block, each tagged [rule-id], and the line "static scan of the rendered DOM; cross-origin CSS not resolved".
|
||||
Do not run npx. Do not fix anything.`,
|
||||
workingDirectory: repoDir,
|
||||
maxTurns: 25,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'design-review-detector-shim-dom',
|
||||
runId,
|
||||
env: { IMPECCABLE_BIN: path.join(engineDir, 'impeccable'), FAKE_IMPECCABLE_OUTPUT: DETECT_SAMPLE, GSTACK_HOME: gstackHome },
|
||||
});
|
||||
logCost('/design-review detector shim (dom)', result);
|
||||
recordE2E(evalCollector, '/design-review detector shim (dom)', 'Design review detector shim E2E (DOM mode)', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
const bash = result.toolCalls.filter(c => c.tool === 'Bash').map(c => String(c.input?.command ?? ''));
|
||||
expect(bash.some(c => c.includes('dom-dump.js') && c.includes('--out') && c.includes('--raw'))).toBe(true);
|
||||
expect(bash.some(c => /gstack-design-detect\.ts scan /.test(c) && c.includes('dom/run1'))).toBe(true);
|
||||
expect(bash.some(c => /gstack-design-detect\.ts scan --changed/.test(c))).toBe(false);
|
||||
const dumps = fs.readdirSync(path.join(allowed, 'dom', 'run1')).filter(f => f.endsWith('.dom.html'));
|
||||
expect(dumps.length).toBeGreaterThan(0);
|
||||
expect(fs.readFileSync(path.join(allowed, 'dom', 'run1', dumps[0]), 'utf-8')).toContain('data-gstack-dom-css');
|
||||
const out = fs.readFileSync(path.join(allowed, 'detector-output.md'), 'utf-8');
|
||||
expect(out).toContain('[ai-color-palette]');
|
||||
expect(out).toContain('static scan of the rendered DOM');
|
||||
} finally {
|
||||
server?.stop(true); server = null;
|
||||
try { spawnSync(browseBin, ['stop'], { stdio: 'pipe', timeout: 10_000 }); } catch {}
|
||||
for (const d of [site, reportDir, gstackHome]) { try { fs.rmSync(d, { recursive: true, force: true }); } catch {} }
|
||||
}
|
||||
},
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
});
|
||||
|
||||
describeIfSelected('Design HTML slop gate E2E', ['design-html-slop-gate'], () => {
|
||||
let workDir: string;
|
||||
let engineDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-html-gate-'));
|
||||
const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: workDir, stdio: 'pipe', timeout: 5000 });
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
const css = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-design-slop.css'), 'utf-8');
|
||||
const html = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-design-slop.html'), 'utf-8')
|
||||
.replace('<link rel="stylesheet" href="styles.css">', `<style>\n${css}\n</style>`);
|
||||
fs.writeFileSync(path.join(workDir, 'finalized.html'), html);
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'finalized html']);
|
||||
engineDir = makeFakeEngine();
|
||||
const full = fs.readFileSync(path.join(ROOT, 'design-html', 'SKILL.md'), 'utf-8');
|
||||
const text = [
|
||||
sliceBetween(full, '**Design detector (optional, deterministic):**', '<!-- SECTION_INDEX'),
|
||||
sliceBetween(full, '### Slop Gate (bounded, never a loop)', '### Verification Screenshots'),
|
||||
].join('\n\n---\n\n').replaceAll('$HOME/.claude/skills/gstack', ROOT).replaceAll('~/.claude/skills/gstack', ROOT);
|
||||
fs.writeFileSync(path.join(workDir, 'design-html-gate.md'), text);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
try { fs.rmSync(engineDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('design-html-slop-gate', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read design-html-gate.md: the /design-html detector probe block and its "Slop Gate (bounded, never a loop)" step.
|
||||
finalized.html in this directory is the finished page. Run the probe (--host claude), then the slop gate on finalized.html exactly as written: one surgical fix pass over the non-advisory findings, one rescan, then stop.
|
||||
Write ${workDir}/gate-output.md listing what you fixed and every remaining finding as accepted-with-reason, each tagged with its [rule-id]. Do not take screenshots, do not run npx, do not scan more than twice.`,
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 20,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'design-html-slop-gate',
|
||||
runId,
|
||||
env: { IMPECCABLE_BIN: path.join(engineDir, 'impeccable'), FAKE_IMPECCABLE_OUTPUT: DETECT_SAMPLE },
|
||||
});
|
||||
|
||||
logCost('/design-html slop gate', result);
|
||||
recordE2E(evalCollector, '/design-html slop gate', 'Design HTML slop gate E2E', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
const scans = result.toolCalls.filter(c => c.tool === 'Bash' && /gstack-design-detect\.ts scan /.test(String(c.input?.command ?? '')));
|
||||
expect(scans.length).toBeGreaterThanOrEqual(1);
|
||||
expect(scans.length).toBeLessThanOrEqual(2);
|
||||
const outPath = path.join(workDir, 'gate-output.md');
|
||||
expect(fs.existsSync(outPath)).toBe(true);
|
||||
const out = fs.readFileSync(outPath, 'utf-8').toLowerCase();
|
||||
expect(out).toContain('ai-color-palette');
|
||||
expect(out).toContain('accepted');
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -172,6 +172,7 @@ The diff adds a new "returned" status to the Order model. Your job is to check i
|
||||
|
||||
describeIfSelected('Review design lite E2E', ['review-design-lite'], () => {
|
||||
let designDir: string;
|
||||
let fakeEngineDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
designDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-design-lite-'));
|
||||
@@ -206,12 +207,22 @@ describeIfSelected('Review design lite E2E', ['review-design-lite'], () => {
|
||||
extractSkillSections(path.join(ROOT, 'review'), REVIEW_E2E_SECTIONS),
|
||||
);
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(designDir, 'review-checklist.md'));
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'design-checklist.md'), path.join(designDir, 'review-design-checklist.md'));
|
||||
// The checklist's mechanical pass (step 0) runs the design detector from the
|
||||
// installed gstack bin; point it at THIS checkout so the test is hermetic.
|
||||
fs.writeFileSync(
|
||||
path.join(designDir, 'review-design-checklist.md'),
|
||||
fs.readFileSync(path.join(ROOT, 'review', 'design-checklist.md'), 'utf-8').replaceAll('~/.claude/skills/gstack/bin', path.join(ROOT, 'bin')),
|
||||
);
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(designDir, 'review-greptile-triage.md'));
|
||||
// Fake impeccable engine OUTSIDE the repo (the wrapper ignores an in-repo IMPECCABLE_BIN).
|
||||
fakeEngineDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-fake-impeccable-'));
|
||||
fs.copyFileSync(path.join(ROOT, 'test', 'fixtures', 'fake-impeccable.ts'), path.join(fakeEngineDir, 'impeccable'));
|
||||
fs.chmodSync(path.join(fakeEngineDir, 'impeccable'), 0o755);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(designDir, { recursive: true, force: true }); } catch {}
|
||||
try { fs.rmSync(fakeEngineDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('review-design-lite', async () => {
|
||||
@@ -233,6 +244,10 @@ Important: The design checklist should catch issues like blacklisted fonts, smal
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-design-lite',
|
||||
runId,
|
||||
env: {
|
||||
IMPECCABLE_BIN: path.join(fakeEngineDir, 'impeccable'),
|
||||
FAKE_IMPECCABLE_OUTPUT: path.join(ROOT, 'test', 'fixtures', 'impeccable-detect-sample.json'),
|
||||
},
|
||||
});
|
||||
|
||||
logCost('/review design lite', result);
|
||||
@@ -259,8 +274,10 @@ Important: The design checklist should catch issues like blacklisted fonts, smal
|
||||
if (review.includes('welcome to') || review.includes('all-in-one') || review.includes('generic') || review.includes('hero copy') || review.includes('ai slop')) detected++;
|
||||
// Issue 7: 3-column feature grid — LOW
|
||||
if (review.includes('3-column') || review.includes('three-column') || review.includes('feature grid') || review.includes('icon') || review.includes('circle')) detected++;
|
||||
// Signal 8: the mechanical pass (fake impeccable engine via IMPECCABLE_BIN) surfaced a detector row
|
||||
if (review.includes('detector') || review.includes('[ai-color-palette]') || review.includes('[low-contrast]') || review.includes('impeccable')) detected++;
|
||||
|
||||
console.log(`Design review detected ${detected}/7 planted issues`);
|
||||
console.log(`Design review detected ${detected}/8 planted signals`);
|
||||
expect(detected).toBeGreaterThanOrEqual(4);
|
||||
}
|
||||
}, CAPTURE_MS);
|
||||
|
||||
Reference in New Issue
Block a user