Merge remote-tracking branch 'origin/test-audit-reduction' into capy/audit-fix-wave

This commit is contained in:
garrytan committed 2026-09-29 15:09:14 +00:00
commit 44177fffc6
666 files changed
+11963 -102378

No files matched your search

+98 -476
View File
@@ -1,500 +1,122 @@
/**
* Tests for the $D serve command — HTTP server for comparison board feedback.
* Legacy single-process board server (`$D compare --serve --no-daemon`).
*
* Tests the stateful server lifecycle:
* - SERVING → POST submit → DONE (exit 0)
* - SERVING → POST regenerate → REGENERATING → POST reload → SERVING
* - Timeout → exit 1
* - Error handling (missing HTML, malformed JSON, missing reload path)
* Runs the real `serve()` from design/src/serve.ts in a child process on an
* ephemeral port (port 0), because serve() never returns and exits the
* process on submit. The daemon owns the default path (daemon.test.ts); this
* file proves the escape hatch still serves, confines /api/reload to the
* board directory, and exits 0 after writing feedback.json on submit.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { generateCompareHtml } from '../src/compare';
import * as fs from 'fs';
import * as path from 'path';
import { afterAll, describe, expect, test } from "bun:test";
import fs from "fs";
import os from "os";
import path from "path";
let tmpDir: string;
let boardHtml: string;
const SERVE_MODULE = path.resolve(import.meta.dir, "../src/serve.ts");
// Create a minimal 1x1 pixel PNG for test variants
function createTestPng(filePath: string): void {
const png = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/58BAwAI/AL+hc2rNAAAAABJRU5ErkJggg==',
'base64'
);
fs.writeFileSync(filePath, png);
interface RunningServe {
proc: ReturnType<typeof Bun.spawn>;
base: string;
dir: string;
html: string;
}
beforeAll(() => {
tmpDir = '/tmp/serve-test-' + Date.now();
fs.mkdirSync(tmpDir, { recursive: true });
const running: RunningServe[] = [];
// Create test PNGs and generate comparison board
createTestPng(path.join(tmpDir, 'variant-A.png'));
createTestPng(path.join(tmpDir, 'variant-B.png'));
createTestPng(path.join(tmpDir, 'variant-C.png'));
const html = generateCompareHtml([
path.join(tmpDir, 'variant-A.png'),
path.join(tmpDir, 'variant-B.png'),
path.join(tmpDir, 'variant-C.png'),
]);
boardHtml = path.join(tmpDir, 'design-board.html');
fs.writeFileSync(boardHtml, html);
});
async function startServe(): Promise<RunningServe> {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "design-serve-"));
const html = path.join(dir, "board.html");
fs.writeFileSync(html, "<html><body>BOARD_V1</body></html>");
const binDir = path.join(dir, "bin");
fs.mkdirSync(binDir);
for (const opener of ["xdg-open", "open"]) {
fs.writeFileSync(path.join(binDir, opener), "#!/bin/sh\nexit 0\n", { mode: 0o755 });
}
const proc = Bun.spawn(
[process.execPath, "-e", `import { serve } from ${JSON.stringify(SERVE_MODULE)}; await serve({ html: ${JSON.stringify(html)}, port: 0, timeout: 60 });`],
{
env: { ...process.env, PATH: `${binDir}${path.delimiter}${process.env.PATH ?? ""}` },
stdout: "pipe",
stderr: "pipe",
},
);
const reader = proc.stderr.getReader();
const decoder = new TextDecoder();
let seen = "";
const deadline = Date.now() + 15_000;
while (Date.now() < deadline) {
const { value, done } = await reader.read();
if (done) break;
seen += decoder.decode(value);
const match = /SERVE_STARTED: port=(\d+)/.exec(seen);
if (match) {
reader.releaseLock();
const handle = { proc, base: `http://127.0.0.1:${match[1]}`, dir, html };
running.push(handle);
return handle;
}
}
proc.kill();
throw new Error(`serve() never reported SERVE_STARTED:\n${seen}`);
}
afterAll(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
for (const { proc, dir } of running) {
proc.kill();
fs.rmSync(dir, { recursive: true, force: true });
}
});
// ─── Serve as HTTP module (not subprocess) ────────────────────────
describe("design serve() (legacy --no-daemon path)", () => {
test("serves the board, confines /api/reload to the board dir, and exits 0 on submit", async () => {
const s = await startServe();
describe('Serve HTTP endpoints', () => {
let server: ReturnType<typeof Bun.serve>;
let baseUrl: string;
let htmlContent: string;
let state: string;
const page = await fetch(`${s.base}/`);
expect(page.status).toBe(200);
expect(await page.text()).toContain("BOARD_V1");
expect(await (await fetch(`${s.base}/api/progress`)).json()).toEqual({ status: "serving" });
beforeAll(() => {
htmlContent = fs.readFileSync(boardHtml, 'utf-8');
state = 'serving';
server = Bun.serve({
port: 0,
fetch(req) {
const url = new URL(req.url);
if (req.method === 'GET' && url.pathname === '/') {
// Board JS uses relative URLs (./api/feedback, ./api/progress)
// and a location.protocol feature-detect; no injection needed.
return new Response(htmlContent, {
headers: { 'Content-Type': 'text/html; charset=utf-8' },
});
}
if (req.method === 'GET' && url.pathname === '/api/progress') {
return Response.json({ status: state });
}
if (req.method === 'POST' && url.pathname === '/api/feedback') {
return (async () => {
let body: any;
try { body = await req.json(); } catch { return Response.json({ error: 'Invalid JSON' }, { status: 400 }); }
if (typeof body !== 'object' || body === null) return Response.json({ error: 'Expected JSON object' }, { status: 400 });
const isSubmit = body.regenerated === false;
const feedbackFile = isSubmit ? 'feedback.json' : 'feedback-pending.json';
fs.writeFileSync(path.join(tmpDir, feedbackFile), JSON.stringify(body, null, 2));
if (isSubmit) {
state = 'done';
return Response.json({ received: true, action: 'submitted' });
}
state = 'regenerating';
return Response.json({ received: true, action: 'regenerate' });
})();
}
if (req.method === 'POST' && url.pathname === '/api/reload') {
return (async () => {
let body: any;
try { body = await req.json(); } catch { return Response.json({ error: 'Invalid JSON' }, { status: 400 }); }
if (!body.html || !fs.existsSync(body.html)) {
return Response.json({ error: `HTML file not found: ${body.html}` }, { status: 400 });
}
htmlContent = fs.readFileSync(body.html, 'utf-8');
state = 'serving';
return Response.json({ reloaded: true });
})();
}
return new Response('Not found', { status: 404 });
},
const outside = path.join(os.tmpdir(), `design-serve-outside-${process.pid}.html`);
fs.writeFileSync(outside, "SECRET");
try {
const escape = await fetch(`${s.base}/api/reload`, {
method: "POST",
body: JSON.stringify({ html: outside }),
});
expect(escape.status).toBe(403);
} finally {
fs.rmSync(outside, { force: true });
}
const dirReload = await fetch(`${s.base}/api/reload`, {
method: "POST",
body: JSON.stringify({ html: s.dir }),
});
baseUrl = `http://localhost:${server.port}`;
});
expect(dirReload.status).toBe(403);
afterAll(() => {
server.stop();
});
const v2 = path.join(s.dir, "board-v2.html");
fs.writeFileSync(v2, "<html><body>BOARD_V2</body></html>");
const reload = await fetch(`${s.base}/api/reload`, { method: "POST", body: JSON.stringify({ html: v2 }) });
expect(await reload.json()).toEqual({ reloaded: true });
expect(await (await fetch(`${s.base}/`)).text()).toContain("BOARD_V2");
test('GET / serves HTML with relative-path board JS (no injection)', async () => {
const res = await fetch(baseUrl);
expect(res.status).toBe(200);
const html = await res.text();
// No more per-origin URL injection; board JS uses relative paths.
expect(html).not.toContain('__GSTACK_SERVER_URL');
expect(html).not.toContain(baseUrl);
// Board JS calls relative endpoints so the same HTML works at / and at
// /boards/<id>/ (daemon mode).
expect(html).toContain("fetch('./api/feedback'");
expect(html).toContain("fetch('./api/progress')");
expect(html).toContain('Design Exploration');
});
test('GET /api/progress returns current state', async () => {
state = 'serving';
const res = await fetch(`${baseUrl}/api/progress`);
const data = await res.json();
expect(data.status).toBe('serving');
});
test('POST /api/feedback with submit sets state to done', async () => {
state = 'serving';
const feedback = {
preferred: 'A',
ratings: { A: 4, B: 3, C: 2 },
comments: { A: 'Good spacing' },
overall: 'Go with A',
const submit = await fetch(`${s.base}/api/feedback`, {
method: "POST",
body: JSON.stringify({ regenerated: false, preferred: "A" }),
});
expect(await submit.json()).toEqual({ received: true, action: "submitted" });
expect(await s.proc.exited).toBe(0);
expect(JSON.parse(fs.readFileSync(path.join(s.dir, "feedback.json"), "utf-8"))).toEqual({
regenerated: false,
};
const res = await fetch(`${baseUrl}/api/feedback`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(feedback),
preferred: "A",
});
const data = await res.json();
expect(data.received).toBe(true);
expect(data.action).toBe('submitted');
expect(state).toBe('done');
// Verify feedback.json was written
const written = JSON.parse(fs.readFileSync(path.join(tmpDir, 'feedback.json'), 'utf-8'));
expect(written.preferred).toBe('A');
expect(written.ratings.A).toBe(4);
});
test('POST /api/feedback with regenerate sets state and writes feedback-pending.json', async () => {
state = 'serving';
// Clean up any prior pending file
const pendingPath = path.join(tmpDir, 'feedback-pending.json');
if (fs.existsSync(pendingPath)) fs.unlinkSync(pendingPath);
const feedback = {
preferred: 'B',
ratings: { A: 3, B: 5, C: 2 },
comments: {},
overall: null,
regenerated: true,
regenerateAction: 'different',
};
const res = await fetch(`${baseUrl}/api/feedback`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(feedback),
});
const data = await res.json();
expect(data.received).toBe(true);
expect(data.action).toBe('regenerate');
expect(state).toBe('regenerating');
// Progress should reflect regenerating state
const progress = await fetch(`${baseUrl}/api/progress`);
const pd = await progress.json();
expect(pd.status).toBe('regenerating');
// Agent can poll for feedback-pending.json
expect(fs.existsSync(pendingPath)).toBe(true);
const pending = JSON.parse(fs.readFileSync(pendingPath, 'utf-8'));
expect(pending.regenerated).toBe(true);
expect(pending.regenerateAction).toBe('different');
});
test('POST /api/feedback with remix contains remixSpec', async () => {
state = 'serving';
const feedback = {
preferred: null,
ratings: { A: 4, B: 3, C: 3 },
comments: {},
overall: null,
regenerated: true,
regenerateAction: 'remix',
remixSpec: { layout: 'A', colors: 'B', typography: 'C' },
};
const res = await fetch(`${baseUrl}/api/feedback`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(feedback),
});
const data = await res.json();
expect(data.received).toBe(true);
expect(state).toBe('regenerating');
});
test('POST /api/feedback with malformed JSON returns 400', async () => {
const res = await fetch(`${baseUrl}/api/feedback`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: 'not json',
});
expect(res.status).toBe(400);
});
test('POST /api/feedback with non-object returns 400', async () => {
const res = await fetch(`${baseUrl}/api/feedback`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: '"just a string"',
});
expect(res.status).toBe(400);
});
test('POST /api/reload swaps HTML and resets state to serving', async () => {
state = 'regenerating';
// Create a new board HTML
const newBoard = path.join(tmpDir, 'new-board.html');
fs.writeFileSync(newBoard, '<html><body>New board content</body></html>');
const res = await fetch(`${baseUrl}/api/reload`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ html: newBoard }),
});
const data = await res.json();
expect(data.reloaded).toBe(true);
expect(state).toBe('serving');
// Verify the new HTML is served
const pageRes = await fetch(baseUrl);
const pageHtml = await pageRes.text();
expect(pageHtml).toContain('New board content');
});
test('POST /api/reload with missing file returns 400', async () => {
const res = await fetch(`${baseUrl}/api/reload`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ html: '/nonexistent/file.html' }),
});
expect(res.status).toBe(400);
});
test('GET /unknown returns 404', async () => {
const res = await fetch(`${baseUrl}/random-path`);
expect(res.status).toBe(404);
});
});
// ─── Path traversal protection in /api/reload ─────────────────────
describe('Serve /api/reload — path traversal protection', () => {
let server: ReturnType<typeof Bun.serve>;
let baseUrl: string;
let htmlContent: string;
let allowedDir: string;
beforeAll(() => {
// Production-equivalent allowedDir anchored to tmpDir
allowedDir = fs.realpathSync(tmpDir);
htmlContent = fs.readFileSync(boardHtml, 'utf-8');
// This server mirrors the production serve() with the path validation fix
server = Bun.serve({
port: 0,
fetch(req) {
const url = new URL(req.url);
if (req.method === 'GET' && url.pathname === '/') {
return new Response(htmlContent, {
headers: { 'Content-Type': 'text/html; charset=utf-8' },
});
}
if (req.method === 'POST' && url.pathname === '/api/reload') {
return (async () => {
let body: any;
try { body = await req.json(); } catch { return Response.json({ error: 'Invalid JSON' }, { status: 400 }); }
if (!body.html || !fs.existsSync(body.html)) {
return Response.json({ error: `HTML file not found: ${body.html}` }, { status: 400 });
}
// Production path validation — same as design/src/serve.ts
const resolvedReload = fs.realpathSync(path.resolve(body.html));
if (!resolvedReload.startsWith(allowedDir + path.sep)) {
return Response.json({ error: `Path must be within: ${allowedDir}` }, { status: 403 });
}
if (!fs.statSync(resolvedReload).isFile()) {
return Response.json({ error: `Path must be a file, not a directory: ${body.html}` }, { status: 400 });
}
htmlContent = fs.readFileSync(resolvedReload, 'utf-8');
return Response.json({ reloaded: true });
})();
}
return new Response('Not found', { status: 404 });
},
});
baseUrl = `http://localhost:${server.port}`;
});
afterAll(() => {
server.stop();
});
test('blocks reload with path outside allowed directory', async () => {
const res = await fetch(`${baseUrl}/api/reload`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ html: '/etc/passwd' }),
});
expect(res.status).toBe(403);
const data = await res.json();
expect(data.error).toContain('Path must be within');
});
test('blocks reload with symlink pointing outside allowed directory', async () => {
const linkPath = path.join(tmpDir, 'evil-link.html');
try {
fs.symlinkSync('/etc/passwd', linkPath);
const res = await fetch(`${baseUrl}/api/reload`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ html: linkPath }),
});
expect(res.status).toBe(403);
} finally {
try { fs.unlinkSync(linkPath); } catch {}
}
});
test('allows reload with file inside allowed directory', async () => {
const goodPath = path.join(tmpDir, 'safe-board.html');
fs.writeFileSync(goodPath, '<html><body>Safe reload</body></html>');
const res = await fetch(`${baseUrl}/api/reload`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ html: goodPath }),
});
expect(res.status).toBe(200);
const data = await res.json();
expect(data.reloaded).toBe(true);
// Verify the new content is served
const page = await fetch(baseUrl);
expect(await page.text()).toContain('Safe reload');
});
// Regression for the directory-instead-of-file guard (Codex finding).
// Before: resolvedReload === allowedDir passed the guard and then
// readFileSync threw EISDIR with no helpful message.
test('blocks reload when path resolves to the allowed directory itself', async () => {
const res = await fetch(`${baseUrl}/api/reload`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ html: tmpDir }),
});
// tmpDir does not satisfy startsWith(allowedDir + sep), so the within-dir
// check rejects with 403 — but importantly, no EISDIR crash.
expect(res.status).toBe(403);
});
test('blocks reload when path is a subdirectory (not a file)', async () => {
const subdir = path.join(tmpDir, 'subdir-not-a-file');
fs.mkdirSync(subdir, { recursive: true });
try {
const res = await fetch(`${baseUrl}/api/reload`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ html: subdir }),
});
// Inside allowedDir but a directory — must fail before readFileSync,
// with a clear "must be a file" error instead of EISDIR.
expect(res.status).toBe(400);
const data = await res.json();
expect(data.error).toContain('must be a file');
} finally {
try { fs.rmSync(subdir, { recursive: true, force: true }); } catch {}
}
});
});
// ─── Full lifecycle: regeneration round-trip ──────────────────────
describe('Full regeneration lifecycle', () => {
let server: ReturnType<typeof Bun.serve>;
let baseUrl: string;
let htmlContent: string;
let state: string;
beforeAll(() => {
htmlContent = fs.readFileSync(boardHtml, 'utf-8');
state = 'serving';
server = Bun.serve({
port: 0,
fetch(req) {
const url = new URL(req.url);
if (req.method === 'GET' && url.pathname === '/') {
return new Response(htmlContent, { headers: { 'Content-Type': 'text/html' } });
}
if (req.method === 'GET' && url.pathname === '/api/progress') {
return Response.json({ status: state });
}
if (req.method === 'POST' && url.pathname === '/api/feedback') {
return (async () => {
const body = await req.json();
if (body.regenerated) { state = 'regenerating'; return Response.json({ received: true, action: 'regenerate' }); }
state = 'done'; return Response.json({ received: true, action: 'submitted' });
})();
}
if (req.method === 'POST' && url.pathname === '/api/reload') {
return (async () => {
const body = await req.json();
if (body.html && fs.existsSync(body.html)) {
htmlContent = fs.readFileSync(body.html, 'utf-8');
state = 'serving';
return Response.json({ reloaded: true });
}
return Response.json({ error: 'Not found' }, { status: 400 });
})();
}
return new Response('Not found', { status: 404 });
},
});
baseUrl = `http://localhost:${server.port}`;
});
afterAll(() => { server.stop(); });
test('regenerate → reload → submit round-trip', async () => {
// Step 1: User clicks regenerate
expect(state).toBe('serving');
const regen = await fetch(`${baseUrl}/api/feedback`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ regenerated: true, regenerateAction: 'different', preferred: null, ratings: {}, comments: {} }),
});
expect((await regen.json()).action).toBe('regenerate');
expect(state).toBe('regenerating');
// Step 2: Progress shows regenerating
const prog1 = await (await fetch(`${baseUrl}/api/progress`)).json();
expect(prog1.status).toBe('regenerating');
// Step 3: Agent generates new variants and reloads
const newBoard = path.join(tmpDir, 'round2-board.html');
fs.writeFileSync(newBoard, '<html><body>Round 2 variants</body></html>');
const reload = await fetch(`${baseUrl}/api/reload`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ html: newBoard }),
});
expect((await reload.json()).reloaded).toBe(true);
expect(state).toBe('serving');
// Step 4: Progress shows serving (board would auto-refresh)
const prog2 = await (await fetch(`${baseUrl}/api/progress`)).json();
expect(prog2.status).toBe('serving');
// Step 5: User submits on round 2
const submit = await fetch(`${baseUrl}/api/feedback`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ regenerated: false, preferred: 'B', ratings: { A: 3, B: 5 }, comments: {}, overall: 'B is great' }),
});
expect((await submit.json()).action).toBe('submitted');
expect(state).toBe('done');
test("a second server in the same process binds its own ephemeral port", async () => {
const a = await startServe();
const b = await startServe();
expect(a.base).not.toBe(b.base);
expect((await fetch(`${a.base}/`)).status).toBe(200);
expect((await fetch(`${b.base}/`)).status).toBe(200);
});
});