#!/usr/bin/env node /** * Sync E2E Test Harness * * This script: * 1. Downloads and starts rclone's S3 server (S3-compatible storage) * 2. Builds and starts donut-sync server * 3. Runs the Rust sync e2e tests * 4. Cleans up all processes * * Usage: node scripts/sync-test-harness.mjs */ import { spawn, spawnSync, execSync } from "child_process"; import { createWriteStream, existsSync, mkdirSync, chmodSync, renameSync, } from "fs"; import { mkdir, rm, writeFile } from "fs/promises"; import http from "http"; import https from "https"; import os from "os"; import path from "path"; import { pipeline } from "stream/promises"; import { fileURLToPath } from "url"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT_DIR = path.resolve(__dirname, ".."); const CACHE_DIR = path.join(ROOT_DIR, ".cache", "sync-test"); const S3_PORT = 9876; const S3_ACCESS_KEY = "donuttestaccesskey"; const S3_SECRET_KEY = "donuttestsecretkey"; // Pinned so a fresh rclone release cannot change what CI runs underneath us. const RCLONE_VERSION = "v1.75.1"; const SYNC_PORT = 3456; // Must be >= 24 chars and not a known default — the server's validateEnv() // rejects short/placeholder tokens and exits at startup otherwise. const SYNC_TOKEN = "test-sync-token-0123456789abcdef"; const processes = []; function log(msg) { console.log(`[sync-harness] ${msg}`); } function error(msg) { console.error(`[sync-harness] ERROR: ${msg}`); } async function downloadFile(url, dest) { return new Promise((resolve, reject) => { const file = createWriteStream(dest); const protocol = url.startsWith("https") ? https : http; protocol .get(url, (response) => { if (response.statusCode === 302 || response.statusCode === 301) { file.close(); downloadFile(response.headers.location, dest) .then(resolve) .catch(reject); return; } if (response.statusCode !== 200) { file.close(); reject(new Error(`Failed to download: ${response.statusCode}`)); return; } // pipeline() resolves once the source ends but doesn't await the // destination fd closing. Linux refuses to exec a file whose write // fd is still open (ETXTBSY), so explicitly wait for 'close'. pipeline(response, file) .then( () => new Promise((res, rej) => { if (file.closed) { res(); } else { file.once("close", res); file.once("error", rej); } }), ) .then(resolve) .catch(reject); }) .on("error", reject); }); } function getRcloneAsset() { const platform = os.platform(); const arch = os.arch(); if (platform === "darwin") { return `rclone-${RCLONE_VERSION}-osx-${arch === "arm64" ? "arm64" : "amd64"}`; } else if (platform === "linux") { return `rclone-${RCLONE_VERSION}-linux-${arch === "arm64" ? "arm64" : "amd64"}`; } else if (platform === "win32") { return `rclone-${RCLONE_VERSION}-windows-amd64`; } throw new Error(`Unsupported platform: ${platform}-${arch}`); } // Mirrors src-tauri/download-xray.mjs: PowerShell owns zip extraction on // Windows, `unzip` everywhere else. Both are present on every runner this // harness targets. function extractZip(archive, destinationDir) { if (os.platform() === "win32") { const result = spawnSync( "powershell", [ "-NoProfile", "-NonInteractive", "-Command", "Expand-Archive -LiteralPath $env:DONUT_RCLONE_ARCHIVE -DestinationPath $env:DONUT_RCLONE_DESTINATION -Force", ], { stdio: "inherit", env: { ...process.env, DONUT_RCLONE_ARCHIVE: archive, DONUT_RCLONE_DESTINATION: destinationDir, }, }, ); if (result.status !== 0) { throw new Error("Failed to extract the rclone archive"); } return; } const result = spawnSync( "unzip", ["-qq", "-j", "-o", archive, "*/rclone", "-d", destinationDir], { stdio: "inherit" }, ); if (result.status !== 0) { throw new Error("Failed to extract the rclone archive"); } } async function ensureS3Binary() { const isWindows = os.platform() === "win32"; const binName = isWindows ? "rclone.exe" : "rclone"; const bin = path.join(CACHE_DIR, binName); if (existsSync(bin)) { log("rclone binary already cached"); return bin; } log("Downloading rclone binary..."); mkdirSync(CACHE_DIR, { recursive: true }); const asset = getRcloneAsset(); const archive = path.join(CACHE_DIR, `${asset}.zip`); await downloadFile( `https://github.com/rclone/rclone/releases/download/${RCLONE_VERSION}/${asset}.zip`, archive, ); extractZip(archive, CACHE_DIR); if (isWindows) { // Expand-Archive keeps the archive's own directory, so lift the binary out. const nested = path.join(CACHE_DIR, asset, binName); if (existsSync(nested)) { renameSync(nested, bin); } } if (!existsSync(bin)) { throw new Error(`rclone archive did not contain ${binName}`); } if (!isWindows) { chmodSync(bin, 0o755); } log("rclone binary downloaded"); return bin; } // MinIO used to play this part, but MinIO withdrew its community binaries and // every dl.min.io path now answers 410, which took this job down. rclone still // publishes a single binary per platform, and `rclone serve s3` speaks enough // of the API for these tests: path-style addressing, a static key pair, and // bucket creation (a bucket is a directory under the served root). async function startS3(bin) { const dataDir = path.join(CACHE_DIR, "s3-data"); await mkdir(dataDir, { recursive: true }); log(`Starting rclone serve s3 on port ${S3_PORT}...`); const proc = spawn( bin, [ "serve", "s3", dataDir, "--addr", `127.0.0.1:${S3_PORT}`, "--auth-key", `${S3_ACCESS_KEY},${S3_SECRET_KEY}`, "--force-path-style", "--vfs-cache-mode", "writes", ], { env: { ...process.env }, stdio: ["ignore", "pipe", "pipe"], }, ); processes.push(proc); proc.stdout.on("data", (data) => { if (process.env.VERBOSE) { console.log(`[s3] ${data.toString().trim()}`); } }); let lastStderr = ""; proc.stderr.on("data", (data) => { lastStderr = data.toString(); if (process.env.VERBOSE) { console.error(`[s3] ${lastStderr.trim()}`); } }); proc.on("error", (err) => { error(`rclone error: ${err.message}`); }); // Without this, a server that refuses to start (a busy port, most often) // surfaces as a bare 30s timeout with the reason buried behind VERBOSE. let exitReason = null; proc.on("exit", (code) => { exitReason = `rclone exited with code ${code}: ${lastStderr.trim() || "no output"}`; }); // No health endpoint: an unauthenticated list is answered once the listener // is up, and any HTTP status proves that much. await waitForListening( `http://127.0.0.1:${S3_PORT}/`, 30000, () => exitReason, ); log("S3 storage is ready"); return proc; } async function buildDonutSync() { log("Building donut-sync..."); // `nest build` runs incremental tsc, which silently skips emit when // tsconfig.build.tsbuildinfo says nothing changed — even if dist/ was // wiped. Drop the cache so we always produce a fresh dist. const syncDir = path.join(ROOT_DIR, "donut-sync"); await rm(path.join(syncDir, "tsconfig.build.tsbuildinfo"), { force: true, }); await rm(path.join(syncDir, "dist"), { recursive: true, force: true }); execSync("pnpm build", { cwd: syncDir, stdio: process.env.VERBOSE ? "inherit" : "ignore", }); if (!existsSync(path.join(syncDir, "dist", "main.js"))) { throw new Error("donut-sync build did not produce dist/main.js"); } log("donut-sync built"); } async function startDonutSync() { log(`Starting donut-sync on port ${SYNC_PORT}...`); const proc = spawn("node", ["dist/main.js"], { cwd: path.join(ROOT_DIR, "donut-sync"), env: { ...process.env, PORT: String(SYNC_PORT), SYNC_TOKEN, S3_ENDPOINT: `http://127.0.0.1:${S3_PORT}`, S3_ACCESS_KEY_ID: S3_ACCESS_KEY, S3_SECRET_ACCESS_KEY: S3_SECRET_KEY, S3_BUCKET: "donut-sync-test", S3_FORCE_PATH_STYLE: "true", }, stdio: ["ignore", "pipe", "pipe"], }); processes.push(proc); proc.stdout.on("data", (data) => { if (process.env.VERBOSE) { console.log(`[donut-sync] ${data.toString().trim()}`); } }); proc.stderr.on("data", (data) => { if (process.env.VERBOSE) { console.error(`[donut-sync] ${data.toString().trim()}`); } }); proc.on("error", (err) => { error(`donut-sync error: ${err.message}`); }); await waitForHealth(`http://localhost:${SYNC_PORT}/health`, 30000); log("donut-sync is ready"); return proc; } async function waitForHealth(url, timeoutMs) { const start = Date.now(); while (Date.now() - start < timeoutMs) { try { await new Promise((resolve, reject) => { http .get(url, (res) => { if (res.statusCode === 200) { resolve(); } else { reject(new Error(`Status ${res.statusCode}`)); } }) .on("error", reject); }); return; } catch { await new Promise((r) => setTimeout(r, 500)); } } throw new Error(`Timeout waiting for ${url}`); } async function waitForListening(url, timeoutMs, failureReason = () => null) { const start = Date.now(); while (Date.now() - start < timeoutMs) { const reason = failureReason(); if (reason) { throw new Error(reason); } try { await new Promise((resolve, reject) => { // A per-attempt timeout matters: something else holding the port can // accept the connection and then never answer, and without this the // loop would wait on that one request forever instead of noticing // that our own server is gone. const req = http.get(url, (res) => { res.resume(); resolve(); }); req.setTimeout(2000, () => { req.destroy(new Error("probe timed out")); }); req.on("error", reject); }); return; } catch { await new Promise((r) => setTimeout(r, 500)); } } throw new Error(`Timeout waiting for ${url}`); } async function runTests() { log("Running Rust sync e2e tests..."); return new Promise((resolve) => { const proc = spawn("cargo", ["test", "--test", "sync_e2e", "--", "--test-threads=1"], { cwd: path.join(ROOT_DIR, "src-tauri"), env: { ...process.env, SYNC_SERVER_URL: `http://localhost:${SYNC_PORT}`, SYNC_TOKEN, }, stdio: "inherit", }); proc.on("close", (code) => { resolve(code || 0); }); }); } function cleanup() { log("Cleaning up..."); for (const proc of processes) { try { if (os.platform() === "win32") { // On Windows, SIGTERM is not supported; use taskkill for reliable cleanup try { execSync(`taskkill /F /T /PID ${proc.pid}`, { stdio: "ignore" }); } catch { // Process may already be dead } } else { proc.kill("SIGTERM"); } } catch { // Already dead } } } async function main() { process.on("SIGINT", () => { cleanup(); process.exit(130); }); process.on("SIGTERM", () => { cleanup(); process.exit(143); }); try { const s3Bin = await ensureS3Binary(); await startS3(s3Bin); await buildDonutSync(); await startDonutSync(); const exitCode = await runTests(); cleanup(); process.exit(exitCode); } catch (err) { error(err.message); cleanup(); process.exit(1); } } main();