diff --git a/runtime/install.js b/runtime/install.js index d6ae3de77..31de301ac 100644 --- a/runtime/install.js +++ b/runtime/install.js @@ -531,7 +531,8 @@ export async function validateRuntimeBundle(directory, context = {}) { const stat = await fs.lstat(absolute).catch(() => null); if (!stat?.isFile()) throw installError(`Runtime bundle file is missing: ${relative}`, "INSTALL_VALIDATION_FAILED"); const digest = await sha256File(absolute); - if (digest !== file.sha256 || stat.size !== file.size || (stat.mode & 0o777) !== file.mode) { + const modeMatches = (context.platform ?? process.platform) === "win32" || (stat.mode & 0o777) === file.mode; + if (digest !== file.sha256 || stat.size !== file.size || !modeMatches) { throw installError(`Runtime bundle file failed integrity validation: ${relative}`, "INSTALL_VALIDATION_FAILED"); } } diff --git a/runtime/storage.js b/runtime/storage.js index 5a5c2585e..090cd16ad 100644 --- a/runtime/storage.js +++ b/runtime/storage.js @@ -5,6 +5,7 @@ import path from "node:path"; import { randomUUID } from "node:crypto"; const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const WINDOWS_TRANSIENT_FS_ERRORS = new Set(["EACCES", "EBUSY", "EPERM"]); export async function pathExists(file) { try { @@ -83,29 +84,65 @@ async function replaceFile(source, destination) { } } -async function syncDirectory(directory) { +export async function syncDirectory(directory, options = {}) { // Directory fsync is supported on Unix and not consistently on Windows. + const open = options.open ?? fs.open; + let handle; + let operationError; try { - const handle = await fs.open(directory, "r"); + handle = await open(directory, "r"); await handle.sync(); - await handle.close(); } catch (error) { - if (!(["EINVAL", "ENOTSUP", "EISDIR", "EPERM", "EACCES"].includes(error?.code))) { - throw error; + operationError = error; + } + let closeError; + if (handle) { + try { + await handle.close(); + } catch (error) { + closeError = error; } } + const unsupported = operationError && ["EINVAL", "ENOTSUP", "EISDIR", "EPERM", "EACCES"].includes(operationError?.code); + if (operationError && !unsupported) { + if (closeError) throw new AggregateError([operationError, closeError], `Directory sync and close failed: ${directory}`); + throw operationError; + } + if (closeError) throw closeError; } export async function acquireLock(lockPath, options = {}) { const timeoutMs = options.timeoutMs ?? 10_000; const staleMs = options.staleMs ?? 120_000; + const platform = options.platform ?? process.platform; + const mkdir = options.mkdir ?? fs.mkdir; const started = Date.now(); const token = randomUUID(); - await fs.mkdir(path.dirname(lockPath), { recursive: true, mode: 0o700 }); + await mkdir(path.dirname(lockPath), { recursive: true, mode: 0o700 }); for (let attempt = 0; ; attempt += 1) { + let mkdirError; + try { + await mkdir(lockPath, { mode: 0o700 }); + } catch (error) { + mkdirError = error; + } + if (mkdirError) { + const contended = mkdirError?.code === "EEXIST"; + const windowsDeleteRace = platform === "win32" && mkdirError?.code === "EPERM"; + if (!contended && !windowsDeleteRace) throw mkdirError; + if (contended) await reapStaleLock(lockPath, staleMs, platform); + if (Date.now() - started >= timeoutMs) { + const timeout = new Error(`Timed out waiting for lock ${lockPath}`); + timeout.code = "LOCK_TIMEOUT"; + throw timeout; + } + const delay = Math.min(20, 2 + Math.floor(attempt / 3)); + await sleep(delay); + continue; + } + try { - await fs.mkdir(lockPath, { mode: 0o700 }); const owner = { token, pid: process.pid, hostname: os.hostname(), createdAt: new Date().toISOString() }; await atomicWriteJson(path.join(lockPath, "owner.json"), owner, { mode: 0o600 }); const heartbeatMs = Math.max(1_000, Math.min(30_000, Math.floor(staleMs / 3))); @@ -125,35 +162,47 @@ export async function acquireLock(lockPath, options = {}) { if (current?.token === token) await fs.rm(lockPath, { recursive: true, force: true }); }; } catch (error) { - if (error?.code !== "EEXIST") throw error; - await reapStaleLock(lockPath, staleMs); - if (Date.now() - started >= timeoutMs) { - const timeout = new Error(`Timed out waiting for lock ${lockPath}`); - timeout.code = "LOCK_TIMEOUT"; - throw timeout; - } - const delay = Math.min(20, 2 + Math.floor(attempt / 3)); - await sleep(delay); + await fs.rm(lockPath, { recursive: true, force: true }).catch(() => {}); + throw error; } } } -async function reapStaleLock(lockPath, staleMs) { +async function reapStaleLock(lockPath, staleMs, platform = process.platform) { try { const stat = await fs.stat(lockPath); if (Date.now() - stat.mtimeMs <= staleMs) return false; const owner = await readJson(path.join(lockPath, "owner.json"), null).catch(() => null); if (owner?.hostname === os.hostname() && processIsAlive(owner.pid)) return false; const staleName = `${lockPath}.stale-${process.pid}-${randomUUID()}`; - await fs.rename(lockPath, staleName); + await renameWithRetry(lockPath, staleName, { platform }); await fs.rm(staleName, { recursive: true, force: true }); return true; } catch (error) { if (["ENOENT", "EEXIST", "ENOTEMPTY"].includes(error?.code)) return false; + if (platform === "win32" && error?.code === "EPERM") return false; throw error; } } +/** Retry Windows rename races without weakening permanent errors elsewhere. */ +export async function renameWithRetry(source, destination, options = {}) { + const platform = options.platform ?? process.platform; + const rename = options.rename ?? fs.rename; + const timeoutMs = options.timeoutMs ?? 2_000; + const started = Date.now(); + for (let attempt = 0; ; attempt += 1) { + try { + return await rename(source, destination); + } catch (error) { + if (platform !== "win32" || !WINDOWS_TRANSIENT_FS_ERRORS.has(error?.code) || Date.now() - started >= timeoutMs) { + throw error; + } + await sleep(Math.min(50, 5 + attempt * 5)); + } + } +} + function processIsAlive(pid) { if (!Number.isInteger(pid) || pid <= 0) return false; try { diff --git a/runtime/upgrade.js b/runtime/upgrade.js index 931a86171..4437c9e7b 100644 --- a/runtime/upgrade.js +++ b/runtime/upgrade.js @@ -2,7 +2,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { randomUUID } from "node:crypto"; import { assertPathInside, resolveRuntimePaths } from "./paths.js"; -import { atomicWriteJson, pathExists, readJson } from "./storage.js"; +import { atomicWriteJson, pathExists, readJson, renameWithRetry } from "./storage.js"; import { assertManagedHome, ensureManagedHome, @@ -55,7 +55,7 @@ export async function stageUpgradeUnlocked(options) { }, { mode: 0o644 }); await assertTreeContainsNoLinks(stage); if (options.verify) await options.verify(stage); - await fs.rename(stage, destination); + await renameWithRetry(stage, destination); staged = true; } catch (error) { await fs.rm(stage, { recursive: true, force: true }).catch(() => {}); @@ -263,7 +263,7 @@ export async function purgeManagedHomeUnlocked(home) { for (const entry of present.filter((name) => managedEntries.has(name) && !preexisting.has(name))) { const source = assertPathInside(resolved, path.join(resolved, entry)); const destination = assertPathInside(quarantine, path.join(quarantine, entry)); - await fs.rename(source, destination); + await renameWithRetry(source, destination); moved.push({ source, destination }); } await fs.rm(quarantine, { recursive: true, force: true }); @@ -273,7 +273,7 @@ export async function purgeManagedHomeUnlocked(home) { return { purged: true, home: resolved, preserved }; } catch (error) { for (const item of moved.reverse()) { - await fs.rename(item.destination, item.source).catch(() => {}); + await renameWithRetry(item.destination, item.source).catch(() => {}); } await fs.rmdir(quarantine).catch(() => {}); throw error; diff --git a/test/gstack2-runtime-cleanup-boundary.test.ts b/test/gstack2-runtime-cleanup-boundary.test.ts index f8e34961c..0b60bfecd 100644 --- a/test/gstack2-runtime-cleanup-boundary.test.ts +++ b/test/gstack2-runtime-cleanup-boundary.test.ts @@ -7,8 +7,10 @@ import { cleanupRuntime, ensureManagedHome, pathExists, + renameWithRetry, resolveRuntimePaths, runtimeLifecycleLockPath, + syncDirectory, } from "../runtime/index.js"; const roots: string[] = []; @@ -31,6 +33,44 @@ afterEach(async () => { }); describe("runtime cleanup boundary", () => { + test("retries transient Windows lock creation and rename races", async () => { + const home = await temporaryHome(); + const lockPath = path.join(home, "locks", "windows-race.lock"); + let mkdirAttempts = 0; + const release = await acquireLock(lockPath, { + platform: "win32", + mkdir: async (target: string, options: Record) => { + if (target === lockPath && mkdirAttempts++ === 0) { + throw Object.assign(new Error("simulated Windows delete race"), { code: "EPERM" }); + } + return fs.mkdir(target, options); + }, + }); + expect(mkdirAttempts).toBe(2); + await release(); + + let renameAttempts = 0; + await renameWithRetry("source", "destination", { + platform: "win32", + timeoutMs: 100, + rename: async () => { + if (renameAttempts++ < 2) throw Object.assign(new Error("simulated scanner race"), { code: "EPERM" }); + }, + }); + expect(renameAttempts).toBe(3); + }); + + test("closes a directory handle when Windows does not support directory fsync", async () => { + let closes = 0; + await expect(syncDirectory("fixture", { + open: async () => ({ + sync: async () => { throw Object.assign(new Error("unsupported directory sync"), { code: "EPERM" }); }, + close: async () => { closes += 1; }, + }), + })).resolves.toBeUndefined(); + expect(closes).toBe(1); + }); + test("removes only allowlisted stale runtime scratch and dead locks", async () => { const home = await temporaryHome(); const paths = resolveRuntimePaths({ home }); diff --git a/test/gstack2-runtime-core.test.ts b/test/gstack2-runtime-core.test.ts index bc003e1b2..d8cb4fced 100644 --- a/test/gstack2-runtime-core.test.ts +++ b/test/gstack2-runtime-core.test.ts @@ -31,8 +31,10 @@ async function temporaryRoot(label = "gstack2 runtime ") { } afterEach(async () => { - await Promise.all(temporaryRoots.splice(0).map((root) => - fs.chmod(root, 0o700).catch(() => {}).then(() => fs.rm(root, { recursive: true, force: true })))); + for (const root of temporaryRoots.splice(0)) { + await fs.chmod(root, 0o700).catch(() => {}); + await fs.rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }); + } }); describe("gstack 2 host-neutral paths and state", () => { @@ -92,10 +94,11 @@ describe("gstack 2 host-neutral paths and state", () => { gitDir: path.join(root, "repo", ".git"), }); await initializeProject(home, identity); - await Promise.all(Array.from({ length: 60 }, () => + const updates = await Promise.allSettled(Array.from({ length: 60 }, () => updateProjectState(home, identity.projectId, (state) => { state.concurrentCounter = Number(state.concurrentCounter ?? 0) + 1; }))); + expect(updates.filter((result) => result.status === "rejected")).toEqual([]); const { state } = await inspectProject(home, identity); expect(state.concurrentCounter).toBe(60); expect(state.revision).toBe(60); diff --git a/test/gstack2-runtime-install.test.ts b/test/gstack2-runtime-install.test.ts index ad750fdb8..d8cb3c49f 100644 --- a/test/gstack2-runtime-install.test.ts +++ b/test/gstack2-runtime-install.test.ts @@ -389,6 +389,7 @@ describe("GStack 2 managed runtime installer", () => { const cli = path.join(result.path, "runtime", "cli.js"); const originalMode = (await fs.stat(cli)).mode & 0o777; await fs.chmod(cli, originalMode === 0o600 ? 0o644 : 0o600); + await expect(validateRuntimeBundle(result.path, { version: "2.0.0", platform: "win32" })).resolves.toBe(true); await expect(validateRuntimeBundle(result.path, { version: "2.0.0" })).rejects.toMatchObject({ code: "INSTALL_VALIDATION_FAILED", }); diff --git a/test/gstack2-runtime-safety-config.test.ts b/test/gstack2-runtime-safety-config.test.ts index adfbeb3e7..7c9981a1f 100644 --- a/test/gstack2-runtime-safety-config.test.ts +++ b/test/gstack2-runtime-safety-config.test.ts @@ -164,7 +164,7 @@ describe("managed-home destructive boundary", () => { describe("one config authority", () => { test("compatibility helper and runtime config share config.json", async () => { const home = path.join(await root(), "state"); - const run = (args: string[]) => spawnSync(configBin, args, { + const run = (args: string[]) => spawnSync(process.execPath, [configBin, ...args], { encoding: "utf8", env: { ...process.env, GSTACK_HOME: home }, }); @@ -201,20 +201,20 @@ describe("one config authority", () => { const home = path.join(await root(), "legacy"); await fs.mkdir(home); await fs.writeFile(path.join(home, "config.yaml"), "telemetry: community\n"); - const get = spawnSync(configBin, ["get", "telemetry"], { + const get = spawnSync(process.execPath, [configBin, "get", "telemetry"], { encoding: "utf8", env: { ...process.env, GSTACK_HOME: home }, }); expect(get.status).toBe(0); expect(get.stdout).toBe("community"); - const set = spawnSync(configBin, ["set", "telemetry", "off"], { + const set = spawnSync(process.execPath, [configBin, "set", "telemetry", "off"], { encoding: "utf8", env: { ...process.env, GSTACK_HOME: home }, }); expect(set.status).toBe(0); expect(await fs.readFile(path.join(home, "config.yaml"), "utf8")).toBe("telemetry: community\n"); expect(JSON.parse(await fs.readFile(path.join(home, "config.json"), "utf8")).telemetry).toBe("off"); - const reread = spawnSync(configBin, ["get", "telemetry"], { + const reread = spawnSync(process.execPath, [configBin, "get", "telemetry"], { encoding: "utf8", env: { ...process.env, GSTACK_HOME: home }, }); diff --git a/test/gstack2-runtime-workflow-state.test.ts b/test/gstack2-runtime-workflow-state.test.ts index 251ca2219..dc40d7baf 100644 --- a/test/gstack2-runtime-workflow-state.test.ts +++ b/test/gstack2-runtime-workflow-state.test.ts @@ -42,8 +42,9 @@ async function fixture(label = "gstack workflow state ", initialize = true) { } afterEach(async () => { - await Promise.all(temporaryRoots.splice(0).map((root) => - fs.rm(root, { recursive: true, force: true }))); + for (const root of temporaryRoots.splice(0)) { + await fs.rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }); + } }); describe("GStack 2 authoritative workflow state", () => { @@ -150,7 +151,7 @@ describe("GStack 2 authoritative workflow state", () => { test("all workflow mutations are locked and concurrent evidence writes are not lost", async () => { const { home, identity } = await fixture(); await beginRun(home, identity.projectId, "qa", { runId: "run_concurrent" }); - await Promise.all(Array.from({ length: 24 }, (_, index) => + const updates = await Promise.allSettled(Array.from({ length: 24 }, (_, index) => updateRunWorkflow(home, identity.projectId, "run_concurrent", { addEvidenceProvenance: { source: "local-test", @@ -158,6 +159,7 @@ describe("GStack 2 authoritative workflow state", () => { capturedAt: `2026-07-16T10:${String(index).padStart(2, "0")}:00.000Z`, }, }))); + expect(updates.filter((result) => result.status === "rejected")).toEqual([]); const inspected = await inspectRun(home, identity.projectId, "run_concurrent"); expect(inspected.reconstruction.evidenceProvenance).toHaveLength(24); expect(new Set(inspected.reconstruction.evidenceProvenance.map((entry: any) => entry.reference)).size).toBe(24);