mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-21 04:10:47 +02:00
stabilize runtime state on native Windows
This commit is contained in:
+2
-1
@@ -531,7 +531,8 @@ export async function validateRuntimeBundle(directory, context = {}) {
|
||||
const stat = await fs.lstat(absolute).catch(() => null);
|
||||
if (!stat?.isFile()) throw installError(`Runtime bundle file is missing: ${relative}`, "INSTALL_VALIDATION_FAILED");
|
||||
const digest = await sha256File(absolute);
|
||||
if (digest !== file.sha256 || stat.size !== file.size || (stat.mode & 0o777) !== file.mode) {
|
||||
const modeMatches = (context.platform ?? process.platform) === "win32" || (stat.mode & 0o777) === file.mode;
|
||||
if (digest !== file.sha256 || stat.size !== file.size || !modeMatches) {
|
||||
throw installError(`Runtime bundle file failed integrity validation: ${relative}`, "INSTALL_VALIDATION_FAILED");
|
||||
}
|
||||
}
|
||||
|
||||
+67
-18
@@ -5,6 +5,7 @@ import path from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const WINDOWS_TRANSIENT_FS_ERRORS = new Set(["EACCES", "EBUSY", "EPERM"]);
|
||||
|
||||
export async function pathExists(file) {
|
||||
try {
|
||||
@@ -83,29 +84,65 @@ async function replaceFile(source, destination) {
|
||||
}
|
||||
}
|
||||
|
||||
async function syncDirectory(directory) {
|
||||
export async function syncDirectory(directory, options = {}) {
|
||||
// Directory fsync is supported on Unix and not consistently on Windows.
|
||||
const open = options.open ?? fs.open;
|
||||
let handle;
|
||||
let operationError;
|
||||
try {
|
||||
const handle = await fs.open(directory, "r");
|
||||
handle = await open(directory, "r");
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
} catch (error) {
|
||||
if (!(["EINVAL", "ENOTSUP", "EISDIR", "EPERM", "EACCES"].includes(error?.code))) {
|
||||
throw error;
|
||||
operationError = error;
|
||||
}
|
||||
let closeError;
|
||||
if (handle) {
|
||||
try {
|
||||
await handle.close();
|
||||
} catch (error) {
|
||||
closeError = error;
|
||||
}
|
||||
}
|
||||
const unsupported = operationError && ["EINVAL", "ENOTSUP", "EISDIR", "EPERM", "EACCES"].includes(operationError?.code);
|
||||
if (operationError && !unsupported) {
|
||||
if (closeError) throw new AggregateError([operationError, closeError], `Directory sync and close failed: ${directory}`);
|
||||
throw operationError;
|
||||
}
|
||||
if (closeError) throw closeError;
|
||||
}
|
||||
|
||||
export async function acquireLock(lockPath, options = {}) {
|
||||
const timeoutMs = options.timeoutMs ?? 10_000;
|
||||
const staleMs = options.staleMs ?? 120_000;
|
||||
const platform = options.platform ?? process.platform;
|
||||
const mkdir = options.mkdir ?? fs.mkdir;
|
||||
const started = Date.now();
|
||||
const token = randomUUID();
|
||||
await fs.mkdir(path.dirname(lockPath), { recursive: true, mode: 0o700 });
|
||||
await mkdir(path.dirname(lockPath), { recursive: true, mode: 0o700 });
|
||||
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
let mkdirError;
|
||||
try {
|
||||
await mkdir(lockPath, { mode: 0o700 });
|
||||
} catch (error) {
|
||||
mkdirError = error;
|
||||
}
|
||||
if (mkdirError) {
|
||||
const contended = mkdirError?.code === "EEXIST";
|
||||
const windowsDeleteRace = platform === "win32" && mkdirError?.code === "EPERM";
|
||||
if (!contended && !windowsDeleteRace) throw mkdirError;
|
||||
if (contended) await reapStaleLock(lockPath, staleMs, platform);
|
||||
if (Date.now() - started >= timeoutMs) {
|
||||
const timeout = new Error(`Timed out waiting for lock ${lockPath}`);
|
||||
timeout.code = "LOCK_TIMEOUT";
|
||||
throw timeout;
|
||||
}
|
||||
const delay = Math.min(20, 2 + Math.floor(attempt / 3));
|
||||
await sleep(delay);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.mkdir(lockPath, { mode: 0o700 });
|
||||
const owner = { token, pid: process.pid, hostname: os.hostname(), createdAt: new Date().toISOString() };
|
||||
await atomicWriteJson(path.join(lockPath, "owner.json"), owner, { mode: 0o600 });
|
||||
const heartbeatMs = Math.max(1_000, Math.min(30_000, Math.floor(staleMs / 3)));
|
||||
@@ -125,35 +162,47 @@ export async function acquireLock(lockPath, options = {}) {
|
||||
if (current?.token === token) await fs.rm(lockPath, { recursive: true, force: true });
|
||||
};
|
||||
} catch (error) {
|
||||
if (error?.code !== "EEXIST") throw error;
|
||||
await reapStaleLock(lockPath, staleMs);
|
||||
if (Date.now() - started >= timeoutMs) {
|
||||
const timeout = new Error(`Timed out waiting for lock ${lockPath}`);
|
||||
timeout.code = "LOCK_TIMEOUT";
|
||||
throw timeout;
|
||||
}
|
||||
const delay = Math.min(20, 2 + Math.floor(attempt / 3));
|
||||
await sleep(delay);
|
||||
await fs.rm(lockPath, { recursive: true, force: true }).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function reapStaleLock(lockPath, staleMs) {
|
||||
async function reapStaleLock(lockPath, staleMs, platform = process.platform) {
|
||||
try {
|
||||
const stat = await fs.stat(lockPath);
|
||||
if (Date.now() - stat.mtimeMs <= staleMs) return false;
|
||||
const owner = await readJson(path.join(lockPath, "owner.json"), null).catch(() => null);
|
||||
if (owner?.hostname === os.hostname() && processIsAlive(owner.pid)) return false;
|
||||
const staleName = `${lockPath}.stale-${process.pid}-${randomUUID()}`;
|
||||
await fs.rename(lockPath, staleName);
|
||||
await renameWithRetry(lockPath, staleName, { platform });
|
||||
await fs.rm(staleName, { recursive: true, force: true });
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (["ENOENT", "EEXIST", "ENOTEMPTY"].includes(error?.code)) return false;
|
||||
if (platform === "win32" && error?.code === "EPERM") return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Retry Windows rename races without weakening permanent errors elsewhere. */
|
||||
export async function renameWithRetry(source, destination, options = {}) {
|
||||
const platform = options.platform ?? process.platform;
|
||||
const rename = options.rename ?? fs.rename;
|
||||
const timeoutMs = options.timeoutMs ?? 2_000;
|
||||
const started = Date.now();
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
try {
|
||||
return await rename(source, destination);
|
||||
} catch (error) {
|
||||
if (platform !== "win32" || !WINDOWS_TRANSIENT_FS_ERRORS.has(error?.code) || Date.now() - started >= timeoutMs) {
|
||||
throw error;
|
||||
}
|
||||
await sleep(Math.min(50, 5 + attempt * 5));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function processIsAlive(pid) {
|
||||
if (!Number.isInteger(pid) || pid <= 0) return false;
|
||||
try {
|
||||
|
||||
+4
-4
@@ -2,7 +2,7 @@ import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { assertPathInside, resolveRuntimePaths } from "./paths.js";
|
||||
import { atomicWriteJson, pathExists, readJson } from "./storage.js";
|
||||
import { atomicWriteJson, pathExists, readJson, renameWithRetry } from "./storage.js";
|
||||
import {
|
||||
assertManagedHome,
|
||||
ensureManagedHome,
|
||||
@@ -55,7 +55,7 @@ export async function stageUpgradeUnlocked(options) {
|
||||
}, { mode: 0o644 });
|
||||
await assertTreeContainsNoLinks(stage);
|
||||
if (options.verify) await options.verify(stage);
|
||||
await fs.rename(stage, destination);
|
||||
await renameWithRetry(stage, destination);
|
||||
staged = true;
|
||||
} catch (error) {
|
||||
await fs.rm(stage, { recursive: true, force: true }).catch(() => {});
|
||||
@@ -263,7 +263,7 @@ export async function purgeManagedHomeUnlocked(home) {
|
||||
for (const entry of present.filter((name) => managedEntries.has(name) && !preexisting.has(name))) {
|
||||
const source = assertPathInside(resolved, path.join(resolved, entry));
|
||||
const destination = assertPathInside(quarantine, path.join(quarantine, entry));
|
||||
await fs.rename(source, destination);
|
||||
await renameWithRetry(source, destination);
|
||||
moved.push({ source, destination });
|
||||
}
|
||||
await fs.rm(quarantine, { recursive: true, force: true });
|
||||
@@ -273,7 +273,7 @@ export async function purgeManagedHomeUnlocked(home) {
|
||||
return { purged: true, home: resolved, preserved };
|
||||
} catch (error) {
|
||||
for (const item of moved.reverse()) {
|
||||
await fs.rename(item.destination, item.source).catch(() => {});
|
||||
await renameWithRetry(item.destination, item.source).catch(() => {});
|
||||
}
|
||||
await fs.rmdir(quarantine).catch(() => {});
|
||||
throw error;
|
||||
|
||||
@@ -7,8 +7,10 @@ import {
|
||||
cleanupRuntime,
|
||||
ensureManagedHome,
|
||||
pathExists,
|
||||
renameWithRetry,
|
||||
resolveRuntimePaths,
|
||||
runtimeLifecycleLockPath,
|
||||
syncDirectory,
|
||||
} from "../runtime/index.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
@@ -31,6 +33,44 @@ afterEach(async () => {
|
||||
});
|
||||
|
||||
describe("runtime cleanup boundary", () => {
|
||||
test("retries transient Windows lock creation and rename races", async () => {
|
||||
const home = await temporaryHome();
|
||||
const lockPath = path.join(home, "locks", "windows-race.lock");
|
||||
let mkdirAttempts = 0;
|
||||
const release = await acquireLock(lockPath, {
|
||||
platform: "win32",
|
||||
mkdir: async (target: string, options: Record<string, unknown>) => {
|
||||
if (target === lockPath && mkdirAttempts++ === 0) {
|
||||
throw Object.assign(new Error("simulated Windows delete race"), { code: "EPERM" });
|
||||
}
|
||||
return fs.mkdir(target, options);
|
||||
},
|
||||
});
|
||||
expect(mkdirAttempts).toBe(2);
|
||||
await release();
|
||||
|
||||
let renameAttempts = 0;
|
||||
await renameWithRetry("source", "destination", {
|
||||
platform: "win32",
|
||||
timeoutMs: 100,
|
||||
rename: async () => {
|
||||
if (renameAttempts++ < 2) throw Object.assign(new Error("simulated scanner race"), { code: "EPERM" });
|
||||
},
|
||||
});
|
||||
expect(renameAttempts).toBe(3);
|
||||
});
|
||||
|
||||
test("closes a directory handle when Windows does not support directory fsync", async () => {
|
||||
let closes = 0;
|
||||
await expect(syncDirectory("fixture", {
|
||||
open: async () => ({
|
||||
sync: async () => { throw Object.assign(new Error("unsupported directory sync"), { code: "EPERM" }); },
|
||||
close: async () => { closes += 1; },
|
||||
}),
|
||||
})).resolves.toBeUndefined();
|
||||
expect(closes).toBe(1);
|
||||
});
|
||||
|
||||
test("removes only allowlisted stale runtime scratch and dead locks", async () => {
|
||||
const home = await temporaryHome();
|
||||
const paths = resolveRuntimePaths({ home });
|
||||
|
||||
@@ -31,8 +31,10 @@ async function temporaryRoot(label = "gstack2 runtime ") {
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(temporaryRoots.splice(0).map((root) =>
|
||||
fs.chmod(root, 0o700).catch(() => {}).then(() => fs.rm(root, { recursive: true, force: true }))));
|
||||
for (const root of temporaryRoots.splice(0)) {
|
||||
await fs.chmod(root, 0o700).catch(() => {});
|
||||
await fs.rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
|
||||
}
|
||||
});
|
||||
|
||||
describe("gstack 2 host-neutral paths and state", () => {
|
||||
@@ -92,10 +94,11 @@ describe("gstack 2 host-neutral paths and state", () => {
|
||||
gitDir: path.join(root, "repo", ".git"),
|
||||
});
|
||||
await initializeProject(home, identity);
|
||||
await Promise.all(Array.from({ length: 60 }, () =>
|
||||
const updates = await Promise.allSettled(Array.from({ length: 60 }, () =>
|
||||
updateProjectState(home, identity.projectId, (state) => {
|
||||
state.concurrentCounter = Number(state.concurrentCounter ?? 0) + 1;
|
||||
})));
|
||||
expect(updates.filter((result) => result.status === "rejected")).toEqual([]);
|
||||
const { state } = await inspectProject(home, identity);
|
||||
expect(state.concurrentCounter).toBe(60);
|
||||
expect(state.revision).toBe(60);
|
||||
|
||||
@@ -389,6 +389,7 @@ describe("GStack 2 managed runtime installer", () => {
|
||||
const cli = path.join(result.path, "runtime", "cli.js");
|
||||
const originalMode = (await fs.stat(cli)).mode & 0o777;
|
||||
await fs.chmod(cli, originalMode === 0o600 ? 0o644 : 0o600);
|
||||
await expect(validateRuntimeBundle(result.path, { version: "2.0.0", platform: "win32" })).resolves.toBe(true);
|
||||
await expect(validateRuntimeBundle(result.path, { version: "2.0.0" })).rejects.toMatchObject({
|
||||
code: "INSTALL_VALIDATION_FAILED",
|
||||
});
|
||||
|
||||
@@ -164,7 +164,7 @@ describe("managed-home destructive boundary", () => {
|
||||
describe("one config authority", () => {
|
||||
test("compatibility helper and runtime config share config.json", async () => {
|
||||
const home = path.join(await root(), "state");
|
||||
const run = (args: string[]) => spawnSync(configBin, args, {
|
||||
const run = (args: string[]) => spawnSync(process.execPath, [configBin, ...args], {
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, GSTACK_HOME: home },
|
||||
});
|
||||
@@ -201,20 +201,20 @@ describe("one config authority", () => {
|
||||
const home = path.join(await root(), "legacy");
|
||||
await fs.mkdir(home);
|
||||
await fs.writeFile(path.join(home, "config.yaml"), "telemetry: community\n");
|
||||
const get = spawnSync(configBin, ["get", "telemetry"], {
|
||||
const get = spawnSync(process.execPath, [configBin, "get", "telemetry"], {
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, GSTACK_HOME: home },
|
||||
});
|
||||
expect(get.status).toBe(0);
|
||||
expect(get.stdout).toBe("community");
|
||||
const set = spawnSync(configBin, ["set", "telemetry", "off"], {
|
||||
const set = spawnSync(process.execPath, [configBin, "set", "telemetry", "off"], {
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, GSTACK_HOME: home },
|
||||
});
|
||||
expect(set.status).toBe(0);
|
||||
expect(await fs.readFile(path.join(home, "config.yaml"), "utf8")).toBe("telemetry: community\n");
|
||||
expect(JSON.parse(await fs.readFile(path.join(home, "config.json"), "utf8")).telemetry).toBe("off");
|
||||
const reread = spawnSync(configBin, ["get", "telemetry"], {
|
||||
const reread = spawnSync(process.execPath, [configBin, "get", "telemetry"], {
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, GSTACK_HOME: home },
|
||||
});
|
||||
|
||||
@@ -42,8 +42,9 @@ async function fixture(label = "gstack workflow state ", initialize = true) {
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(temporaryRoots.splice(0).map((root) =>
|
||||
fs.rm(root, { recursive: true, force: true })));
|
||||
for (const root of temporaryRoots.splice(0)) {
|
||||
await fs.rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
|
||||
}
|
||||
});
|
||||
|
||||
describe("GStack 2 authoritative workflow state", () => {
|
||||
@@ -150,7 +151,7 @@ describe("GStack 2 authoritative workflow state", () => {
|
||||
test("all workflow mutations are locked and concurrent evidence writes are not lost", async () => {
|
||||
const { home, identity } = await fixture();
|
||||
await beginRun(home, identity.projectId, "qa", { runId: "run_concurrent" });
|
||||
await Promise.all(Array.from({ length: 24 }, (_, index) =>
|
||||
const updates = await Promise.allSettled(Array.from({ length: 24 }, (_, index) =>
|
||||
updateRunWorkflow(home, identity.projectId, "run_concurrent", {
|
||||
addEvidenceProvenance: {
|
||||
source: "local-test",
|
||||
@@ -158,6 +159,7 @@ describe("GStack 2 authoritative workflow state", () => {
|
||||
capturedAt: `2026-07-16T10:${String(index).padStart(2, "0")}:00.000Z`,
|
||||
},
|
||||
})));
|
||||
expect(updates.filter((result) => result.status === "rejected")).toEqual([]);
|
||||
const inspected = await inspectRun(home, identity.projectId, "run_concurrent");
|
||||
expect(inspected.reconstruction.evidenceProvenance).toHaveLength(24);
|
||||
expect(new Set(inspected.reconstruction.evidenceProvenance.map((entry: any) => entry.reference)).size).toBe(24);
|
||||
|
||||
Reference in New Issue
Block a user