fix(cso): record each CSO build's outcome and stop arming the parent trap across the native publisher exec (#3071)

scripts/build-cso.sh now writes bin/.gstack-cso-build-result: result,
failed stage (build or publish), reason, this revision, the revision of
the generation still installed, whether its launcher is usable, and the
diagnostic log. An 'interrupted' entry is written before each stage so a
killed run is visible. A failed publish that restores the old generation
records it as still installed.

Before exec'ing the native publisher, the parent re-checks the locker,
clears its EXIT trap and enables execfail: the locked child's own trap
owns the stage. On Git Bash an exec of a native program may not replace
the shell, so the parent's trap could otherwise delete the stage under
the running child (the vanishing files in #3071; root cause unproven).
If the exec itself fails the parent cleans the stage and records it.

The Windows build prefers PowerShell 7 (pwsh) over Windows PowerShell
5.1, which crashed outright on the reporter's machine.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

.gitignore

@@ -19,6 +19,8 @@ bin/.gstack-cso-stage.*
 bin/.gstack-cso-build.lock/
 bin/.gstack-cso-generation.lock
 bin/.gstack-cso-generation
+bin/.gstack-cso-build-result
+bin/.gstack-cso-build.log
 lib/cso/images/gstack-cso-verifier
 lib/cso/images/gstack-cso-preparation
 .gstack/

scripts/build-cso.sh

@@ -15,6 +15,12 @@ CSO_FINAL_CORE="$CSO_BUILD_ROOT/bin/gstack-cso-core$CSO_EXE"
 CSO_FINAL_LAUNCHER="$CSO_BUILD_ROOT/bin/gstack-cso-launcher$CSO_EXE"
 CSO_FINAL_WATCHDOG="$CSO_BUILD_ROOT/bin/gstack-cso-watchdog"
 CSO_FINAL_GENERATION="$CSO_BUILD_ROOT/bin/.gstack-cso-generation"
+# Build-result record read by setup's summary and gstack-doctor (#3071).
+CSO_RESULT_FILE="$CSO_BUILD_ROOT/bin/.gstack-cso-build-result"
+CSO_RESULT_RECORDING=0
+CSO_RESULT_STAGE=build
+CSO_REVISION=""
+CSO_PREVIOUS_INSTALLED=""
 CSO_STAGE=""
 CSO_PUBLISHING=0
 CSO_COMMITTED=0
@@ -38,6 +44,25 @@ cso_checkpoint() {
   if [ "${GSTACK_CSO_BUILD_TEST_KILL_AFTER:-}" = "$1" ]; then
     kill -KILL "$$"
   fi
+  if [ "${GSTACK_CSO_BUILD_TEST_REMOVE_LOCKER_AFTER:-}" = "$1" ]; then
+    rm -f "$CSO_STAGE_LOCKER"
+  fi
+}
+
+cso_record_field() { sed -n "s/^$1=//p" "$CSO_RESULT_FILE" 2>/dev/null | head -1; }
+
+# One record per run: result (ok/failed), stage (build/publish), reason, this
+# checkout's revision, the revision of the CSO generation still installed, and
+# whether that launcher is usable. A SIGKILL leaves the last "interrupted" entry.
+cso_write_result() {
+  launcher=no
+  if cso_valid_artifact "$CSO_FINAL_LAUNCHER" && cso_valid_artifact "$CSO_FINAL_CORE"; then launcher=yes; fi
+  installed=""
+  if [ "$1" = ok ]; then installed=$CSO_REVISION
+  elif [ "$launcher" = yes ]; then installed=${CSO_PREVIOUS_INSTALLED:-an earlier build}; fi
+  # Written in place: publication's signal tests count on mv moving artifacts only.
+  printf 'result=%s\nstage=%s\nreason=%s\nrevision=%s\ninstalled=%s\nlauncher=%s\ndiagnostic=%s\n' \
+    "$1" "$2" "$3" "$CSO_REVISION" "$installed" "$launcher" "${GSTACK_CSO_BUILD_LOG:-the scripts/build-cso.sh output}" > "$CSO_RESULT_FILE" || true
 }
 
 cso_restore_previous() {
@@ -89,6 +114,10 @@ cso_cleanup() {
   fi
   if [ "$cleanup_stage" -eq 1 ] && [ -n "$CSO_STAGE" ] && [ ! -f "$CSO_STAGE/.retain-recovery" ]; then rm -rf "$CSO_STAGE"; fi
   if [ -n "$CSO_EVALUATION_UNIT" ]; then rm -rf "$CSO_EVALUATION_UNIT"; fi
+  if [ "$CSO_RESULT_RECORDING" -eq 1 ]; then
+    if [ "$status" -eq 0 ] && [ "$CSO_COMMITTED" -eq 1 ]; then cso_write_result ok publish committed
+    else cso_write_result failed "$CSO_RESULT_STAGE" "exit $status"; fi
+  fi
   exit "$status"
 }
 trap 'cso_cleanup $?' EXIT
@@ -281,8 +310,18 @@ cso_publish_locked() {
   CSO_COMMITTED=1
 }
 
+cso_start_result() {
+  CSO_RESULT_RECORDING=1
+  CSO_RESULT_STAGE=$1
+  CSO_REVISION="$(head -1 "$CSO_BUILD_ROOT/VERSION" 2>/dev/null | tr -cd '0-9A-Za-z.+-')"
+  revision_sha="$(git -C "$CSO_BUILD_ROOT" rev-parse --short=12 HEAD 2>/dev/null | tr -cd '0-9a-f')"
+  CSO_REVISION="${CSO_REVISION:-unknown}${revision_sha:+ ($revision_sha)}"
+  CSO_PREVIOUS_INSTALLED="$(cso_record_field installed)"
+}
+
 if [ "${1:-}" = __publish_locked ];then
   shift
+  cso_start_result publish
   cso_publish_locked "$@"
   exit 0
 fi
@@ -291,6 +330,8 @@ if [ "${1:-}" = --evaluation-candidate ];then
   exit 0
 fi
 [ "$#" -eq 0 ] || { echo 'Usage: build-cso.sh [--evaluation-candidate <cso-eval-catalog.json> --output <new-directory>]' >&2;exit 64; }
+cso_start_result build
+cso_write_result failed build interrupted
 # Distribution builds embed only the committed catalog, never an evaluation one.
 if [ -f lib/cso/runtime-catalog.json ] && grep -q '"revision": *"cso-eval-' lib/cso/runtime-catalog.json;then
   echo 'lib/cso/runtime-catalog.json is an evaluation-only catalog; distribution builds refuse it. Restore the committed catalog.' >&2
@@ -339,12 +380,14 @@ CSO_CORE_SHA256="$(cso_sha256 "$CSO_STAGE_CORE")"
 printf '%s\n' "$CSO_CORE_SHA256" > "$CSO_STAGE_GENERATION"
 
 if [ -n "$CSO_EXE" ];then
-  if ! command -v powershell.exe >/dev/null 2>&1||! command -v cygpath >/dev/null 2>&1;then
-    echo 'CSO Windows build requires Git Bash and Windows PowerShell with MSVC Build Tools.' >&2;exit 1
+  # PowerShell 7 first: Windows PowerShell 5.1 can crash outright (#3071).
+  CSO_POWERSHELL="$(command -v pwsh 2>/dev/null || command -v powershell.exe 2>/dev/null || true)"
+  if [ -z "$CSO_POWERSHELL" ]||! command -v cygpath >/dev/null 2>&1;then
+    echo 'CSO Windows build requires Git Bash and PowerShell (pwsh or Windows PowerShell) with MSVC Build Tools.' >&2;exit 1
   fi
   CSO_WINDOWS_GIT="$(type -P git 2>/dev/null || true)"
   [ -n "$CSO_WINDOWS_GIT" ] && [ -f "$CSO_WINDOWS_GIT" ] || { echo 'CSO Windows build requires the Git for Windows git.exe selected by Git Bash.' >&2;exit 1; }
-  powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass \
+  "$CSO_POWERSHELL" -NoProfile -NonInteractive -ExecutionPolicy Bypass \
     -File "$(cygpath -w "$CSO_BUILD_ROOT/scripts/build-cso-windows.ps1")" \
     -RepoRoot "$(cygpath -w "$CSO_BUILD_ROOT")" \
     -OutputPath "$(cygpath -w "$CSO_STAGE_LAUNCHER")" \
@@ -376,4 +419,18 @@ if [ -n "$CSO_EXE" ];then
 else
   case "${BASH:-}" in /*) CSO_PUBLISH_SHELL=$BASH;; *) echo 'CSO publication requires an absolute Bash executable.' >&2;exit 1;;esac
 fi
+cso_valid_artifact "$CSO_STAGE_LOCKER" || { echo 'Staged CSO publisher lock is not one executable regular file.' >&2; exit 1; }
+CSO_RESULT_STAGE=publish
+cso_write_result failed publish interrupted
+cso_checkpoint before-exec
+# From here the locked child owns the stage: its own EXIT trap restores and
+# cleans it. Git Bash's exec of a native program may not replace this shell,
+# so this trap must not be armed under the running child (#3071). If the exec
+# itself fails, clean up here.
+set +e
+shopt -s execfail
+trap - EXIT
 exec "$CSO_STAGE_LOCKER" "$CSO_BUILD_ROOT/bin" "$CSO_PUBLISH_SHELL" "$CSO_BUILD_ROOT/scripts/build-cso.sh" __publish_locked "$CSO_STAGE"
+cso_exec_status=$?
+echo "CSO publication could not start $CSO_STAGE_LOCKER (exit $cso_exec_status)." >&2
+cso_cleanup "$cso_exec_status"

test/build-cso-result-record.test.ts

@@ -0,0 +1,129 @@
+import { afterEach, describe, expect, test } from 'bun:test';
+import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { runBashScript } from './helpers/bash-script';
+
+// #3071: scripts/build-cso.sh records each run's outcome in
+// bin/.gstack-cso-build-result for setup's summary and gstack-doctor, and the
+// parent clears its EXIT trap before exec'ing the native publisher so only the
+// locked child cleans the stage. The real script runs; only compilation and
+// the publisher lock are substituted (same seam as test/cso-distribution.test.ts).
+const ROOT = resolve(import.meta.dir, '..');
+const temps: string[] = [];
+afterEach(() => { for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true }); });
+const quote = (s: string) => "'" + s.replaceAll("'", "'\\''") + "'";
+
+function fixture() {
+  const dir = mkdtempSync(join(tmpdir(), 'gstack-cso-result-')); temps.push(dir);
+  for (const sub of ['scripts', 'bin', 'lib/cso']) mkdirSync(join(dir, sub), { recursive: true });
+  copyFileSync(join(ROOT, 'scripts/build-cso.sh'), join(dir, 'scripts/build-cso.sh'));
+  writeFileSync(join(dir, 'VERSION'), '9.9.9.9\n');
+  writeFileSync(join(dir, 'publish lock'), '#!/bin/sh\nset -eu\nshift\nexport GSTACK_CSO_PUBLISH_LOCKED=1\nexec "$@"\n');
+  chmodSync(join(dir, 'publish lock'), 0o755);
+  const compiler = [
+    '#!/bin/sh', 'set -eu',
+    'count_file="$CSO_FIXTURE/compiler.count"; count=0',
+    '[ ! -f "$count_file" ] || count=$(cat "$count_file")',
+    'count=$((count+1)); printf "%s\\n" "$count" > "$count_file"',
+    'if [ "${CSO_FAIL_COMPILER_N:-0}" = "$count" ]; then echo "fixture compiler failed" >&2; exit 42; fi',
+    'output=""',
+    'while [ "$#" -gt 0 ]; do case "$1" in --outfile|-o) shift; output="$1" ;; esac; shift; done',
+    'if [ -n "$output" ]; then',
+    '  case "$output" in',
+    '    *gstack-cso-publish-lock*) cp "$CSO_FIXTURE/publish lock" "$output" ;;',
+    '    *) printf "#!/bin/sh\\nprintf NEW\\\\n\\n" > "$output" ;;',
+    '  esac',
+    '  chmod +x "$output"',
+    'fi', '',
+  ].join('\n');
+  writeFileSync(join(dir, 'compiler'), compiler); chmodSync(join(dir, 'compiler'), 0o755);
+  return dir;
+}
+
+function seedOld(dir: string, installed = '9.9.9.8 (0123456789ab)') {
+  for (const name of ['gstack-cso-launcher', 'gstack-cso-core', 'gstack-cso-watchdog']) {
+    writeFileSync(join(dir, 'bin', name), '#!/bin/sh\nprintf OLD\\\\n\n'); chmodSync(join(dir, 'bin', name), 0o755);
+  }
+  writeFileSync(join(dir, 'bin', '.gstack-cso-generation'), '0'.repeat(64) + '\n');
+  writeFileSync(join(dir, 'bin', '.gstack-cso-build-result'), `result=ok\nstage=publish\nreason=committed\nrevision=${installed}\ninstalled=${installed}\nlauncher=yes\ndiagnostic=old\n`);
+}
+
+function build(dir: string, env = '') {
+  return runBashScript(`${env} GIT_CEILING_DIRECTORIES=${quote(dirname(dir))} CSO_FIXTURE=${quote(dir)} BUN_CMD=${quote(join(dir, 'compiler'))} CSO_CC=${quote(join(dir, 'compiler'))} bash ${quote(join(dir, 'scripts/build-cso.sh'))}`, { timeout: 15_000 });
+}
+
+function record(dir: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const line of readFileSync(join(dir, 'bin', '.gstack-cso-build-result'), 'utf8').trim().split('\n')) {
+    const at = line.indexOf('=');
+    out[line.slice(0, at)] = line.slice(at + 1);
+  }
+  return out;
+}
+const stages = (dir: string) => readdirSync(join(dir, 'bin')).filter(name => name.startsWith('.gstack-cso-stage.'));
+const launcherOutput = (dir: string) => readFileSync(join(dir, 'bin', 'gstack-cso-launcher'), 'utf8');
+
+describe.skipIf(process.platform === 'win32')('build-cso.sh build-result record (#3071)', () => {
+  test('a successful build records ok with this revision installed', () => {
+    const dir = fixture();
+    const r = build(dir, `GSTACK_CSO_BUILD_LOG=${quote(join(dir, 'cso.log'))}`);
+    expect(r.status, r.stderr).toBe(0);
+    expect(record(dir)).toMatchObject({ result: 'ok', stage: 'publish', revision: '9.9.9.9', installed: '9.9.9.9', launcher: 'yes', diagnostic: join(dir, 'cso.log') });
+    expect(stages(dir)).toEqual([]);
+  });
+
+  test('a fresh build failure records the build stage with no usable launcher', () => {
+    const dir = fixture();
+    const r = build(dir, 'CSO_FAIL_COMPILER_N=1');
+    expect(r.status).toBe(42);
+    expect(record(dir)).toMatchObject({ result: 'failed', stage: 'build', reason: 'exit 42', installed: '', launcher: 'no' });
+    expect(stages(dir)).toEqual([]);
+  });
+
+  test('a failed upgrade keeps the previous generation and names its revision', () => {
+    const dir = fixture(); seedOld(dir);
+    const r = build(dir, 'GSTACK_CSO_BUILD_TESTING=1 GSTACK_CSO_BUILD_TEST_FAIL_AFTER=publish-core');
+    expect(r.status).toBe(86);
+    expect(launcherOutput(dir)).toContain('OLD');
+    expect(record(dir)).toMatchObject({ result: 'failed', stage: 'publish', reason: 'exit 86', revision: '9.9.9.9', installed: '9.9.9.8 (0123456789ab)', launcher: 'yes' });
+    expect(stages(dir)).toEqual([]);
+  });
+
+  test('a publish killed mid-flight leaves an interrupted publish record', () => {
+    const dir = fixture(); seedOld(dir);
+    const r = build(dir, 'GSTACK_CSO_BUILD_TESTING=1 GSTACK_CSO_BUILD_TEST_KILL_AFTER=publish-core');
+    expect(r.status).not.toBe(0);
+    expect(record(dir)).toMatchObject({ result: 'failed', stage: 'publish', reason: 'interrupted' });
+    expect(existsSync(join(dir, 'bin', 'gstack-cso-launcher'))).toBe(false);
+  });
+
+  test('a missing publisher at exec time cleans the stage and keeps the old generation', () => {
+    const dir = fixture(); seedOld(dir);
+    const r = build(dir, 'GSTACK_CSO_BUILD_TESTING=1 GSTACK_CSO_BUILD_TEST_REMOVE_LOCKER_AFTER=before-exec');
+    expect(r.status).toBe(127);
+    expect(r.stderr).toContain('CSO publication could not start');
+    expect(stages(dir)).toEqual([]);
+    expect(launcherOutput(dir)).toContain('OLD');
+    expect(record(dir)).toMatchObject({ result: 'failed', stage: 'publish', reason: 'exit 127', installed: '9.9.9.8 (0123456789ab)', launcher: 'yes' });
+  });
+});
+
+describe('build-cso.sh publisher handoff (#3071)', () => {
+  const source = readFileSync(join(ROOT, 'scripts/build-cso.sh'), 'utf8');
+  test('the parent checks the publisher, then clears its EXIT trap, then execs', () => {
+    const exec = source.indexOf('exec "$CSO_STAGE_LOCKER"');
+    const clear = source.lastIndexOf('trap - EXIT', exec);
+    const check = source.lastIndexOf('cso_valid_artifact "$CSO_STAGE_LOCKER"', clear);
+    expect(exec).toBeGreaterThan(0);
+    expect(clear).toBeGreaterThan(check);
+    expect(check).toBeGreaterThan(0);
+    expect(source.slice(check, exec)).toContain('shopt -s execfail');
+    expect(source.slice(exec)).toContain('cso_cleanup "$cso_exec_status"');
+  });
+
+  test('the Windows build prefers PowerShell 7 and falls back to Windows PowerShell', () => {
+    expect(source).toContain('CSO_POWERSHELL="$(command -v pwsh 2>/dev/null || command -v powershell.exe 2>/dev/null || true)"');
+    expect(source).not.toMatch(/^\s*powershell\.exe /m);
+  });
+});

test/cso-windows-launcher.test.ts

@@ -141,7 +141,7 @@ describe('CSO native Windows build contract', () => {
   test('Windows builds use MSVC with a static CRT and no Bun-hosted public launcher', () => {
     const build = fs.readFileSync(path.join(ROOT, 'scripts/build-cso.sh'), 'utf8');
     const msvc = fs.readFileSync(path.join(ROOT, 'scripts/build-cso-windows.ps1'), 'utf8');
-    expect(build).toContain('powershell.exe -NoProfile -NonInteractive');
+    expect(build).toContain('"$CSO_POWERSHELL" -NoProfile -NonInteractive');
     expect(build).toContain('build-cso-windows.ps1');
     expect(build).toContain('-OutputPath "$(cygpath -w "$CSO_STAGE_LAUNCHER")"');
     expect(build).toContain('-LockOutputPath "$(cygpath -w "$CSO_STAGE_LOCKER")"');