fix(updater): derive the Windows installer launch error from ShellExecuteW (#3692)

* fix(updater): derive the Windows installer launch error from ShellExecuteW

A failed `ShellExecuteW` was reported with `std::io::Error::last_os_error()`,
but `ShellExecuteW` reports failures through its return value (`SE_ERR_*`),
not reliably through `GetLastError`, so the message could be meaningless. The
return value is now mapped to an error (the codes shared with Win32 errors,
like `SE_ERR_ACCESSDENIED` when the UAC prompt is declined, keep their OS
message).

Running `on_before_exit` only once the installer launched changes when a
public hook runs and is left for v3.

Compile-checked only (clippy for x86_64-pc-windows-msvc).

plugins/updater/src/updater.rs

* use constants

.changes/updater-windows-shellexecute-error.md

@@ -0,0 +1,6 @@
+---
+updater: patch
+updater-js: patch
+---
+
+On Windows, report why launching the installer failed from the `ShellExecuteW` return value (for example "Access is denied" when the user declines the UAC prompt) instead of an unrelated `GetLastError` message.

plugins/updater/src/updater.rs

@@ -1044,7 +1044,7 @@ impl Update {
             )
         };
         if result as isize <= 32 {
-            return Err(crate::Error::Io(std::io::Error::last_os_error()));
+            return Err(crate::Error::Io(shell_execute_error(result as isize)));
         }
 
         std::process::exit(0);
@@ -1979,6 +1979,48 @@ fn encode_wide(string: impl AsRef<OsStr>) -> Vec<u16> {
         .collect()
 }
 
+/// Converts a failed `ShellExecuteW` return value (`<= 32`) into an error.
+///
+/// `ShellExecuteW` reports failures through its return value, not reliably through
+/// `GetLastError`.
+#[cfg(windows)]
+fn shell_execute_error(code: isize) -> std::io::Error {
+    use std::io::{Error, ErrorKind};
+    use windows_sys::Win32::{
+        Foundation::ERROR_BAD_FORMAT,
+        UI::Shell::{
+            SE_ERR_ACCESSDENIED, SE_ERR_ASSOCINCOMPLETE, SE_ERR_DDEBUSY, SE_ERR_DDEFAIL,
+            SE_ERR_DDETIMEOUT, SE_ERR_DLLNOTFOUND, SE_ERR_FNF, SE_ERR_NOASSOC, SE_ERR_OOM,
+            SE_ERR_PNF, SE_ERR_SHARE,
+        },
+    };
+
+    let Ok(error) = u32::try_from(code) else {
+        return Error::other(format!("ShellExecuteW failed with code {code}"));
+    };
+
+    match error {
+        0 => Error::new(
+            ErrorKind::OutOfMemory,
+            "the system is out of memory or resources",
+        ),
+        // SE_ERR_FNF, SE_ERR_PNF, SE_ERR_ACCESSDENIED (also returned when the user declines the
+        // UAC prompt), SE_ERR_OOM and ERROR_BAD_FORMAT share the values of the Win32 error codes
+        SE_ERR_FNF | SE_ERR_PNF | SE_ERR_ACCESSDENIED | SE_ERR_OOM | ERROR_BAD_FORMAT => {
+            Error::from_raw_os_error(error as i32)
+        }
+        SE_ERR_SHARE => Error::other("a sharing violation occurred"),
+        SE_ERR_ASSOCINCOMPLETE | SE_ERR_NOASSOC => {
+            Error::other("no application is associated with the installer")
+        }
+        SE_ERR_DDETIMEOUT | SE_ERR_DDEFAIL | SE_ERR_DDEBUSY => {
+            Error::other("the DDE transaction failed")
+        }
+        SE_ERR_DLLNOTFOUND => Error::other("the specified DLL was not found"),
+        _ => Error::other(format!("ShellExecuteW failed with code {code}")),
+    }
+}
+
 #[cfg(windows)]
 trait PathExt {
     fn wrap_in_quotes(&self) -> Self;