ctrld/cmd/cli/upgrade_rollback.go

package cli

import (
	"context"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/kardianos/service"
)

const (
	// upgradeStopTimeout bounds how long rollback waits for the replacement process to
	// exit, and for Windows to release the lock on its image afterwards.
	upgradeStopTimeout = 30 * time.Second
	// upgradeStopPollInterval is how often the service status is re-checked while
	// waiting for the process to exit.
	upgradeStopPollInterval = 500 * time.Millisecond
	// binaryVersionTimeout bounds the "--version" probe, so a binary that hangs on
	// startup cannot hang the upgrade.
	binaryVersionTimeout = 10 * time.Second
)

// rollbackToPreviousBinary restores oldBin over bin after the replacement failed to
// become ready, and restarts the service on the restored binary.
//
// stop must leave the replacement's process gone, because every step here modifies
// the executable that process is running from. It is called first for that reason:
// readiness failing does not mean the process exited - the service manager can report
// a started service whose process never became operational. Windows holds an
// exclusive lock on a running executable's image, so the previous code's
// os.Remove(bin) failed there with "Access is denied", and because that was fatal the
// restore never ran: the broken binary stayed installed with the previous one
// stranded at its _previous name.
//
// Stopping first also puts the host back in a known state, since a stopped ctrld
// holds no DNS or intercept enforcement.
func rollbackToPreviousBinary(bin, oldBin string, stop func() error, restart func() bool) error {
	if err := stop(); err != nil {
		mainLog.Load().Error().Err(err).Msg("Could not confirm the service stopped; not modifying its binary")
		return err
	}

	// Only restore a previous binary that actually runs: a _previous file that exists
	// but reports no version would replace a service that starts and hangs with one
	// that cannot start at all.
	//
	// The probe is retried for the same reason removeBinaryWithRetry is: on Windows a
	// single exec can fail transiently while antivirus scans the file or the disk is
	// busy, and treating that as "no usable previous binary" leaves the host stopped
	// with the broken binary installed - an end state worse than restoring a binary
	// that turns out to be bad, which the restart check below catches.
	//
	// Running "--version" proves the file executes. It is not an authenticity check:
	// nothing here compares a signature or checksum before a file becomes the installed
	// service binary. That is acceptable only because the install directory is writable
	// by administrators alone, which is this command's standing assumption.
	prevVer, err := binaryVersionWithRetry(oldBin, upgradeStopTimeout)
	if err != nil {
		mainLog.Load().Error().Err(err).Msgf("Previous binary at %s is not usable, keeping it for inspection", oldBin)
		mainLog.Load().Notice().Msgf("Service is stopped and %s is still the installed binary", bin)
		return fmt.Errorf("upgrade failed and no usable previous binary to restore: %w", err)
	}

	mainLog.Load().Warn().Msgf("Restoring previous binary: %s (%s)", oldBin, prevVer)
	if err := removeBinaryWithRetry(bin, upgradeStopTimeout); err != nil {
		mainLog.Load().Error().Err(err).Msg("Failed to remove new binary")
		mainLog.Load().Notice().Msg("Service is stopped")
		return err
	}
	if err := os.Rename(oldBin, bin); err != nil {
		mainLog.Load().Error().Err(err).Msg("Failed to restore old binary")
		mainLog.Load().Notice().Msgf("Service is stopped and %s is missing; reinstall ctrld to recover", bin)
		return err
	}
	if restart() {
		mainLog.Load().Notice().Msgf("Restored previous binary successfully - %s", prevVer)
		return nil
	}

	mainLog.Load().Error().Msg("Restored the previous binary but it did not become ready either")
	return errors.New("upgrade failed and the restored binary did not become ready")
}

// stopServiceAndWait stops the service and waits until the service manager reports
// it stopped. Rollback needs the process gone, not merely asked to stop: a stop
// request returns before the process exits, and on Windows the executable stays
// locked until it does.
func stopServiceAndWait(s service.Service, timeout time.Duration) error {
	if err := s.Stop(); err != nil {
		// Not fatal: the service may already be stopped, or stopping may fail while
		// the process is exiting anyway. The status poll below decides.
		mainLog.Load().Debug().Err(err).Msg("Stop request failed, waiting for the process to exit anyway")
	}
	deadline := time.Now().Add(timeout)
	statusReadable := false
	var lastErr error
	for {
		status, err := s.Status()
		switch {
		case errors.Is(err, service.ErrNotInstalled):
			return nil
		case err == nil:
			statusReadable = true
			if status == service.StatusStopped {
				return nil
			}
		default:
			lastErr = err
		}
		if !time.Now().Before(deadline) {
			if !statusReadable {
				// The status was never readable, so "did not stop" was never observed -
				// only "could not be observed". Refusing to continue here would leave the
				// broken binary installed with the service stopped, which is the outcome
				// rollback exists to avoid. Let the caller proceed: the remove is retried
				// while the image is locked, and the restart check still has to pass
				// before this reports success.
				mainLog.Load().Warn().Err(lastErr).Msgf("Could not read service status within %s; continuing with rollback", timeout)
				return nil
			}
			return fmt.Errorf("service did not stop within %s", timeout)
		}
		time.Sleep(upgradeStopPollInterval)
	}
}

// binaryVersionWithRetry probes a binary's version, retrying transient exec failures
// until timeout. Only the last error is reported: the earlier attempts are noise once a
// retry has been made.
func binaryVersionWithRetry(path string, timeout time.Duration) (string, error) {
	deadline := time.Now().Add(timeout)
	for {
		version, err := binaryVersionFn(path)
		if err == nil {
			return version, nil
		}
		if !time.Now().Before(deadline) {
			return "", err
		}
		mainLog.Load().Debug().Err(err).Msgf("Version probe of %s failed, retrying", path)
		time.Sleep(upgradeStopPollInterval)
	}
}

// removeBinaryWithRetry removes path, retrying while it is still locked. Windows
// releases the lock on an executable's image asynchronously after its process exits,
// so a remove issued immediately after the service reports stopped can still fail
// with "Access is denied".
func removeBinaryWithRetry(path string, timeout time.Duration) error {
	deadline := time.Now().Add(timeout)
	for {
		err := os.Remove(path)
		if err == nil || errors.Is(err, os.ErrNotExist) {
			return nil
		}
		if !time.Now().Before(deadline) {
			return fmt.Errorf("could not remove %s within %s: %w", path, timeout, err)
		}
		time.Sleep(upgradeStopPollInterval)
	}
}

// binaryVersionFn is indirected so rollback can be tested without staging a runnable
// executable per platform. The probe itself is covered directly against the test
// binary; see TestBinaryVersion.
var binaryVersionFn = binaryVersion

// binaryVersion runs path with "--version" and returns the version it reports. It
// answers "can this binary actually run on this host", which is what rollback needs
// to know before making a file the installed ctrld.
//
// On Windows path is ctrld.exe_previous, whose extension is not in PATHEXT. That
// resolves because os/exec only falls back to appending PATHEXT entries when the path
// has no extension at all (lp_windows.go findExecutable): with one present and the
// file on disk, it is used as-is. A suffix that left no extension - renaming
// oldBinSuffix such that the result is "ctrld_previous" - would break this probe with
// "executable file not found in %PATH%", and rollback would then refuse to restore a
// perfectly good binary.
func binaryVersion(path string) (string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), binaryVersionTimeout)
	defer cancel()
	out, err := exec.CommandContext(ctx, path, "--version").CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("running %s --version: %w", path, err)
	}
	ver, found := strings.CutPrefix(strings.TrimSpace(string(out)), "ctrld version ")
	if !found {
		return "", fmt.Errorf("unexpected --version output from %s: %q", path, strings.TrimSpace(string(out)))
	}
	return ver, nil
}