ctrld/cmd/cli/api_maintenance.go

package cli

import (
	"crypto/sha256"
	"encoding/hex"
	"math/rand"
	"os"
	"strings"
	"time"

	"github.com/go-playground/validator/v10"

	"github.com/Control-D-Inc/ctrld"
	"github.com/Control-D-Inc/ctrld/internal/controld"
)

// configReadFromDisk records that startup read a configuration file this host
// already had, instead of generating a default one because none existed.
//
// The distinction only matters while the API is in maintenance. A generated
// default carries the public freedns upstreams, so falling back to it would
// answer queries outside the Control D profile the operator provisioned —
// quietly serving a policy nobody chose, which is worse than not starting.
// Only a file the host already had can stand in for the configuration the API
// would have returned.
var configReadFromDisk bool

// maintenanceNoConfigDetail is what ctrld reports when maintenance stops a
// start that has nothing to fall back on. It names the operation that cannot
// happen rather than the fetch that failed, because "could not fetch resolver
// config" reads as a network fault and sends the reader after connectivity
// that is not broken.
const maintenanceNoConfigDetail = "cannot provision during Control D API maintenance: " +
	"this host has no previously working configuration to fall back on. Retry once maintenance ends."

// managedConfigMarkerFile records that the configuration file next to it was
// written from an API answer, and for which identity.
//
// File presence cannot carry this on its own. A start that has no
// configuration writes the seeded default before the API is ever reached, so a
// provisioning that failed leaves a file behind that the next start reads
// successfully. That file holds the public freedns upstreams, so trusting it
// would run the endpoint outside the Control D profile it was provisioned for
// — the exact outcome the fallback exists to avoid.
const managedConfigMarkerFile = ".managed_config"

// managedConfigMarkerPath is a var so tests can place the marker in a temp dir.
var managedConfigMarkerPath = func() string { return absHomeDir(managedConfigMarkerFile) }

// managedConfigMarker fingerprints both the identity and the configuration file
// the marker is recorded for.
//
// The identity alone cannot carry it. A marker that names only the identity
// outlives the file it vouched for: delete or move that file and the next start
// regenerates the seeded default, which the stale marker then certifies. The
// same happens when --config points somewhere else under the same home. Binding
// to the contents means any file the API did not produce fails the check,
// without having to detect how it got there.
//
// The raw identity is kept off disk: it is redacted from retained logs, and a
// marker only ever has to answer whether this is the same identity.
func managedConfigMarker(uid, path string) (string, error) {
	data, err := os.ReadFile(path)
	if err != nil {
		return "", err
	}
	identity := sha256.Sum256([]byte(uid))
	contents := sha256.Sum256(data)
	return hex.EncodeToString(identity[:]) + ":" + hex.EncodeToString(contents[:]), nil
}

// recordManagedConfig notes that the configuration on disk came from the API
// for uid. The caller must have persisted that configuration first, or the
// marker would vouch for a file that does not match it.
func recordManagedConfig(uid string) error {
	if uid == "" {
		return nil
	}
	marker, err := managedConfigMarker(uid, defaultConfigFile)
	if err != nil {
		return err
	}
	return os.WriteFile(managedConfigMarkerPath(), []byte(marker), 0600)
}

// managedConfigRecordedFor reports whether the configuration file in use was
// written from an API answer for uid.
func managedConfigRecordedFor(uid string) bool {
	if uid == "" {
		return false
	}
	recorded, err := os.ReadFile(managedConfigMarkerPath())
	if err != nil {
		return false
	}
	want, err := managedConfigMarker(uid, defaultConfigFile)
	if err != nil {
		return false
	}
	return strings.TrimSpace(string(recorded)) == want
}

// lastKnownConfigUsable reports whether cfg can carry this start by itself.
//
// It is deliberately quiet: this is a probe, and the caller reports either
// outcome. Startup validates the configuration again further on and reports
// its own errors there, so a failure here must not also print them.
func lastKnownConfigUsable(cfg *ctrld.Config) bool {
	if !configReadFromDisk || cfg == nil {
		return false
	}
	// A file this host already had is necessary but not sufficient: it must
	// also be one the API produced, for the identity being asked for now.
	if !managedConfigRecordedFor(cdUID) {
		return false
	}
	return ctrld.ValidateConfig(validator.New(), cfg) == nil
}

// useLastKnownConfigDuringMaintenance reports whether startup may continue on
// the configuration already on disk because the API is in maintenance.
//
// Maintenance says nothing about this host's configuration — the same request
// succeeds once the window ends — so stopping DNS for its duration turns a
// temporary API outage into an endpoint outage. Continuing is only honest when
// the configuration on disk is one the API itself produced on an earlier run:
// configReadFromDisk establishes that this host already had the file, and the
// managed-config marker that the API produced it for this identity. The marker
// is the load-bearing half - a file alone can be a generated default.
//
// The resolver config stays nil: this run has none, and the reload path
// already treats a nil one as "no previous response to compare against", so
// the first successful reload after maintenance adopts the fresh answer.
func useLastKnownConfigDuringMaintenance(err error, cfg *ctrld.Config) bool {
	if !controld.IsMaintenance(err) {
		return false
	}
	logger := mainLog.Load().With().Str("mode", "cd")
	if !lastKnownConfigUsable(cfg) {
		journal(logger.Error()).Str("reason", "api_maintenance").
			Msg("Control D API is in maintenance and no previously working configuration is on disk")
		return false
	}
	journal(logger.Warn()).Str("reason", "api_maintenance").
		Msg("Control D API is in maintenance; starting with the last known working configuration on disk")
	return true
}

// logResolverConfigFetchFailure reports a resolver-config fetch that failed on
// a path which keeps serving DNS afterwards: the periodic reload and the
// self-uninstall probe.
//
// Maintenance gets its own line, and it is journaled. "could not fetch
// resolver config" reads as a fault on this host and sends the reader after
// connectivity that is not broken, which is how the reported incident was
// first triaged. It also leaves a later "why is my configuration stale"
// question unanswerable once the debug stream has rotated. The reload runs
// hourly by default, so one retained line per window is bounded.
func logResolverConfigFetchFailure(logger *ctrld.Logger, err error) {
	if controld.IsMaintenance(err) {
		journal(logger.Warn()).Err(err).Str("reason", "api_maintenance").
			Msg("Control D API is in maintenance; keeping the current configuration until it returns")
		return
	}
	logger.Warn().Err(err).Msg("Could not fetch resolver config")
}

const (
	// maintenanceReloadInterval is how soon a run that started on the
	// configuration on disk asks the API again. It is deliberately far below
	// refetch_time: such a run holds no answer from the API at all, so its
	// exclude list, Internal Domains, deactivation pin and self-upgrade check
	// are all still those of the previous run until one arrives.
	maintenanceReloadInterval = 5 * time.Minute
	// maintenanceReloadJitter spreads those retries across the hosts that
	// started during the same window.
	//
	// The hourly ticker needs no jitter because each host starts it at its own
	// boot time, which spreads the fleet on its own. A maintenance window
	// removes that spread: every host that falls back does so within the same
	// few minutes, so a fixed retry would put them all in lockstep and land as
	// one burst on the API exactly as it comes back.
	maintenanceReloadJitter = 2 * time.Minute
)

// maintenanceReloadDelay returns the wait before the next retry of a run that
// started on the configuration on disk. It is a var so a test can drive the
// reload loop without waiting out a real interval.
var maintenanceReloadDelay = func() time.Duration {
	return maintenanceReloadInterval + time.Duration(rand.Int63n(int64(maintenanceReloadJitter)))
}

// maintenanceConfigApplied records that a run which started on the
// configuration on disk has handed an answer from the API to the reload path.
//
// It is deliberately not called when the fetch succeeds, nor when the response
// is handed to the reload. The reload receiver reads the configuration and
// fetches again, and either step can fail; until it does not, the endpoint is
// still serving what it started with. Clearing the flag earlier would stop the
// faster retry and report a recovery that had not happened.
func (p *prog) maintenanceConfigApplied(logger *ctrld.Logger) {
	if p.startedInAPIMaintenance.CompareAndSwap(true, false) {
		journal(logger.Info()).Str("reason", "api_maintenance").
			Msg("Control D API answered again; this run applied the configuration it returned")
	}
}

// persistManagedConfig writes cfg and, on a run the API manages, records that
// the file it just wrote came from there.
//
// The two belong together. The marker is bound to the file's contents, and
// every API-driven change rewrites that file, so a write that does not refresh
// the marker leaves the host ineligible for the very fallback its own
// configuration is meant to provide - including recovery from a maintenance
// start. A failed write certifies nothing, and a run with no resolver UID
// writes nothing the API produced: generated defaults and NextDNS
// configurations must never be certified.
func persistManagedConfig(cfg *ctrld.Config, logger *ctrld.Logger) {
	if err := writeConfigFile(cfg); err != nil {
		logger.Error().Err(err).Msg("Could not write new config")
		return
	}
	if cdUID == "" {
		return
	}
	if err := recordManagedConfig(cdUID); err != nil {
		logger.Warn().Err(err).Msg("Could not record that this configuration came from the API")
	}
}

// recordManagedConfigForStart refreshes the marker once a start has finished
// writing its configuration.
//
// fetchedThisRun is whether the API answered this run. wasManagedBeforeWrite is
// whether the file already carried a marker for this identity before anything
// this start rewrote it. Either is enough: a maintenance fallback fetches
// nothing, yet still rewrites the file through the listener and intercept-mode
// updates, and the marker is bound to the file's contents - so without the
// second the next start in the same window loses the fallback this one used.
// Neither means the file is a generated default or otherwise unmanaged, which
// must never be certified.
func recordManagedConfigForStart(fetchedThisRun, wasManagedBeforeWrite bool) {
	if !fetchedThisRun && !wasManagedBeforeWrite {
		return
	}
	if err := recordManagedConfig(cdUID); err != nil {
		// Not fatal: this run is fine. It only costs the next start its
		// fallback if the API is in maintenance then.
		mainLog.Load().Warn().Err(err).Msg("Could not record that this configuration came from the API")
	}
}