cmd/cli: bound API preflight by service lifetime

processCDFlags retries the resolver-config fetch indefinitely by design: a
device that has no working network at boot must eventually come up. The loop had
no cancellation, so a stop request arriving while the API is unreachable was
ignored - the process kept retrying long after the service reported itself
stopped, doing work on behalf of a service the OS considers stopped.

Thread a context through processCDFlags and derive it from p.stopCh, in both the
startup preflight and the config-reload path. The loop now returns as soon as the
context is cancelled, checked both before a retry and after backoff returns
(backoff can wake up on cancellation). A stop during preflight exits the way a
normal stop does, without Fatal, so the service manager does not treat it as a
failed start and apply its restart policy to a service the operator just asked
to stop.

Bind the two API requests themselves as well, so a stop does not have to wait out
an in-flight request. Without this the loop honours a stop only between attempts,
which leaves up to defaultTimeout (20s) of a request the service is no longer
interested in - the same "still working after Service stopped" the loop change
exists to end, one layer down.

Doing so means a context parameter on FetchResolverConfig, FetchResolverUID,
UpdateCustomLastFailed and SendLogs, since all four reach a request builder. The
callers that have no context pass context.Background(), which is what master
effectively does at those sites: its loggerCtx carries a logger, not
cancellation. doWithFallback needs no parameter, because it clones the request
with req.Context() and so inherits the binding. This also repairs
internal/controld/controld_test.go, which is behind //go:build controld and had
already been written against the context-taking signature, so it could not
compile.

Cover the cancellation paths; removing either check makes the tests hang until
timeout.

Sampling the stop state is the whole point of runAPIPreflight rather than doing
this inline. A stop and a failure need opposite handling - one exits quietly, the
other self-uninstalls a deleted device, surfaces the error to a mobile app, and
reports a failed start - so the two must not be confused. Reading it from the
context after cancelling would report "stopped" for every failure, since
CancelFunc sets ctx.Err() regardless of whether anyone asked to stop; the stop
channel is read directly instead, which also does not depend on the context
watcher goroutine having been scheduled.

cmd/cli/cli.go

@@ -318,23 +318,56 @@ func run(appCallback *AppCallback, stopCh chan struct{}) {
 	}
 	if cdUID != "" {
 		validateCdUpstreamProtocol()
-		if rc, err := processCDFlags(&cfg); err != nil {
+		// Bound API preflight by the service lifetime. Without this, a stop request
+		// arriving while the API is unreachable leaves this retry/backoff loop running
+		// after "service stopped" was logged, so the process keeps working on behalf of
+		// a service the OS considers stopped.
+		pf := runAPIPreflight(p.stopCh, &cfg)
+		switch {
+		case pf.stopRequested:
+			// Stop requested during preflight, whether or not the fetch itself
+			// succeeded. A successful fetch does not entitle startup to continue: the
+			// operator asked for a stop, and carrying on would set up listeners and
+			// interception for a service the OS already considers stopping.
+			//
+			// Exit the way a normal stop does: no Fatal, so the OS service manager does
+			// not see a failed start and apply its restart policy to a service the
+			// operator just asked to stop.
+			mainLog.Load().Notice().Msg("stop requested while fetching resolver config, shutting down")
+			notifyExitToLogServer()
+			return
+		case pf.err != nil:
 			if isMobile() {
-				appCallback.Exit(err.Error())
+				appCallback.Exit(pf.err.Error())
 				return
 			}
 
 			cdLogger := mainLog.Load().With().Str("mode", "cd").Logger()
 			// Performs self-uninstallation if the ControlD device does not exist.
 			var uer *controld.ErrorResponse
-			if errors.As(err, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode {
+			if errors.As(pf.err, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode {
 				_ = uninstallInvalidCdUID(p, cdLogger, false)
 			}
+			if rejection, ok := permanentAPIRejection(pf.err); ok {
+				// The API answered and rejected this request permanently. Restarting
+				// cannot change that answer, so exit cleanly rather than through Fatal:
+				// an abnormal exit spends one of the service manager's restart actions,
+				// and on Windows those are what bring enforcement back after a real
+				// crash. Burning that budget on a config problem also buries the API's
+				// reason under repeated start failures.
+				cdLogger.Error().Err(pf.err).Int("status", rejection.StatusCode).Msg("failed to fetch resolver config, the API rejected this configuration")
+				notifyExitToLogServer()
+				return
+			}
 			notifyExitToLogServer()
-			cdLogger.Fatal().Err(err).Msg("failed to fetch resolver config")
-		} else {
+			// Everything else - a denied socket, an unreachable API, a proxy in the way,
+			// an API that is having a bad day - is a condition a later start may not hit,
+			// so keep the abnormal exit and let the service manager's recovery policy
+			// retry.
+			cdLogger.Fatal().Err(pf.err).Msg("failed to fetch resolver config")
+		default:
 			p.mu.Lock()
-			p.rc = rc
+			p.rc = pf.rc
 			p.mu.Unlock()
 		}
 	}
@@ -649,24 +682,148 @@ func deactivationPinSet() bool {
 	return cdDeactivationPin.Load() != defaultDeactivationPin
 }
 
-func processCDFlags(cfg *ctrld.Config) (*controld.ResolverConfig, error) {
+// fetchResolverConfig is a test seam for the ControlD resolver-config API call.
+var fetchResolverConfig = controld.FetchResolverConfig
+
+// apiPreflight is the outcome of the API preflight fetch: the resolver config, the
+// error if any, and whether the service was asked to stop while it ran.
+type apiPreflight struct {
+	rc            *controld.ResolverConfig
+	err           error
+	stopRequested bool
+}
+
+// runAPIPreflight fetches the ControlD resolver config bounded by the service
+// lifetime, and reports whether a stop was requested while it ran.
+//
+// The distinction matters because the caller does very different things with it: a stop
+// exits quietly, while a failure self-uninstalls a deleted device, surfaces the error to
+// a mobile app, and reports a failed start to the service manager.
+//
+// stopRequested must not be derived from the context once it has been cancelled.
+// context.CancelFunc sets ctx.Err() unconditionally, so reading it after the cancel
+// classifies *every* failure - a deleted device, an exhausted retry, a mobile caller
+// with no stop channel - as an operator stop. Reading the stop channel directly is also
+// independent of whether the context's watcher goroutine has been scheduled yet.
+func runAPIPreflight(stopCh <-chan struct{}, cfg *ctrld.Config) apiPreflight {
+	rc, err := fetchCDConfigBoundedBy(stopCh, cfg)
+	return apiPreflight{rc: rc, err: err, stopRequested: stopRequested(stopCh)}
+}
+
+// permanentAPIRejection reports whether err is the API refusing this request in a way
+// that a restart cannot change, and returns the rejection when it is.
+//
+// The type alone does not answer this. controld builds an *ErrorResponse for *any*
+// non-200 whose body decodes, so a 502 from a load balancer and a 404 for a deleted
+// device arrive as the same Go type. Treating both as permanent would let a few minutes
+// of API trouble stop ctrld on every host with no service-manager retry behind it, which
+// is strictly worse than the abnormal exit it replaced.
+//
+// So the HTTP status decides, and only a client-error status counts:
+//
+//   - 4xx: the API examined this request and refused it - a deleted device, a revoked
+//     token, a malformed UID. The same request will be refused again.
+//   - 408 and 429 are the exceptions: they are the API asking for another attempt later.
+//   - 5xx, or no recorded status, says nothing about this configuration. Retry.
+func permanentAPIRejection(err error) (*controld.ErrorResponse, bool) {
+	var uer *controld.ErrorResponse
+	if !errors.As(err, &uer) {
+		return nil, false
+	}
+	switch uer.StatusCode {
+	case http.StatusRequestTimeout, http.StatusTooManyRequests:
+		return nil, false
+	}
+	if uer.StatusCode < 400 || uer.StatusCode >= 500 {
+		return nil, false
+	}
+	return uer, true
+}
+
+// processCDFlagsFn is the API fetch, indirected so the lifetime binding around it can be
+// tested without reaching the network.
+var processCDFlagsFn = processCDFlags
+
+// fetchCDConfigBoundedBy runs the API fetch bounded by stopCh, so a fetch that cannot
+// reach the API stops when the service is asked to stop instead of working on behalf of a
+// service the OS already considers stopped. The derived context is always cancelled, which
+// releases the goroutine watching stopCh.
+func fetchCDConfigBoundedBy(stopCh <-chan struct{}, cfg *ctrld.Config) (*controld.ResolverConfig, error) {
+	ctx, cancel := contextFromStopCh(stopCh)
+	defer cancel()
+	return processCDFlagsFn(ctx, cfg)
+}
+
+// fetchCDConfigBoundedByLifetime is the reload path's fetch. Reload binds the same stop
+// primitives as startup - it used to wire them up itself, where a dropped cancel or the
+// wrong channel would have failed nothing.
+func (p *prog) fetchCDConfigBoundedByLifetime(cfg *ctrld.Config) (*controld.ResolverConfig, error) {
+	return fetchCDConfigBoundedBy(p.stopCh, cfg)
+}
+
+// stopRequested reports whether stopCh has been closed. A nil channel - mobile passes
+// none - blocks forever, so the default case is taken and it reads as "no stop".
+func stopRequested(stopCh <-chan struct{}) bool {
+	select {
+	case <-stopCh:
+		return true
+	default:
+		return false
+	}
+}
+
+// contextFromStopCh returns a context that is cancelled when stopCh closes, so
+// long-running startup work stops as soon as the service is asked to stop. The
+// returned cancel func must be called to release the watcher goroutine.
+func contextFromStopCh(stopCh <-chan struct{}) (context.Context, context.CancelFunc) {
+	ctx, cancel := context.WithCancel(context.Background())
+	if stopCh == nil {
+		return ctx, cancel
+	}
+	go func() {
+		select {
+		case <-stopCh:
+			cancel()
+		case <-ctx.Done():
+		}
+	}()
+	return ctx, cancel
+}
+
+// processCDFlags fetches the ControlD configuration for cdUID and applies it to cfg.
+//
+// ctx bounds the bootstrap-DNS retry loop below. That loop retries indefinitely by
+// design (a device with no network yet must eventually come up), so it must be
+// cancellable: otherwise a stop request during preflight is ignored and the process
+// keeps retrying after the service reports itself stopped.
+func processCDFlags(ctx context.Context, cfg *ctrld.Config) (*controld.ResolverConfig, error) {
 	logger := mainLog.Load().With().Str("mode", "cd").Logger()
 	logger.Info().Msgf("fetching Controld D configuration from API: %s", cdUID)
 	bo := backoff.NewBackoff("processCDFlags", logf, 30*time.Second)
 	bo.LogLongerThan = 30 * time.Second
 
-	ctx := context.Background()
+	if ctx == nil {
+		ctx = context.Background()
+	}
 	req := &controld.ResolverConfigRequest{
 		RawUID:   cdUID,
 		Version:  rootCmd.Version,
-		Metadata: ctrld.SystemMetadataRuntime(context.Background()),
+		Metadata: ctrld.SystemMetadataRuntime(ctx),
 	}
-	resolverConfig, err := controld.FetchResolverConfig(req, cdDev)
+	resolverConfig, err := fetchResolverConfig(ctx, req, cdDev)
 	for {
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			logger.Debug().Msg("resolver config fetch cancelled")
+			return nil, ctxErr
+		}
 		if errUrlNetworkError(err) {
 			bo.BackOff(ctx, err)
+			if ctxErr := ctx.Err(); ctxErr != nil {
+				logger.Debug().Msg("resolver config fetch cancelled during backoff")
+				return nil, ctxErr
+			}
 			logger.Warn().Msg("could not fetch resolver using bootstrap DNS, retrying...")
-			resolverConfig, err = controld.FetchResolverConfig(req, cdDev)
+			resolverConfig, err = fetchResolverConfig(ctx, req, cdDev)
 			continue
 		}
 		break
@@ -1644,7 +1801,7 @@ func cdUIDFromProvToken() string {
 		Metadata:  ctrld.SystemMetadata(context.Background()),
 	}
 	// Process provision token if provided.
-	resolverConfig, err := controld.FetchResolverUID(req, rootCmd.Version, cdDev)
+	resolverConfig, err := controld.FetchResolverUID(context.Background(), req, rootCmd.Version, cdDev)
 	if err != nil {
 		mainLog.Load().Fatal().Err(err).Msgf("failed to fetch resolver uid with provision token: %s", redactToken(cdOrg))
 	}
@@ -1998,7 +2155,7 @@ func doValidateCdRemoteConfig(cdUID string, fatal bool) error {
 		Version:  rootCmd.Version,
 		Metadata: ctrld.SystemMetadataRuntime(context.Background()),
 	}
-	rc, err := controld.FetchResolverConfig(req, cdDev)
+	rc, err := controld.FetchResolverConfig(context.Background(), req, cdDev)
 	if err != nil {
 		logger := mainLog.Load().Fatal()
 		if !fatal {