ctrld/cmd/cli/dns_proxy.go

package cli

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"errors"
	"fmt"
	"net"
	"net/netip"
	"os/exec"
	"runtime"
	"slices"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/miekg/dns"
	"golang.org/x/sync/errgroup"
	"tailscale.com/net/netmon"
	"tailscale.com/net/tsaddr"

	"github.com/Control-D-Inc/ctrld"
	"github.com/Control-D-Inc/ctrld/internal/controld"
	"github.com/Control-D-Inc/ctrld/internal/dnscache"
	ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
	"github.com/Control-D-Inc/ctrld/internal/rulematcher"
)

// DNS proxy constants for configuration and behavior control
const (
	// staleTTL is the TTL for stale cache entries
	// This allows serving cached responses even when upstreams are temporarily unavailable
	staleTTL = 60 * time.Second

	// localTTL is the TTL for local network responses
	// Longer TTL for local queries reduces unnecessary repeated lookups
	localTTL = 3600 * time.Second
	// zpaDNSEchoDomain is Zscaler Client Connector's fixed public health-check
	// name. Client Connector must see the answer to it arrive on its own DNS
	// path before it will synthesize addresses for Private Access applications.
	zpaDNSEchoDomain = "dnsechotest.zscaler.com"
	// zpaDNSEchoPolicyName is reported as the matched policy when the built-in
	// ZPA DNS echo bypass routes a query, so log lines name the reason.
	zpaDNSEchoPolicyName = "Built-in ZPA DNS echo bypass"
	// noRuleMatched is the matchedRule placeholder used when no listener domain
	// rule matched the query. It is compared, not only logged: the built-in ZPA
	// echo bypass yields to an explicit domain rule for the same name.
	noRuleMatched = "no rule"

	// EDNS0_OPTION_MAC is dnsmasq EDNS0 code for adding mac option.
	// https://thekelleys.org.uk/gitweb/?p=dnsmasq.git;a=blob;f=src/dns-protocol.h;h=76ac66a8c28317e9c121a74ab5fd0e20f6237dc8;hb=HEAD#l81
	// This is also dns.EDNS0LOCALSTART, but define our own constant here for clarification.
	// This enables MAC address-based client identification for policy routing
	EDNS0_OPTION_MAC = 0xFDE9

	// selfUninstallMaxQueries is number of REFUSED queries seen before checking for self-uninstallation.
	// This prevents premature self-uninstallation due to temporary network issues
	selfUninstallMaxQueries = 32
)

// zpaDNSEchoBypassEnabled limits the built-in ZPA DNS echo bypass to macOS,
// the only platform where pf interception is known to break Zscaler Private
// Access. Tests override it.
var zpaDNSEchoBypassEnabled = runtime.GOOS == "darwin"

// zpaDNSEchoBypassActive reports whether the built-in Zscaler DNS echo bypass
// applies. Like every other VPN accommodation it is skipped in
// --intercept-mode hard, where the operator has asked for all DNS to go
// through ctrld with no split routing.
func zpaDNSEchoBypassActive() bool {
	return zpaDNSEchoBypassEnabled && dnsIntercept && !hardIntercept
}

// osUpstreamConfig defines the default OS resolver configuration
// This is used as a fallback when all configured upstreams fail
var osUpstreamConfig = &ctrld.UpstreamConfig{
	Name:    "OS resolver",
	Type:    ctrld.ResolverTypeOS,
	Timeout: 3000,
}

// privateUpstreamConfig defines the default private resolver configuration
// This is used for internal network queries that should not go to public resolvers
var privateUpstreamConfig = &ctrld.UpstreamConfig{
	Name:    "Private resolver",
	Type:    ctrld.ResolverTypePrivate,
	Timeout: 2000,
}

// proxyRequest contains data for proxying a DNS query to upstream.
// This structure encapsulates all the information needed to process a DNS request
type proxyRequest struct {
	msg             *dns.Msg
	ci              *ctrld.ClientInfo
	failoverRcodes  []int
	ufr             *upstreamForResult
	staleAnswer     *dns.Msg
	isLanOrPtrQuery bool
	upstreamConfigs []*ctrld.UpstreamConfig
}

// proxyResponse contains data for proxying a DNS response from upstream.
// This structure encapsulates the response and metadata for logging and metrics
type proxyResponse struct {
	answer     *dns.Msg
	upstream   string
	cached     bool
	clientInfo bool
	refused    bool
}

// upstreamForResult represents the result of processing rules for a request.
// This contains the matched policy information for logging and debugging
type upstreamForResult struct {
	upstreams      []string
	matchedPolicy  string
	matchedNetwork string
	matchedRule    string
	matched        bool
	srcAddr        string
	// zpaDNSEcho keeps the health check on the OS path and out of the cache.
	zpaDNSEcho bool
}

func (p *prog) addCachedResponse(key dnscache.Key, answer *dns.Msg) {
	ttl := ttlFromMsg(answer)
	now := time.Now()
	expired := now.Add(time.Duration(ttl) * time.Second)
	if cachedTTL := p.cfg.Service.CacheTTLOverride; cachedTTL > 0 {
		expired = now.Add(time.Duration(cachedTTL) * time.Second)
	}
	setCachedAnswerTTL(answer, now, expired)
	p.cache.Add(key, dnscache.NewValue(answer, expired))
}

func (p *prog) cachedResponse(req *dns.Msg, upstream string, dns64Prefix netip.Prefix, dns64Active bool, now time.Time) (answer, stale *dns.Msg, hit, dns64Hit, dns64Bypass bool) {
	if dns64Active && dns64RequestAllowed(req) {
		if cachedValue := p.cache.Get(dns64CacheKey(req, upstream, dns64Prefix)); cachedValue != nil {
			answer = cachedValue.Msg.Copy()
			ctrld.SetCacheReply(answer, req, answer.Rcode)
			if cachedValue.Expire.After(now) {
				setCachedAnswerTTL(answer, now, cachedValue.Expire)
				return answer, nil, true, true, false
			}
			stale = answer
		}
	}

	cachedValue := p.cache.Get(dnscache.NewKey(req, upstream))
	if cachedValue == nil {
		return nil, stale, false, false, false
	}
	answer = cachedValue.Msg.Copy()
	ctrld.SetCacheReply(answer, req, answer.Rcode)
	if cachedValue.Expire.After(now) {
		if dns64Eligible(req, answer) && dns64Active {
			if stale == nil {
				stale = answer
			}
			return nil, stale, false, false, true
		}
		setCachedAnswerTTL(answer, now, cachedValue.Expire)
		return answer, stale, true, false, false
	}
	if stale == nil {
		stale = answer
	}
	return nil, stale, false, false, false
}

// serveDNS sets up and starts a DNS server on the specified listener, handling DNS queries and network monitoring.
// This is the main entry point for DNS server functionality
func (p *prog) serveDNS(ctx context.Context, listenerNum string) error {
	logger := p.logger.Load()
	logger.Debug().Msg("DNS server setup started")

	listenerConfig := p.cfg.Listener[listenerNum]
	if allocErr := p.allocateIP(listenerConfig.IP); allocErr != nil {
		p.Error().Err(allocErr).Str("ip", listenerConfig.IP).Msg("serveUDP: Failed to allocate listen IP")
		return allocErr
	}

	handler := dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) {
		p.handleDNSQuery(w, m, listenerNum, listenerConfig)
	})

	logger.Debug().Msg("DNS server setup completed")
	return p.startListeners(ctx, listenerConfig, handler)
}

// startListeners starts DNS listeners on specified configurations, supporting UDP and TCP protocols.
// It handles local IPv6, RFC 1918, and specified IP listeners, reacting to stop signals or errors.
// This function manages the lifecycle of DNS server listeners
func (p *prog) startListeners(ctx context.Context, cfg *ctrld.ListenerConfig, handler dns.Handler) error {
	logger := p.logger.Load()
	logger.Debug().Msg("Starting DNS listeners")

	g, gctx := errgroup.WithContext(ctx)

	for _, proto := range []string{"udp", "tcp"} {
		if needLocalIPv6Listener() {
			logger.Debug().Str("protocol", proto).Msg("Starting local IPv6 listener")
			g.Go(func() error {
				s, errCh := runDNSServer(net.JoinHostPort("::1", strconv.Itoa(cfg.Port)), proto, handler)
				defer s.Shutdown()
				select {
				case <-p.stopCh:
				case <-p.runAbortCh:
				case <-gctx.Done():
				case err := <-errCh:
					p.Warn().Err(err).Msg("Local IPv6 listener failed")
				}
				return nil
			})
		}

		// When we spawn a listener on 127.0.0.1, also spawn listeners on the RFC1918 addresses of the machine
		// if explicitly set via setting rfc1918 flag, so ctrld could receive queries from LAN clients.
		if needRFC1918Listeners(cfg) {
			logger.Debug().Str("protocol", proto).Msg("Starting RFC1918 listeners")
			g.Go(func() error {
				for _, addr := range ctrld.Rfc1918Addresses() {
					func() {
						listenAddr := net.JoinHostPort(addr, strconv.Itoa(cfg.Port))
						s, errCh := runDNSServer(listenAddr, proto, handler)
						defer s.Shutdown()
						select {
						case <-p.stopCh:
						case <-p.runAbortCh:
						case <-gctx.Done():
						case err := <-errCh:
							p.Warn().Err(err).Msgf("Could not listen on %s: %s", proto, listenAddr)
						}
					}()
				}
				return nil
			})
		}

		logger.Debug().Str("protocol", proto).Str("ip", cfg.IP).Int("port", cfg.Port).Msg("Starting main listener")
		g.Go(func() error {
			addr := net.JoinHostPort(cfg.IP, strconv.Itoa(cfg.Port))
			s, errCh := runDNSServer(addr, proto, handler)
			defer s.Shutdown()
			select {
			case p.started <- struct{}{}:
			case <-p.stopCh:
				return nil
			case <-p.runAbortCh:
				return nil
			case <-gctx.Done():
				return nil
			}
			select {
			case <-p.stopCh:
			case <-p.runAbortCh:
			case <-gctx.Done():
			case err := <-errCh:
				return err
			}
			return nil
		})
	}

	logger.Debug().Msg("DNS listeners started successfully")
	return g.Wait()
}

// handleDNSQuery processes incoming DNS queries, validates client access, and routes the query to appropriate handlers.
// This is the main entry point for all DNS query processing
func (p *prog) handleDNSQuery(w dns.ResponseWriter, m *dns.Msg, listenerNum string, listenerConfig *ctrld.ListenerConfig) {
	p.sema.acquire()
	defer p.sema.release()

	if len(m.Question) == 0 {
		sendDNSResponse(w, m, dns.RcodeFormatError)
		return
	}

	reqID := requestID()
	ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, reqID)
	ctx = ctrld.LoggerCtx(ctx, p.logger.Load())

	ctrld.Log(ctx, p.Debug(), "Processing DNS query from %s", w.RemoteAddr().String())

	if !listenerConfig.AllowWanClients && isWanClient(w.RemoteAddr()) {
		ctrld.Log(ctx, p.Debug(), "Query refused, listener does not allow WAN clients: %s", w.RemoteAddr().String())
		sendDNSResponse(w, m, dns.RcodeRefused)
		return
	}

	go p.detectLoop(m)

	q := m.Question[0]
	domain := canonicalName(q.Name)

	if p.handleSpecialDomains(ctx, w, m, domain) {
		ctrld.Log(ctx, p.Debug(), "Special domain query handled")
		return
	}

	ctrld.Log(ctx, p.Debug(), "Processing standard query for domain: %s", domain)
	p.processStandardQuery(&standardQueryRequest{
		ctx:            ctx,
		writer:         w,
		msg:            m,
		listenerNum:    listenerNum,
		listenerConfig: listenerConfig,
		domain:         domain,
	})
}

// handleSpecialDomains processes special domain queries, handles errors, purges cache if necessary, and returns a bool status.
// This handles internal test domains and cache management commands
func (p *prog) handleSpecialDomains(ctx context.Context, w dns.ResponseWriter, m *dns.Msg, domain string) bool {
	switch {
	case domain == "":
		ctrld.Log(ctx, p.Debug(), "Empty domain query, sending format error")
		sendDNSResponse(w, m, dns.RcodeFormatError)
		return true
	case domain == selfCheckInternalTestDomain:
		ctrld.Log(ctx, p.Debug(), "Internal test domain query: %s", domain)
		answer := resolveInternalDomainTestQuery(ctx, domain, m)
		_ = w.WriteMsg(answer)
		return true
	}

	// Interception probe: if we're expecting a probe query and this matches,
	// signal the prober and respond NXDOMAIN. Used by both macOS pf probes
	// (_pf-probe-*) and Windows NRPT probes (_nrpt-probe-*) to verify that
	// DNS interception is actually routing queries to ctrld's listener.
	if p.signalInterceptProbe(domain) {
		sendDNSResponse(w, m, dns.RcodeNameError) // NXDOMAIN
		return true
	}

	if _, ok := p.cacheFlushDomainsMap[domain]; ok && p.cache != nil {
		p.cache.Purge()
		ctrld.Log(ctx, p.Debug(), "Received query %q, local cache is purged", domain)
	}

	return false
}

// standardQueryRequest represents a standard DNS query request with associated context and configuration.
// This encapsulates all the data needed to process a standard DNS query
type standardQueryRequest struct {
	ctx            context.Context
	writer         dns.ResponseWriter
	msg            *dns.Msg
	listenerNum    string
	listenerConfig *ctrld.ListenerConfig
	domain         string
}

// processStandardQuery handles a standard DNS query by routing it through appropriate upstreams and writing a DNS response.
// This is the main processing pipeline for normal DNS queries
func (p *prog) processStandardQuery(req *standardQueryRequest) {
	ctrld.Log(req.ctx, p.Debug(), "Processing standard query started")

	remoteIP, _, _ := net.SplitHostPort(req.writer.RemoteAddr().String())
	ci := p.getClientInfo(remoteIP, req.msg)
	ci.ClientIDPref = p.cfg.Service.ClientIDPref

	stripClientSubnet(req.msg)
	remoteAddr := spoofRemoteAddr(req.writer.RemoteAddr(), ci)
	fmtSrcToDest := fmtRemoteToLocal(req.listenerNum, ci.Hostname, remoteAddr.String())

	startTime := time.Now()
	q := req.msg.Question[0]
	ctrld.Log(req.ctx, p.Info(), "QUERY: %s: %s %s", fmtSrcToDest, dns.TypeToString[q.Qtype], req.domain)

	ur := p.upstreamFor(req.ctx, req.listenerNum, req.listenerConfig, remoteAddr, ci.Mac, req.domain)

	var answer *dns.Msg
	// Handle restricted listener case
	if !ur.matched && req.listenerConfig.Restricted {
		ctrld.Log(req.ctx, p.Debug(), "Query refused, %s does not match any network policy", remoteAddr.String())
		answer = new(dns.Msg)
		answer.SetRcode(req.msg, dns.RcodeRefused)
		// The grade of the window must not depend on the goroutine below.
		p.health.countQuery()
		// Process the refused query
		go p.postProcessStandardQuery(ci, req.listenerConfig, q, &proxyResponse{answer: answer, refused: true})
	} else {
		// Process a normal query
		ctrld.Log(req.ctx, p.Debug(), "Starting proxy query processing")
		pr := p.proxy(req.ctx, &proxyRequest{
			msg:            req.msg,
			ci:             ci,
			failoverRcodes: p.getFailoverRcodes(req.listenerConfig),
			ufr:            ur,
		})

		rtt := time.Since(startTime)
		ctrld.Log(req.ctx, p.Debug(), "Received response of %d bytes in %s", pr.answer.Len(), rtt)

		// Firewall mode must learn resolved IPs before the DNS response is sent
		// back to the client. Otherwise the app can receive the answer and attempt
		// the first connection before the platform allowlist has been updated.
		if p.firewallModeEnabled() && pr.answer != nil {
			p.firewallRecordResolvedIPs(pr.answer, canonicalName(q.Name))
		}

		// The grade of the window must not depend on the goroutine below.
		p.health.countQuery()
		go p.postProcessStandardQuery(ci, req.listenerConfig, q, pr)
		answer = pr.answer
	}

	if err := req.writer.WriteMsg(answer); err != nil {
		ctrld.Log(req.ctx, p.querySampler.event(sampleClassSendResponseFailed, "").Err(err), "serveDNS: failed to send DNS response to client")
	}

	ctrld.Log(req.ctx, p.Debug(), "Standard query processing completed")
}

// postProcessStandardQuery performs additional actions after processing a standard DNS query, such as metrics recording,
// handling canonical name adjustments, and triggering specific post-query actions like uninstallation procedures.
func (p *prog) postProcessStandardQuery(ci *ctrld.ClientInfo, listenerConfig *ctrld.ListenerConfig, q dns.Question, pr *proxyResponse) {
	p.doSelfUninstall(pr)
	p.recordMetrics(ci, listenerConfig, q, pr)
	p.forceFetchingAPI(canonicalName(q.Name))

}

// getFailoverRcodes retrieves the failover response codes from the provided ListenerConfig. Returns nil if no policy exists.
func (p *prog) getFailoverRcodes(cfg *ctrld.ListenerConfig) []int {
	if cfg.Policy != nil {
		return cfg.Policy.FailoverRcodeNumbers
	}
	return nil
}

// recordMetrics updates Prometheus metrics for DNS queries, including query count and client-specific query statistics.
func (p *prog) recordMetrics(ci *ctrld.ClientInfo, cfg *ctrld.ListenerConfig, q dns.Question, pr *proxyResponse) {
	upstream := pr.upstream
	switch {
	case pr.cached:
		upstream = "cache"
	case pr.clientInfo:
		upstream = "client_info_table"
	}
	labelValues := []string{
		net.JoinHostPort(cfg.IP, strconv.Itoa(cfg.Port)),
		ci.IP,
		ci.Mac,
		ci.Hostname,
		upstream,
		dns.TypeToString[q.Qtype],
		dns.RcodeToString[pr.answer.Rcode],
	}
	p.WithLabelValuesInc(statsQueriesCount, labelValues...)
	p.WithLabelValuesInc(statsClientQueriesCount, []string{ci.IP, ci.Mac, ci.Hostname}...)
}

// sendDNSResponse sends a DNS response with the specified RCODE to the client using the provided ResponseWriter.
func sendDNSResponse(w dns.ResponseWriter, m *dns.Msg, rcode int) {
	answer := new(dns.Msg)
	answer.SetRcode(m, rcode)
	_ = w.WriteMsg(answer)
}

// upstreamForRequest contains all parameters needed for upstream determination
type upstreamForRequest struct {
	DefaultUpstreamNum string
	ListenerConfig     *ctrld.ListenerConfig
	Addr               net.Addr
	SrcMac             string
	Domain             string
	MatchingConfig     *rulematcher.MatchingConfig
}

// upstreamFor returns the list of upstreams for resolving the given domain,
// matching by policies defined in the listener config. The second return value
// reports whether the domain matches the policy.
//
// Though domain policy has higher priority than network policy, it is still
// processed later, because policy logging want to know whether a network rule
// is disregarded in favor of the domain level rule.
func (p *prog) upstreamFor(ctx context.Context, defaultUpstreamNum string, lc *ctrld.ListenerConfig, addr net.Addr, srcMac, domain string) (res *upstreamForResult) {
	var matchingConfig *rulematcher.MatchingConfig
	if lc.Policy != nil && lc.Policy.Matching != nil {
		// Convert string-based order to RuleType enum
		var order []rulematcher.RuleType
		for _, ruleTypeStr := range lc.Policy.Matching.Order {
			switch ruleTypeStr {
			case "network":
				order = append(order, rulematcher.RuleTypeNetwork)
			case "mac":
				order = append(order, rulematcher.RuleTypeMac)
			case "domain":
				order = append(order, rulematcher.RuleTypeDomain)
			}
		}

		matchingConfig = &rulematcher.MatchingConfig{
			Order: order,
		}
	}

	req := &upstreamForRequest{
		DefaultUpstreamNum: defaultUpstreamNum,
		ListenerConfig:     lc,
		Addr:               addr,
		SrcMac:             srcMac,
		Domain:             domain,
		MatchingConfig:     matchingConfig,
	}

	return p.upstreamForWithConfig(ctx, req)
}

// upstreamForWithConfig determines upstreams using configurable rule matching
func (p *prog) upstreamForWithConfig(ctx context.Context, req *upstreamForRequest) (res *upstreamForResult) {
	// Default upstreams
	upstreams := []string{upstreamPrefix + req.DefaultUpstreamNum}
	res = &upstreamForResult{srcAddr: req.Addr.String()}
	defer func() {
		// Zscaler Private Access: under macOS Intercept Mode, pf routes Client
		// Connector's health-check query for zpaDNSEchoDomain into ctrld, which
		// answers it from the Control D upstream. Client Connector never sees the
		// answer on its own DNS path, so it keeps Private Access disabled.
		//
		// Route only that one exact name the way a Control D "bypass" rule does:
		// an empty upstream list, which proxy() resolves through upstream.os —
		// the system resolver set, which under Intercept Mode still includes
		// Client Connector's own DNS servers. Matching on the name alone covers
		// every record type, the same as a bypass rule.
		//
		// This runs after policy evaluation, and never touches `matched`. That
		// flag is also the source authorization bit: serveDNS refuses a query on
		// a Restricted listener when it is false, so forcing it true here would
		// let an unauthorized client resolve this name through such a listener.
		//
		// The scope is one fixed public health-check name: every other domain,
		// including all Private Access application domains, stays intercepted and
		// filtered by Control D. --intercept-mode hard opts out, the same as it
		// opts out of VPN DNS split routing.
		if zpaDNSEchoBypassActive() && canonicalName(req.Domain) == zpaDNSEchoDomain {
			switch {
			case res.matchedRule == noRuleMatched:
				// The listener policy says nothing about this name, so the
				// built-in route decides. Network- and MAC-policy targets route
				// every name from a source and so state nothing about this one:
				// override them, and the default upstream, and label the result
				// so logs name the reason.
				res.upstreams = nil
				res.matchedPolicy = zpaDNSEchoPolicyName
				res.matchedRule = zpaDNSEchoDomain
				res.zpaDNSEcho = true
			case len(res.upstreams) == 0:
				// An explicit domain rule already selects the OS path: a rule
				// with empty targets, which is how a Control D profile's bypass
				// list reaches us (see the cfg.Listener rules built from
				// resolverConfig.Exclude). That is the documented pre-fix
				// workaround for this very bug, so keep the operator's own
				// labels but still classify it as the health check — otherwise
				// anyone who keeps the workaround after upgrading would route
				// correctly and then have the answer served from our cache,
				// losing the reconnect fix.
				res.zpaDNSEcho = true
			default:
				// Any other explicit domain rule is a deliberate non-OS route
				// for this exact name. That is operator intent, and the
				// supported way to opt out without leaving Intercept Mode, so
				// leave the policy's decision untouched.
			}
		}
	}()

	// If no policy, return default upstreams
	if req.ListenerConfig.Policy == nil {
		res.upstreams = upstreams
		res.matched = false
		res.matchedPolicy = "no policy"
		res.matchedNetwork = "no network"
		res.matchedRule = noRuleMatched
		return
	}

	// Extract source IP from address
	var sourceIP net.IP
	switch addr := req.Addr.(type) {
	case *net.UDPAddr:
		sourceIP = addr.IP
	case *net.TCPAddr:
		sourceIP = addr.IP
	}

	// Create match request
	matchRequest := &rulematcher.MatchRequest{
		SourceIP:  sourceIP,
		SourceMac: req.SrcMac,
		Domain:    req.Domain,
		Policy:    req.ListenerConfig.Policy,
		Config:    p.cfg,
	}

	// Use matching engine to find upstreams
	engine := rulematcher.NewMatchingEngine(req.MatchingConfig)
	matchResult := engine.FindUpstreams(ctx, matchRequest)

	// Convert result to upstreamForResult format
	res.upstreams = matchResult.Upstreams
	res.matched = matchResult.Matched
	res.matchedPolicy = matchResult.MatchedPolicy
	res.matchedNetwork = matchResult.MatchedNetwork
	res.matchedRule = matchResult.MatchedRule

	// If no match found, use default upstreams
	if !matchResult.Matched {
		res.upstreams = upstreams
	}

	return
}

// proxyPrivatePtrLookup performs a private PTR DNS lookup based on the client info table for the given query.
// It prevents DNS loops by locking the processing of the same domain name simultaneously.
// If a valid IP-to-hostname mapping exists, it creates a PTR DNS record as the response.
// Returns the DNS response if a hostname is found or nil otherwise.
func (p *prog) proxyPrivatePtrLookup(ctx context.Context, msg *dns.Msg) *dns.Msg {
	cDomainName := msg.Question[0].Name
	locked := p.ptrLoopGuard.TryLock(cDomainName)
	defer p.ptrLoopGuard.Unlock(cDomainName)
	if !locked {
		return nil
	}
	ip := ipFromARPA(cDomainName)
	if name := p.ciTable.LookupHostname(ip.String(), ""); name != "" {
		answer := new(dns.Msg)
		answer.SetReply(msg)
		answer.Compress = true
		answer.Answer = []dns.RR{&dns.PTR{
			Hdr: dns.RR_Header{
				Name:   msg.Question[0].Name,
				Rrtype: dns.TypePTR,
				Class:  dns.ClassINET,
			},
			Ptr: dns.Fqdn(name),
		}}
		ctrld.Log(ctx, p.Info(), "Private PTR lookup, using client info table")
		ctrld.Log(ctx, p.Debug(), "Client info: %v", ctrld.ClientInfo{
			Mac:      p.ciTable.LookupMac(ip.String()),
			IP:       ip.String(),
			Hostname: name,
		})
		return answer
	}
	return nil
}

// proxyLanHostnameQuery resolves LAN hostnames to their corresponding IP addresses based on the dns.Msg request.
// It uses a loop guard mechanism to prevent DNS query loops and ensures a hostname is processed only once at a time.
// This method queries the client info table for the hostname's IP address and logs relevant debug and client info.
// If the hostname matches known IPs in the table, it generates an appropriate dns.Msg response; otherwise, it returns nil.
func (p *prog) proxyLanHostnameQuery(ctx context.Context, msg *dns.Msg) *dns.Msg {
	q := msg.Question[0]
	hostname := strings.TrimSuffix(q.Name, ".")
	locked := p.lanLoopGuard.TryLock(hostname)
	defer p.lanLoopGuard.Unlock(hostname)
	if !locked {
		return nil
	}
	if ip := p.ciTable.LookupIPByHostname(hostname, q.Qtype == dns.TypeAAAA); ip != nil {
		answer := new(dns.Msg)
		answer.SetReply(msg)
		answer.Compress = true
		switch {
		case ip.Is4():
			answer.Answer = []dns.RR{&dns.A{
				Hdr: dns.RR_Header{
					Name:   msg.Question[0].Name,
					Rrtype: dns.TypeA,
					Class:  dns.ClassINET,
					Ttl:    uint32(localTTL.Seconds()),
				},
				A: ip.AsSlice(),
			}}
		case ip.Is6():
			answer.Answer = []dns.RR{&dns.AAAA{
				Hdr: dns.RR_Header{
					Name:   msg.Question[0].Name,
					Rrtype: dns.TypeAAAA,
					Class:  dns.ClassINET,
					Ttl:    uint32(localTTL.Seconds()),
				},
				AAAA: ip.AsSlice(),
			}}
		}
		ctrld.Log(ctx, p.Info(), "Lan hostname lookup, using client info table")
		ctrld.Log(ctx, p.Debug(), "Client info: %v", ctrld.ClientInfo{
			Mac:      p.ciTable.LookupMac(ip.String()),
			IP:       ip.String(),
			Hostname: hostname,
		})
		return answer
	}
	return nil
}

// handleSpecialQueryTypes processes specific types of DNS queries such as SRV, PTR, and LAN hostname lookups.
// It modifies upstreams and upstreamConfigs based on the query type and updates the query context accordingly.
// Returns a proxyResponse if the query is resolved locally; otherwise, returns nil to proceed with upstream processing.
func (p *prog) handleSpecialQueryTypes(ctx *context.Context, req *proxyRequest, upstreams *[]string, upstreamConfigs *[]*ctrld.UpstreamConfig) *proxyResponse {
	if req.ufr.matched {
		ctrld.Log(*ctx, p.Debug(), "%s, %s, %s -> %v",
			req.ufr.matchedPolicy, req.ufr.matchedNetwork, req.ufr.matchedRule, *upstreams)
		// A rule sending an Active Directory name to the OS resolver (the
		// auto-added AD split rule, or an operator's equivalent) is a LAN
		// lookup: keep it off Control D's public resolver.
		if osUpstreamOnly(*upstreams) && len(req.msg.Question) > 0 && inActiveDirectoryDomain(req.msg.Question[0].Name) {
			*ctx = ctrld.LanQueryCtx(*ctx)
			ctrld.Log(*ctx, p.Debug(), "Active Directory domain lookup, skipping public DNS")
		}
		return nil
	}

	switch {
	case req.ufr.zpaDNSEcho:
		ctrld.Log(*ctx, p.Debug(), "%s, %s -> %v", req.ufr.matchedPolicy, req.ufr.matchedRule, *upstreams)
		return nil
	case isSrvLanLookup(req.msg):
		*upstreams = []string{upstreamOS}
		*upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig}
		*ctx = ctrld.LanQueryCtx(*ctx)
		ctrld.Log(*ctx, p.Debug(), "SRV record lookup, using upstreams: %v", *upstreams)
		return nil
	case isPrivatePtrLookup(req.msg):
		req.isLanOrPtrQuery = true
		if answer := p.proxyPrivatePtrLookup(*ctx, req.msg); answer != nil {
			return &proxyResponse{answer: answer, clientInfo: true}
		}
		*upstreams, *upstreamConfigs = p.upstreamsAndUpstreamConfigForPtr(*upstreams, *upstreamConfigs)
		*ctx = ctrld.LanQueryCtx(*ctx)
		ctrld.Log(*ctx, p.Debug(), "Private PTR lookup, using upstreams: %v", *upstreams)
		return nil
	case isLanHostnameQuery(req.msg):
		req.isLanOrPtrQuery = true
		if answer := p.proxyLanHostnameQuery(*ctx, req.msg); answer != nil {
			return &proxyResponse{answer: answer, clientInfo: true}
		}
		*upstreams = []string{upstreamOS}
		*upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig}
		*ctx = ctrld.LanQueryCtx(*ctx)
		ctrld.Log(*ctx, p.Debug(), "Lan hostname lookup, using upstreams: %v", *upstreams)
		return nil
	default:
		ctrld.Log(*ctx, p.Debug(), "No explicit policy matched, using default routing -> %v", *upstreams)
		return nil
	}
}

// proxy handles DNS query proxying by selecting upstreams, attempting cache lookups, and querying configured resolvers.
func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse {
	ctrld.Log(ctx, p.Debug(), "Proxy query processing started")

	// DNS intercept recovery bypass: forward all queries to OS/DHCP resolver.
	// This runs when upstreams are unreachable (e.g., captive portal network)
	// and allows the network's DNS to handle authentication pages.
	// Explicit Internal Domain resolvers remain exclusive during recovery:
	// general DNS failure does not authorize disclosing private names to DHCP DNS.
	if dnsIntercept && p.recoveryBypass.Load() && !internalDomainExplicitUpstreams(req.ufr.upstreams) {
		ctrld.Log(ctx, p.Debug(), "Recovery bypass active: forwarding to OS resolver")
		// The bypass runs before rule handling, so an Active Directory name
		// needs its LAN mark here too, or it races the domain controller
		// against Control D's public resolver in plaintext.
		bypassCtx := ctx
		if len(req.msg.Question) > 0 && inActiveDirectoryDomain(req.msg.Question[0].Name) {
			bypassCtx = ctrld.LanQueryCtx(ctx)
			ctrld.Log(ctx, p.Debug(), "Active Directory domain lookup, skipping public DNS")
		}
		answer := p.queryUpstream(bypassCtx, req, upstreamOS, osUpstreamConfig)
		if answer != nil {
			return &proxyResponse{answer: answer, upstream: osUpstreamConfig.Endpoint}
		}
		ctrld.Log(ctx, p.Debug(), "OS resolver failed during recovery bypass")
		// Fall through to normal flow as last resort
	}

	upstreams, upstreamConfigs := p.initializeUpstreams(req)
	ctrld.Log(ctx, p.Debug(), "Initialized upstreams: %v", upstreams)

	if specialRes := p.handleSpecialQueryTypes(&ctx, req, &upstreams, &upstreamConfigs); specialRes != nil {
		ctrld.Log(ctx, p.Debug(), "Special query type handled")
		return specialRes
	}

	if cachedRes := p.tryCache(ctx, req, upstreams); cachedRes != nil {
		ctrld.Log(ctx, p.Debug(), "Cache hit, returning cached response")
		p.health.countCacheHit()
		return cachedRes
	}

	// VPN DNS split routing (only in dns-intercept mode). Explicit Internal
	// Domain resolvers take precedence over auto-detected VPN routes. OS-mode
	// Internal Domains still follow the endpoint's VPN resolver.
	if dnsIntercept && p.vpnDNS != nil && len(req.msg.Question) > 0 && !internalDomainExplicitUpstreams(upstreams) {
		domain := req.msg.Question[0].Name
		if vpnServers := p.vpnDNS.UpstreamForDomain(domain); len(vpnServers) > 0 {
			ctrld.Log(ctx, p.Debug(), "VPN DNS route matched for domain %s, using servers: %v", domain, vpnServers)

			var gotTransportFailure bool
			for _, server := range vpnServers {
				upstreamConfig := p.vpnDNS.upstreamConfigFor(server)
				ctrld.Log(ctx, p.Debug(), "Querying VPN DNS server: %s", server)

				answer := p.queryUpstream(ctx, req, "vpn-dns", upstreamConfig)
				if answer != nil {
					p.vpnDNS.VPNDNSReachable()
					ctrld.Log(ctx, p.Debug(), "VPN DNS query successful")

					// Update cache if enabled
					if p.cache != nil {
						p.updateCache(ctx, req, answer, "vpn-dns")
					}

					return &proxyResponse{answer: answer, cached: false}
				}
				gotTransportFailure = true
				ctrld.Log(ctx, p.Debug(), "VPN DNS server %s failed", server)
			}

			// Explicit VPN DNS routes are authoritative for their suffix. If all
			// routed servers fail at the transport layer while Windows is serving
			// retained VPN DNS state, fail closed instead of leaking VPN/internal
			// names to normal upstreams.
			if gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, vpnServers) {
				ctrld.Log(ctx, p.Debug(),
					"All VPN DNS servers had transport failures for %s; returning SERVFAIL while retained VPN DNS state is active", domain)
				p.countFailedClientQuery(upstreams)
				answer := new(dns.Msg)
				answer.SetRcode(req.msg, dns.RcodeServerFailure)
				return &proxyResponse{answer: answer, cached: false}
			}

			ctrld.Log(ctx, p.Debug(), "All VPN DNS servers failed, falling back to normal upstreams")
		}
	}

	// Domain-less VPN DNS fallback: when a query is going to upstream.os via a
	// split-rule (matched policy) and we have VPN DNS servers with no associated
	// domains, try those servers for this query. This handles cases like F5 VPN
	// where the VPN doesn't advertise DNS search domains but its DNS servers
	// know the internal zones referenced by split-rules (e.g., *.provisur.local).
	// These servers are NOT used for general OS resolver queries to avoid
	// polluting captive portal / DHCP flows.
	if dnsIntercept && p.vpnDNS != nil && req.ufr.matched &&
		len(upstreams) > 0 && upstreams[0] == upstreamOS &&
		len(req.msg.Question) > 0 {
		if dlServers := p.vpnDNS.DomainlessServers(); len(dlServers) > 0 {
			domain := req.msg.Question[0].Name
			ctrld.Log(ctx, p.Debug(),
				"Split-rule query %s going to upstream.os, trying %d domain-less VPN DNS servers first: %v",
				domain, len(dlServers), dlServers)

			var gotDNSAnswer bool
			var gotTransportFailure bool
			for _, server := range dlServers {
				upstreamConfig := p.vpnDNS.upstreamConfigFor(server)
				ctrld.Log(ctx, p.Debug(), "Querying domain-less VPN DNS server: %s", server)

				answer := p.queryUpstream(ctx, req, "vpn-dns", upstreamConfig)
				if answer != nil {
					gotDNSAnswer = true
					p.vpnDNS.VPNDNSReachable()
				}
				if answer != nil && answer.Rcode == dns.RcodeSuccess {
					ctrld.Log(ctx, p.Debug(),
						"Domain-less VPN DNS server %s answered %s successfully", server, domain)
					return &proxyResponse{answer: answer, cached: false}
				}
				if answer != nil {
					ctrld.Log(ctx, p.Debug(),
						"Domain-less VPN DNS server %s returned %s for %s, trying next",
						server, dns.RcodeToString[answer.Rcode], domain)
				} else {
					gotTransportFailure = true
					ctrld.Log(ctx, p.Debug(), "Domain-less VPN DNS server %s failed for %s", server, domain)
				}
			}

			// If every domainless VPN DNS attempt failed before receiving a DNS
			// packet while Windows is serving retained VPN DNS state, fail closed
			// instead of asking LAN/public DNS about internal split-rule names and
			// caching false negatives. Reachable negative DNS responses still fall
			// through to the old OS fallback behavior below.
			if !gotDNSAnswer && gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, dlServers) {
				ctrld.Log(ctx, p.Debug(),
					"All domain-less VPN DNS servers had transport failures for %s; returning SERVFAIL while retained VPN DNS state is active", domain)
				p.countFailedClientQuery(upstreams)
				answer := new(dns.Msg)
				answer.SetRcode(req.msg, dns.RcodeServerFailure)
				return &proxyResponse{answer: answer, cached: false}
			}

			ctrld.Log(ctx, p.Debug(),
				"All domain-less VPN DNS servers failed for %s, falling back to OS resolver", domain)
		}
	}

	ctrld.Log(ctx, p.Debug(), "No cache hit, trying upstreams")
	if res := p.tryUpstreams(ctx, req, upstreams, upstreamConfigs); res != nil {
		ctrld.Log(ctx, p.Debug(), "Upstream query successful")
		return res
	}

	ctrld.Log(ctx, p.Debug(), "All upstreams failed, handling failure")
	return p.handleAllUpstreamsFailure(ctx, req, upstreams)
}

// initializeUpstreams determines which upstreams and configurations to use for a given proxyRequest.
// If no upstreams are configured, it defaults to the operating system's resolver configuration.
// Returns a slice of upstream names and their corresponding configurations.
func (p *prog) initializeUpstreams(req *proxyRequest) ([]string, []*ctrld.UpstreamConfig) {
	upstreams := req.ufr.upstreams
	upstreamConfigs := p.upstreamConfigsFromUpstreamNumbers(upstreams)
	if len(upstreamConfigs) == 0 {
		return []string{upstreamOS}, []*ctrld.UpstreamConfig{osUpstreamConfig}
	}
	return upstreams, upstreamConfigs
}

// tryCache attempts to retrieve a cached response for the given DNS request from specified upstreams.
// Returns a proxyResponse if a cache hit occurs; otherwise, returns nil.
// Skips cache checking if caching is disabled or the request is a PTR query.
// Iterates through the provided upstreams to find a cached response using the checkCache method.
func (p *prog) tryCache(ctx context.Context, req *proxyRequest, upstreams []string) *proxyResponse {
	if p.cache == nil || req.msg.Question[0].Qtype == dns.TypePTR || req.ufr.zpaDNSEcho { // https://www.rfc-editor.org/rfc/rfc1035#section-7.4
		ctrld.Log(ctx, p.Debug(), "Cache disabled, PTR query, or ZPA DNS echo, skipping cache lookup")
		return nil
	}

	ctrld.Log(ctx, p.Debug(), "Checking cache for upstreams: %v", upstreams)
	for _, upstream := range upstreams {
		if res := p.checkCache(ctx, req, upstream); res != nil {
			ctrld.Log(ctx, p.Debug(), "Cache hit found for upstream: %s", upstream)
			return res
		}
	}

	ctrld.Log(ctx, p.Debug(), "No cache hit found")
	return nil
}

// checkCache checks if a cached DNS response exists for the given request and upstream.
// Returns a proxyResponse with the cached response if found and valid, or nil otherwise.
func (p *prog) checkCache(ctx context.Context, req *proxyRequest, upstream string) *proxyResponse {
	dns64Prefix, dns64Active := netip.Prefix{}, false
	if req.msg.Question[0].Qtype == dns.TypeAAAA {
		dns64Prefix, dns64Active = p.activeDNS64Prefix()
	}

	answer, stale, hit, dns64Hit, dns64Bypass := p.cachedResponse(req.msg, upstream, dns64Prefix, dns64Active, time.Now())
	if stale != nil {
		req.staleAnswer = stale
	}
	if dns64Bypass {
		ctrld.Log(ctx, p.Debug(), "DNS64: bypassing cached empty-AAAA answer for synthesis")
	}
	if !hit {
		ctrld.Log(ctx, p.Debug(), "No usable cached value found for upstream: %s", upstream)
		return nil
	}

	if dns64Hit {
		ctrld.Log(ctx, p.Debug(), "DNS64: hit cached response variant")
	} else {
		ctrld.Log(ctx, p.Debug(), "Hit cached response")
	}

	// Firewall mode: refresh allowlist entries from cached responses.
	// Even though these IPs were already added when the response was first
	// resolved, the allowlist entries may have expired (TTL-based reaper)
	// while the DNS cache entry is still valid. Refreshing here ensures
	// the allowlist stays populated for as long as the cached DNS entry is served.
	if p.firewallModeEnabled() {
		domain := canonicalName(req.msg.Question[0].Name)
		p.firewallRecordResolvedIPs(answer, domain)
	}

	return &proxyResponse{answer: answer, cached: true}
}

// updateCache updates the DNS response cache with the given request, response, TTL, and upstream information.
func (p *prog) updateCache(ctx context.Context, req *proxyRequest, answer *dns.Msg, upstream string) {
	if req.ufr.zpaDNSEcho {
		return
	}
	p.addCachedResponse(dnscache.NewKey(req.msg, upstream), answer)
	ctrld.Log(ctx, p.Debug(), "Added cached response")
}

// serveStaleResponse serves a stale cached DNS response when an upstream query fails, updating TTL for cached records.
func (p *prog) serveStaleResponse(ctx context.Context, staleAnswer *dns.Msg) *proxyResponse {
	ctrld.Log(ctx, p.Debug(), "Serving stale cached response")
	p.health.countCacheHit()
	now := time.Now()
	setCachedAnswerTTL(staleAnswer, now, now.Add(staleTTL))
	return &proxyResponse{answer: staleAnswer, cached: true}
}

// handleAllUpstreamsFailure handles the failure scenario when all upstream resolvers fail to respond or process the request.
func (p *prog) handleAllUpstreamsFailure(ctx context.Context, req *proxyRequest, upstreams []string) *proxyResponse {
	ctrld.Log(ctx, p.querySampler.event(sampleClassAllEndpointsFailed, ""), "All %v endpoints failed", journalUpstreamNames(upstreams))

	// An unavailable internal resolver must not trigger endpoint-wide recovery
	// or send the private name through the OS-resolver catch-all.
	if internalDomainExplicitUpstreams(upstreams) {
		ctrld.Log(ctx, p.Debug(), "Internal domain resolvers unreachable; not falling back")
	} else if p.leakOnUpstreamFailure() {
		ctrld.Log(ctx, p.Debug(), "Leak on upstream failure enabled")
		if p.um.countHealthy(upstreams) == 0 {
			ctrld.Log(ctx, p.Debug(), "Selected upstreams unavailable; requesting recovery")
			p.triggerRecovery(upstreams[0] == upstreamOS)
		} else {
			ctrld.Log(ctx, p.Debug(), "One upstream is down but at least one is healthy; skipping recovery trigger")
		}

		if upstreams[0] != upstreamOS {
			ctrld.Log(ctx, p.Debug(), "Trying OS resolver as fallback")
			if answer := p.tryOSResolver(ctx, req); answer != nil {
				ctrld.Log(ctx, p.Debug(), "OS resolver fallback successful")
				return answer
			}
		}
	}

	ctrld.Log(ctx, p.Debug(), "Returning server failure response")
	p.countFailedClientQuery(upstreams)
	answer := new(dns.Msg)
	answer.SetRcode(req.msg, dns.RcodeServerFailure)
	return &proxyResponse{answer: answer}
}

// countFailedClientQuery grades one client query that ended without an answer.
// A query that only Internal Domain resolvers serve stays out of the grade: an
// endpoint away from the organization network never reaches them, and that is
// not an outage of the query path.
func (p *prog) countFailedClientQuery(upstreams []string) {
	if internalDomainExplicitUpstreams(upstreams) {
		return
	}
	p.health.countFailedQuery()
}

// shouldContinueWithNextUpstream determines whether processing should continue with the next upstream based on response conditions.
func (p *prog) shouldContinueWithNextUpstream(ctx context.Context, req *proxyRequest, answer *dns.Msg, upstream string, lastUpstream bool) bool {
	if answer.Rcode == dns.RcodeSuccess {
		ctrld.Log(ctx, p.Debug(), "Successful response, not continuing to next upstream")
		return false
	}

	// We are doing LAN/PTR lookup using private resolver, so always process the next one.
	// Except for the last, we want to send a response instead of saying all upstream failed.
	if req.isLanOrPtrQuery && !lastUpstream {
		ctrld.Log(ctx, p.Debug(), "No response for LAN/PTR query from %s, process to next upstream", upstream)
		return true
	}

	if len(req.upstreamConfigs) > 1 && slices.Contains(req.failoverRcodes, answer.Rcode) {
		ctrld.Log(ctx, p.Debug(), "Failover rcode matched, process to next upstream")
		return true
	}

	ctrld.Log(ctx, p.Debug(), "Not continuing to next upstream")
	return false
}

// prepareSuccessResponse prepares a successful DNS response for a given request, logs it, and updates the cache if applicable.
func (p *prog) prepareSuccessResponse(ctx context.Context, req *proxyRequest, answer *dns.Msg, upstream string, upstreamConfig *ctrld.UpstreamConfig) *proxyResponse {
	ctrld.Log(ctx, p.Debug(), "Preparing success response")

	answer.Compress = true

	// The echo must reach Client Connector on every poll, including reconnects.
	cacheable := p.cache != nil && req.msg.Question[0].Qtype != dns.TypePTR && !req.ufr.zpaDNSEcho
	if cacheable {
		ctrld.Log(ctx, p.Debug(), "Updating cache with successful response")
		p.updateCache(ctx, req, answer, upstream)
	}

	// Apply DNS64 only after policy processing, and resolve the companion A
	// question through the same upstream that produced the approved answer.
	var synthesizedPrefix netip.Prefix
	answer, synthesizedPrefix = p.maybeDNS64(ctx, req.msg, answer, func(aReq *dns.Msg) *dns.Msg {
		key := dnscache.NewKey(aReq, upstream)
		if cacheable {
			if cachedValue := p.cache.Get(key); cachedValue != nil {
				now := time.Now()
				if cachedValue.Expire.After(now) {
					cached := cachedValue.Msg.Copy()
					ctrld.SetCacheReply(cached, aReq, cached.Rcode)
					setCachedAnswerTTL(cached, now, cachedValue.Expire)
					return cached
				}
			}
		}

		aProxyReq := *req
		aProxyReq.msg = aReq
		resolved := p.queryUpstream(ctx, &aProxyReq, upstream, upstreamConfig)
		if cacheable && resolved != nil && sameQuestion(aReq, resolved) {
			p.addCachedResponse(key, resolved)
		}
		return resolved
	})
	if cacheable && synthesizedPrefix.IsValid() {
		p.addCachedResponse(dns64CacheKey(req.msg, upstream, synthesizedPrefix), answer)
		ctrld.Log(ctx, p.Debug(), "DNS64: added cached response variant")
	}

	hostname := ""
	if req.ci != nil {
		hostname = req.ci.Hostname
	}

	ctrld.Log(ctx, p.Info(), "REPLY: %s -> %s (%s): %s",
		upstream, req.ufr.srcAddr, hostname, dns.RcodeToString[answer.Rcode])

	return &proxyResponse{
		answer:   answer,
		upstream: upstreamConfig.Endpoint,
	}
}

// tryUpstreams attempts to proxy a DNS request through the provided upstreams and their configurations sequentially.
// It returns a successful proxyResponse if any upstream processes the request successfully, or nil otherwise.
// The function supports "serve stale" for cache by utilizing cached responses when upstreams fail.
func (p *prog) tryUpstreams(ctx context.Context, req *proxyRequest, upstreams []string, upstreamConfigs []*ctrld.UpstreamConfig) *proxyResponse {
	serveStaleCache := p.cache != nil && p.cfg.Service.CacheServeStale && !req.ufr.zpaDNSEcho
	req.upstreamConfigs = upstreamConfigs

	ctrld.Log(ctx, p.Debug(), "Trying %d upstreams", len(upstreamConfigs))

	for n, upstreamConfig := range upstreamConfigs {
		last := n == len(upstreamConfigs)-1
		ctrld.Log(ctx, p.Debug(), "Processing upstream %d/%d: %s", n+1, len(upstreamConfigs), upstreams[n])

		if res := p.processUpstream(ctx, req, upstreams[n], upstreamConfig, serveStaleCache, last); res != nil {
			ctrld.Log(ctx, p.Debug(), "Upstream %s succeeded", upstreams[n])
			return res
		}

		ctrld.Log(ctx, p.Debug(), "Upstream %s failed", upstreams[n])
	}

	ctrld.Log(ctx, p.Debug(), "All upstreams failed")
	return nil
}

// processUpstream proxies a DNS query to a given upstream server and processes the response based on the provided configuration.
// It supports serving stale cache when upstream queries fail, and checks if processing should continue to another upstream.
// Returns a proxyResponse on success or nil if the upstream query fails or processing conditions are not met.
func (p *prog) processUpstream(ctx context.Context, req *proxyRequest, upstream string, upstreamConfig *ctrld.UpstreamConfig, serveStaleCache, lastUpstream bool) *proxyResponse {
	if upstreamConfig == nil {
		ctrld.Log(ctx, p.Debug(), "Upstream config is nil, skipping")
		return nil
	}
	if p.isLoop(upstreamConfig) {
		logger := p.Debug().
			Str("upstream", upstreamConfig.String()).
			Str("query", req.msg.Question[0].Name).
			Bool("is_lan_query", req.isLanOrPtrQuery)
		ctrld.Log(ctx, logger, "DNS loop detected")
		return nil
	}

	ctrld.Log(ctx, p.Debug(), "Querying upstream: %s", upstream)
	answer := p.queryUpstream(ctx, req, upstream, upstreamConfig)
	if answer == nil {
		ctrld.Log(ctx, p.Debug(), "Upstream query failed")
		if serveStaleCache && req.staleAnswer != nil {
			ctrld.Log(ctx, p.Debug(), "Serving stale response due to upstream failure")
			return p.serveStaleResponse(ctx, req.staleAnswer)
		}
		return nil
	}

	ctrld.Log(ctx, p.Debug(), "Upstream query successful")
	// Reject an answer whose question does not match the request before it
	// can be served or cached. A mismatched question means the upstream
	// answered a different name/type than asked; caching it would poison
	// the shared cache with wrong-domain records for the requested name.
	// See github.com/Control-D-Inc/ctrld/issues/322.
	if !sameQuestion(req.msg, answer) {
		ctrld.Log(ctx, p.Debug(),
			"Discarding answer from %s: question mismatch (asked %q, got %q)",
			upstream, questionString(req.msg), questionString(answer))
		return nil
	}
	if p.shouldContinueWithNextUpstream(ctx, req, answer, upstream, lastUpstream) {
		return nil
	}
	return p.prepareSuccessResponse(ctx, req, answer, upstream, upstreamConfig)
}

// queryUpstream sends a DNS query to a specified upstream using its configuration and handles errors and retries.
func (p *prog) queryUpstream(ctx context.Context, req *proxyRequest, upstream string, upstreamConfig *ctrld.UpstreamConfig) *dns.Msg {
	if isInternalDomainUpstream(upstream) {
		ctx = ctrld.PrivateResolverCtx(ctx)
	}
	if upstreamConfig.UpstreamSendClientInfo() && req.ci != nil {
		ctrld.Log(ctx, p.Debug(), "Adding client info to upstream query")
		ctx = context.WithValue(ctx, ctrld.ClientInfoCtxKey{}, req.ci)
	}

	ctrld.Log(ctx, p.Debug(), "Sending query to %s: %s", upstream, upstreamConfig.Name)
	dnsResolver, err := newResolverFn(ctx, upstreamConfig)
	if err != nil {
		ctrld.Log(ctx, p.Error().Err(err), "Failed to create resolver")
		return nil
	}

	resolveCtx, cancel := upstreamConfig.Context(ctx)
	defer cancel()

	ctrld.Log(ctx, p.Debug(), "Resolving query with upstream")
	answer, err := dnsResolver.Resolve(resolveCtx, req.msg)
	if answer != nil {
		ctrld.Log(ctx, p.Debug(), "Upstream resolution successful")
		// reset is not used here, because it also stops the failure count for
		// one second, and the failures already in flight must still count.
		p.um.noteSuccess(upstream)
		return answer
	}

	// A failing minute must not push the state events out of the retained log,
	// so the sampler sets the level of these lines.
	// Transport errors contain the organization's private resolver address.
	// Keep that detail at debug while retaining a visible failure classification.
	if isInternalDomainUpstream(upstream) {
		ctrld.Log(ctx, p.Debug().Err(err), "Failed to resolve query")
		ctrld.Log(ctx, p.querySampler.event(sampleClassInternalDomain, upstream).Str("failure", internalDomainFailureReason(err)),
			"Failed to resolve query using an Internal Domain resolver")
	} else {
		ctrld.Log(ctx, p.querySampler.event(sampleClassResolveFailed, upstream).Err(err), "Failed to resolve query")
	}
	// Increasing the failure count when there is no answer regardless of what kind of error we get
	p.um.increaseFailureCount(upstream)
	if err != nil {
		// For timeout error (i.e: context deadline exceed), force re-bootstrapping.
		var e net.Error
		if errors.As(err, &e) && e.Timeout() {
			ctrld.Log(ctx, p.Debug(), "Timeout error, forcing re-bootstrapping")
			upstreamConfig.ReBootstrap(ctx)
		}
		// For network error, turn ipv6 off if enabled.
		if ctrld.HasIPv6(ctx) && (errUrlNetworkError(err) || errNetworkError(err)) {
			ctrld.Log(ctx, p.Debug(), "Network error, disabling IPv6")
			ctrld.DisableIPv6(ctx)
		}
	}
	return nil
}

// triggerRecovery attempts to initiate a recovery process if no healthy upstreams are detected.
// If "isOSFailure" is true, the recovery will account for an operating system failure.
// Logs are generated to indicate whether recovery is triggered or already in progress.
func (p *prog) triggerRecovery(isOSFailure bool) {
	p.recoveryCancelMu.Lock()
	defer p.recoveryCancelMu.Unlock()

	if p.recoveryCancel == nil {
		var reason RecoveryReason
		if isOSFailure {
			reason = RecoveryReasonOSFailure
		} else {
			reason = RecoveryReasonRegularFailure
		}
		p.Debug().Msgf("Selected upstreams unavailable; requesting recovery with reason: %v", reason)
		go queryRecoveryFn(p, reason)
	} else {
		p.Debug().Msg("Recovery already in progress; skipping duplicate trigger from down detection")
	}
}

// tryOSResolver attempts to query the OS resolver as a fallback mechanism when other upstreams fail.
// Logs success or failure of the query attempt and returns a proxyResponse or nil based on query result.
func (p *prog) tryOSResolver(ctx context.Context, req *proxyRequest) *proxyResponse {
	ctrld.Log(ctx, p.Debug(), "Attempting query to OS resolver as a retry catch all")
	// An Active Directory name that falls back from its explicit
	// upstream still must not reach Control D's public resolver.
	if len(req.msg.Question) > 0 && inActiveDirectoryDomain(req.msg.Question[0].Name) {
		ctx = ctrld.LanQueryCtx(ctx)
		ctrld.Log(ctx, p.Debug(), "Active Directory domain lookup, skipping public DNS")
	}
	answer := p.queryUpstream(ctx, req, upstreamOS, osUpstreamConfig)
	if answer != nil {
		ctrld.Log(ctx, p.Debug(), "OS resolver retry query successful")
		return &proxyResponse{answer: answer, upstream: osUpstreamConfig.Endpoint}
	}
	ctrld.Log(ctx, p.Debug(), "OS resolver retry query failed")
	return nil
}

// upstreamsAndUpstreamConfigForPtr returns the updated upstreams and upstreamConfigs for a private PTR lookup scenario.
func (p *prog) upstreamsAndUpstreamConfigForPtr(upstreams []string, upstreamConfigs []*ctrld.UpstreamConfig) ([]string, []*ctrld.UpstreamConfig) {
	if len(p.localUpstreams) > 0 {
		tmp := make([]string, 0, len(p.localUpstreams)+len(upstreams))
		tmp = append(tmp, p.localUpstreams...)
		tmp = append(tmp, upstreams...)
		return tmp, p.upstreamConfigsFromUpstreamNumbers(tmp)
	}
	return append([]string{upstreamOS}, upstreams...), append([]*ctrld.UpstreamConfig{privateUpstreamConfig}, upstreamConfigs...)
}

// upstreamConfigsFromUpstreamNumbers converts a list of upstream names into their corresponding UpstreamConfig objects.
func (p *prog) upstreamConfigsFromUpstreamNumbers(upstreams []string) []*ctrld.UpstreamConfig {
	upstreamConfigs := make([]*ctrld.UpstreamConfig, 0, len(upstreams))
	for _, upstream := range upstreams {
		upstreamNum := strings.TrimPrefix(upstream, upstreamPrefix)
		upstreamConfigs = append(upstreamConfigs, p.cfg.Upstream[upstreamNum])
	}
	return upstreamConfigs
}

// canonicalName returns canonical name from FQDN with "." trimmed.
func canonicalName(fqdn string) string {
	q := strings.TrimSpace(fqdn)
	q = strings.TrimSuffix(q, ".")
	// https://datatracker.ietf.org/doc/html/rfc4343
	q = strings.ToLower(q)

	return q
}

// wildcardMatches reports whether string str matches the wildcard pattern in case-insensitive manner.
func wildcardMatches(wildcard, str string) bool {
	// Wildcard match.
	wildCardParts := strings.Split(strings.ToLower(wildcard), "*")
	if len(wildCardParts) != 2 {
		return false
	}

	str = strings.ToLower(str)
	switch {
	case len(wildCardParts[0]) > 0 && len(wildCardParts[1]) > 0:
		// Domain must match both prefix and suffix.
		return strings.HasPrefix(str, wildCardParts[0]) && strings.HasSuffix(str, wildCardParts[1])

	case len(wildCardParts[1]) > 0:
		// Only suffix must match.
		return strings.HasSuffix(str, wildCardParts[1])

	case len(wildCardParts[0]) > 0:
		// Only prefix must match.
		return strings.HasPrefix(str, wildCardParts[0])
	}

	return false
}

// fmtRemoteToLocal formats a remote address to indicate its mapping to a local listener using listener number and hostname.
func fmtRemoteToLocal(listenerNum, hostname, remote string) string {
	return fmt.Sprintf("%s (%s) -> listener.%s", remote, hostname, listenerNum)
}

// requestID generates a random 6-character hexadecimal string to uniquely identify a request. It panics on error.
func requestID() string {
	b := make([]byte, 3) // 6 chars
	if _, err := rand.Read(b); err != nil {
		panic(err)
	}
	return hex.EncodeToString(b)
}

// sameQuestion reports whether the upstream answer echoes the request's
// question. A well-behaved resolver always copies the question section from
// the query (RFC 1035 section 4.1.2); names are compared case-insensitively
// because DNS names are case-insensitive. A mismatch means the upstream
// answered a different name/type than asked - malformed or malicious - and the
// answer must not be served or cached, or it would poison the shared cache with
// wrong-domain records. See github.com/Control-D-Inc/ctrld/issues/322.
func sameQuestion(req, answer *dns.Msg) bool {
	if req == nil || answer == nil {
		return false
	}
	if len(req.Question) == 0 || len(answer.Question) == 0 {
		return false
	}
	rq, aq := req.Question[0], answer.Question[0]
	return rq.Qtype == aq.Qtype && rq.Qclass == aq.Qclass && strings.EqualFold(rq.Name, aq.Name)
}

// questionString renders a message's first question as "name/type" for logging.
func questionString(msg *dns.Msg) string {
	if msg == nil || len(msg.Question) == 0 {
		return "<none>"
	}
	q := msg.Question[0]
	return q.Name + "/" + dns.TypeToString[q.Qtype]
}

// setCachedAnswerTTL updates the TTL of each DNS record in the provided message based on the current and expiration times.
func setCachedAnswerTTL(answer *dns.Msg, now, expiredTime time.Time) {
	ttlSecs := expiredTime.Sub(now).Seconds()
	if ttlSecs < 0 {
		return
	}

	ttl := uint32(ttlSecs)
	for _, rr := range answer.Answer {
		rr.Header().Ttl = ttl
	}
	for _, rr := range answer.Ns {
		rr.Header().Ttl = ttl
	}
	for _, rr := range answer.Extra {
		if rr.Header().Rrtype != dns.TypeOPT {
			rr.Header().Ttl = ttl
		}
	}
}

// ttlFromMsg extracts and returns the TTL value from the first record in the Answer or Ns sections of a DNS message.
// If no records exist in either section, the function returns 0.
func ttlFromMsg(msg *dns.Msg) uint32 {
	for _, rr := range msg.Answer {
		return rr.Header().Ttl
	}
	for _, rr := range msg.Ns {
		return rr.Header().Ttl
	}
	return 0
}

// needLocalIPv6Listener checks if a local IPv6 listener is required on Windows by verifying IPv6 support and the OS type.
func needLocalIPv6Listener() bool {
	// On Windows, there's no easy way for disabling/removing IPv6 DNS resolver, so we check whether we can
	// listen on ::1, then spawn a listener for receiving DNS requests.
	return ctrldnet.SupportsIPv6ListenLocal() && runtime.GOOS == "windows"
}

// ipAndMacFromMsg extracts IP and MAC information included in a DNS message, if any.
func ipAndMacFromMsg(msg *dns.Msg) (string, string) {
	ip, mac := "", ""
	if opt := msg.IsEdns0(); opt != nil {
		for _, s := range opt.Option {
			switch e := s.(type) {
			case *dns.EDNS0_LOCAL:
				if e.Code == EDNS0_OPTION_MAC {
					mac = net.HardwareAddr(e.Data).String()
				}
			case *dns.EDNS0_SUBNET:
				if len(e.Address) > 0 && !e.Address.IsLoopback() {
					ip = e.Address.String()
				}
			}
		}
	}
	return ip, mac
}

// stripClientSubnet removes EDNS0_SUBNET from DNS message if the IP is RFC1918 or loopback address,
// passing them to upstream is pointless, these cannot be used by anything on the WAN.
func stripClientSubnet(msg *dns.Msg) {
	if opt := msg.IsEdns0(); opt != nil {
		opts := make([]dns.EDNS0, 0, len(opt.Option))
		for _, s := range opt.Option {
			if e, ok := s.(*dns.EDNS0_SUBNET); ok && (e.Address.IsPrivate() || e.Address.IsLoopback()) {
				continue
			}
			opts = append(opts, s)
		}
		if len(opts) != len(opt.Option) {
			opt.Option = opts
		}
	}
}

func spoofRemoteAddr(addr net.Addr, ci *ctrld.ClientInfo) net.Addr {
	if ci != nil && ci.IP != "" {
		switch addr := addr.(type) {
		case *net.UDPAddr:
			udpAddr := &net.UDPAddr{
				IP:   net.ParseIP(ci.IP),
				Port: addr.Port,
				Zone: addr.Zone,
			}
			return udpAddr
		case *net.TCPAddr:
			udpAddr := &net.TCPAddr{
				IP:   net.ParseIP(ci.IP),
				Port: addr.Port,
				Zone: addr.Zone,
			}
			return udpAddr
		}
	}
	return addr
}

// runDNSServer starts a DNS server for given address and network,
// with the given handler. It ensures the server has started listening.
// Any error will be reported to the caller via returned channel.
//
// It's the caller responsibility to call Shutdown to close the server.
func runDNSServer(addr, network string, handler dns.Handler) (*dns.Server, <-chan error) {
	mainLog.Load().Debug().Str("address", addr).Str("network", network).Msg("Starting DNS server")

	s := &dns.Server{
		Addr:    addr,
		Net:     network,
		Handler: handler,
	}

	startedCh := make(chan struct{})
	s.NotifyStartedFunc = func() { sync.OnceFunc(func() { close(startedCh) })() }

	errCh := make(chan error, 1)
	go func() {
		defer close(errCh)
		if err := s.ListenAndServe(); err != nil {
			s.NotifyStartedFunc()
			mainLog.Load().Error().Err(err).Msgf("Could not listen and serve on: %s", s.Addr)
			errCh <- err
		}
	}()
	<-startedCh
	mainLog.Load().Debug().Str("address", addr).Str("network", network).Msg("DNS server started successfully")
	return s, errCh
}

func (p *prog) getClientInfo(remoteIP string, msg *dns.Msg) *ctrld.ClientInfo {
	ci := &ctrld.ClientInfo{}
	if p.appCallback != nil {
		ci.IP = p.appCallback.LanIp()
		ci.Mac = p.appCallback.MacAddress()
		ci.Hostname = p.appCallback.HostName()
		ci.Self = true
		return ci
	}
	ci.IP, ci.Mac = ipAndMacFromMsg(msg)
	switch {
	case ci.IP != "" && ci.Mac != "":
		// Nothing to do.
	case ci.IP == "" && ci.Mac != "":
		// Have MAC, no IP.
		ci.IP = p.ciTable.LookupIP(ci.Mac)
	case ci.IP == "" && ci.Mac == "":
		// Have nothing, use remote IP then lookup MAC.
		ci.IP = remoteIP
		fallthrough
	case ci.IP != "" && ci.Mac == "":
		// Have IP, no MAC.
		ci.Mac = p.ciTable.LookupMac(ci.IP)
	}

	// If MAC is still empty here, that mean the requests are made from virtual interface,
	// like VPN/Wireguard clients, so we use ci.IP as hostname to distinguish those clients.
	if ci.Mac == "" {
		if hostname := p.ciTable.LookupHostname(ci.IP, ""); hostname != "" {
			ci.Hostname = hostname
		} else {
			// Only use IP as hostname for IPv4 clients.
			// For Android devices, when it joins the network, it uses ctrld to resolve
			// its private DNS once and never reaches ctrld again. For each time, it uses
			// a different IPv6 address, which causes hundreds/thousands different client
			// IDs created for the same device, which is pointless.
			//
			// TODO(cuonglm): investigate whether this can be a false positive for other clients?
			if !ctrldnet.IsIPv6(ci.IP) {
				ci.Hostname = ci.IP
				p.ciTable.StoreVPNClient(ci)
			}
		}
	} else {
		ci.Hostname = p.ciTable.LookupHostname(ci.IP, ci.Mac)
	}

	if ci.IP == "" {
		p.Debug().Msgf("client info entry with empty IP address: %v", ci)
	} else {
		ci.Self = p.queryFromSelf(ci.IP)
	}

	// In DNS intercept mode, ALL queries are from the local machine — pf/WFP
	// intercepts outbound DNS and redirects to ctrld. The source IP may be a
	// virtual interface (Tailscale, VPN) that has no ARP/MAC entry, causing
	// missing x-cd-mac, x-cd-host, and x-cd-os headers. Force Self=true and
	// populate from the primary physical interface info.
	if dnsIntercept && !ci.Self {
		ci.Self = true
	}

	// If this is a query from self, but ci.IP is not loopback IP,
	// try using hostname mapping for lookback IP if presents.
	if ci.Self {
		if name := p.ciTable.LocalHostname(); name != "" {
			ci.Hostname = name
		}
		// If MAC is still empty (e.g., query arrived via virtual interface IP
		// like Tailscale), fall back to the loopback MAC mapping which addSelf()
		// populates from the primary physical interface.
		if ci.Mac == "" {
			if mac := p.ciTable.LookupMac("127.0.0.1"); mac != "" {
				ci.Mac = mac
			}
		}
	}
	p.spoofLoopbackIpInClientInfo(ci)
	return ci
}

// spoofLoopbackIpInClientInfo replaces loopback IPs in client info.
//
// - Preference IPv4.
// - Preference RFC1918.
func (p *prog) spoofLoopbackIpInClientInfo(ci *ctrld.ClientInfo) {
	if ip := net.ParseIP(ci.IP); ip == nil || !ip.IsLoopback() {
		return
	}
	if ip := p.ciTable.LookupRFC1918IPv4(ci.Mac); ip != "" {
		ci.IP = ip
	}
}

// doSelfUninstall performs self-uninstall if these condition met:
//
// - There is only 1 ControlD upstream in-use.
// - Number of refused queries seen so far equals to selfUninstallMaxQueries.
// - The cdUID is deleted.
func (p *prog) doSelfUninstall(pr *proxyResponse) {
	answer := pr.answer
	if pr.refused || !p.canSelfUninstall.Load() || answer == nil || answer.Rcode != dns.RcodeRefused {
		return
	}

	p.selfUninstallMu.Lock()
	defer p.selfUninstallMu.Unlock()
	if p.checkingSelfUninstall {
		return
	}

	logger := p.logger.Load().With().Str("mode", "self-uninstall")
	if p.refusedQueryCount > selfUninstallMaxQueries {
		p.checkingSelfUninstall = true
		loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load())
		req := &controld.ResolverConfigRequest{
			RawUID:   cdUID,
			Version:  appVersion,
			Metadata: ctrld.SystemMetadata(loggerCtx),
		}
		_, err := controld.FetchResolverConfig(loggerCtx, req, cdDev)
		logger.Debug().Msg("Maximum number of refused queries reached, checking device status")
		selfUninstallCheck(err, p, logger)

		if err != nil {
			logger.Warn().Err(err).Msg("Could not fetch resolver config")
		}
		// Cool-of period to prevent abusing the API.
		go p.selfUninstallCoolOfPeriod()
		return
	}
	p.refusedQueryCount++
}

// selfUninstallCoolOfPeriod waits for 30 minutes before
// calling API again for checking ControlD device status.
func (p *prog) selfUninstallCoolOfPeriod() {
	t := time.NewTimer(time.Minute * 30)
	defer t.Stop()
	<-t.C
	p.selfUninstallMu.Lock()
	p.checkingSelfUninstall = false
	p.refusedQueryCount = 0
	p.selfUninstallMu.Unlock()
}

// forceFetchingAPI sends signal to force syncing API config if run in cd mode,
// and the domain == "cdUID.verify.controld.com"
func (p *prog) forceFetchingAPI(domain string) {
	if !p.beginNetworkActivity() {
		return
	}
	defer p.netMonitorWG.Done()
	if cdUID == "" {
		return
	}
	resolverID, parent, _ := strings.Cut(domain, ".")
	if resolverID != cdUID {
		return
	}
	switch {
	case cdDev && parent == "verify.controld.dev":
		// match ControlD dev
	case parent == "verify.controld.com":
		// match ControlD
	default:
		return
	}
	_ = p.apiForceReloadGroup.DoChan("force_sync_api", func() (interface{}, error) {
		if !p.beginNetworkActivity() {
			return nil, nil
		}
		defer p.netMonitorWG.Done()
		shutdown := p.networkActivityDone()
		select {
		case <-shutdown:
			return nil, nil
		case p.apiForceReloadCh <- struct{}{}:
		case <-p.stopCh:
			return nil, nil
		case <-p.runAbortCh:
			return nil, nil
		}
		// Wait here to prevent abusing API if we are flooded.
		p.mu.Lock()
		wait := timeDurationOrDefault(p.cfg.Service.ForceRefetchWaitTime, 30) * time.Second
		p.mu.Unlock()
		timer := time.NewTimer(wait)
		defer timer.Stop()
		select {
		case <-timer.C:
		case <-shutdown:
		case <-p.stopCh:
		case <-p.runAbortCh:
		}
		return nil, nil
	})
}

// timeDurationOrDefault returns time duration value from n if not nil.
// Otherwise, it returns time duration value defaultN.
func timeDurationOrDefault(n *int, defaultN int) time.Duration {
	if n != nil && *n > 0 {
		return time.Duration(*n)
	}
	return time.Duration(defaultN)
}

// queryFromSelf reports whether the input IP is from device running ctrld.
func (p *prog) queryFromSelf(ip string) bool {
	if val, ok := p.queryFromSelfMap.Load(ip); ok {
		return val.(bool)
	}
	netIP, err := netip.ParseAddr(ip)
	if err != nil {
		p.Debug().Err(err).Msgf("could not parse IP: %q", ip)
		return false
	}

	regularIPs, loopbackIPs, err := netmon.LocalAddresses()
	if err != nil {
		p.Warn().Err(err).Msg("Could not get local addresses")
		return false
	}
	for _, localIP := range slices.Concat(regularIPs, loopbackIPs) {
		if localIP.Compare(netIP) == 0 {
			p.queryFromSelfMap.Store(ip, true)
			return true
		}
	}
	p.queryFromSelfMap.Store(ip, false)
	return false
}

// needRFC1918Listeners reports whether ctrld need to spawn listener for RFC 1918 addresses.
// This is helpful for non-desktop platforms to receive queries from LAN clients.
func needRFC1918Listeners(lc *ctrld.ListenerConfig) bool {
	return rfc1918 && lc.IP == "127.0.0.1" && lc.Port == 53
}

// ipFromARPA parses a FQDN arpa domain and return the IP address if valid.
func ipFromARPA(arpa string) net.IP {
	if arpa, ok := strings.CutSuffix(arpa, ".in-addr.arpa."); ok {
		if ptrIP := net.ParseIP(arpa); ptrIP != nil {
			return net.IP{ptrIP[15], ptrIP[14], ptrIP[13], ptrIP[12]}
		}
	}
	if arpa, ok := strings.CutSuffix(arpa, ".ip6.arpa."); ok {
		l := net.IPv6len * 2
		base := 16
		ip := make(net.IP, net.IPv6len)
		for i := 0; i < l && arpa != ""; i++ {
			idx := strings.LastIndexByte(arpa, '.')
			off := idx + 1
			if idx == -1 {
				idx = 0
				off = 0
			} else if idx == len(arpa)-1 {
				return nil
			}
			n, err := strconv.ParseUint(arpa[off:], base, 8)
			if err != nil {
				return nil
			}
			b := byte(n)
			ii := i / 2
			if i&1 == 1 {
				b |= ip[ii] << 4
			}
			ip[ii] = b
			arpa = arpa[:idx]
		}
		return ip
	}
	return nil
}

// isPrivatePtrLookup reports whether DNS message is an PTR query for LAN/CGNAT network.
func isPrivatePtrLookup(m *dns.Msg) bool {
	if m == nil || len(m.Question) == 0 {
		return false
	}
	q := m.Question[0]
	if ip := ipFromARPA(q.Name); ip != nil {
		if addr, ok := netip.AddrFromSlice(ip); ok {
			return addr.IsPrivate() ||
				addr.IsLoopback() ||
				addr.IsLinkLocalUnicast() ||
				tsaddr.CGNATRange().Contains(addr) ||
				isServiceContinuityAddr(addr)
		}
	}
	return false
}

// isLanHostnameQuery reports whether DNS message is an A/AAAA query with LAN hostname.
func isLanHostnameQuery(m *dns.Msg) bool {
	if m == nil || len(m.Question) == 0 {
		return false
	}
	q := m.Question[0]
	switch q.Qtype {
	case dns.TypeA, dns.TypeAAAA:
	default:
		return false
	}
	return isLanHostname(q.Name)
}

// isSrvLanLookup reports whether DNS message is an SRV query of a LAN hostname.
func isSrvLanLookup(m *dns.Msg) bool {
	if m == nil || len(m.Question) == 0 {
		return false
	}
	q := m.Question[0]
	return q.Qtype == dns.TypeSRV && isLanHostname(q.Name)
}

// isLanHostname reports whether name is a LAN hostname.
func isLanHostname(name string) bool {
	name = strings.TrimSuffix(name, ".")
	return !strings.Contains(name, ".") ||
		strings.HasSuffix(name, ".domain") ||
		strings.HasSuffix(name, ".lan") ||
		strings.HasSuffix(name, ".local")
}

// ipv4ServiceContinuityPrefix is the RFC 7335 IPv4 Service Continuity Prefix
// (192.0.0.0/29), used by the CLAT in 464XLAT/DS-Lite transition setups. On such
// networks (common on IPv6-only cellular carriers and iPhone hotspots) the local
// machine's DNS queries reach ctrld with a source in this range (e.g. 192.0.0.2),
// so they must be treated as local, not WAN. Go's netip.IsPrivate does not cover
// this range — the same reason the CGNAT range is special-cased below. See #552.
var ipv4ServiceContinuityPrefix = netip.MustParsePrefix("192.0.0.0/29")

// isServiceContinuityAddr reports whether ip is in the RFC 7335 IPv4 Service
// Continuity Prefix (464XLAT/DS-Lite CLAT).
func isServiceContinuityAddr(ip netip.Addr) bool {
	return ipv4ServiceContinuityPrefix.Contains(ip)
}

// isWanClient reports whether the input is a WAN address.
func isWanClient(na net.Addr) bool {
	var ip netip.Addr
	if ap, err := netip.ParseAddrPort(na.String()); err == nil {
		ip = ap.Addr()
	}
	return !ip.IsLoopback() &&
		!ip.IsPrivate() &&
		!ip.IsLinkLocalUnicast() &&
		!ip.IsLinkLocalMulticast() &&
		!tsaddr.CGNATRange().Contains(ip) &&
		!isServiceContinuityAddr(ip)
}

// resolveInternalDomainTestQuery resolves internal test domain query, returning the answer to the caller.
func resolveInternalDomainTestQuery(ctx context.Context, domain string, m *dns.Msg) *dns.Msg {
	logger := ctrld.LoggerFromCtx(ctx)
	ctrld.Log(ctx, logger.Debug(), "Internal domain test query")

	q := m.Question[0]
	answer := new(dns.Msg)
	rrStr := fmt.Sprintf("%s A %s", domain, net.IPv4zero)
	if q.Qtype == dns.TypeAAAA {
		rrStr = fmt.Sprintf("%s AAAA %s", domain, net.IPv6zero)
	}
	rr, err := dns.NewRR(rrStr)
	if err == nil {
		answer.Answer = append(answer.Answer, rr)
	}
	answer.SetReply(m)
	return answer
}

// FlushDNSCache flushes the DNS cache on macOS.
func FlushDNSCache() error {
	// if not macOS, return
	if runtime.GOOS != "darwin" {
		return nil
	}

	// Flush the DNS cache via mDNSResponder.
	// This is typically needed on modern macOS systems.
	if out, err := exec.Command("killall", "-HUP", "mDNSResponder").CombinedOutput(); err != nil {
		return fmt.Errorf("failed to flush mDNSResponder: %w, output: %s", err, string(out))
	}

	// Optionally, flush the directory services cache.
	if out, err := exec.Command("dscacheutil", "-flushcache").CombinedOutput(); err != nil {
		return fmt.Errorf("failed to flush dscacheutil: %w, output: %s", err, string(out))
	}

	return nil
}

// networkChangeCallback fences even callbacks already dispatched by netmon
// when Close begins. The monitor does not track those callback goroutines.
func (p *prog) networkChangeCallback(fn netmon.ChangeFunc) netmon.ChangeFunc {
	return func(delta *netmon.ChangeDelta) {
		if !p.beginNetworkActivity() {
			return
		}
		defer p.netMonitorWG.Done()
		fn(delta)
	}
}

// monitorNetworkChanges starts monitoring for network interface changes
func (p *prog) monitorNetworkChanges(ctx context.Context) error {
	mon, err := newNetworkChangeMonitorFn(func(format string, args ...any) {
		// Always fetch the latest logger (and inject the prefix)
		p.logger.Load().Printf("netmon: "+format, args...)
	})
	if err != nil {
		return fmt.Errorf("creating network monitor: %w", err)
	}

	mon.RegisterChangeCallback(p.networkChangeCallback(func(delta *netmon.ChangeDelta) {
		p.handleNetworkChange(ctx, delta, mon.IsMajorChangeFrom(delta.Old, delta.New))
	}))
	if !p.setNetMonitor(mon) {
		_ = mon.Close()
		p.Debug().Msg("Network monitor discarded, ctrld is stopping")
		return nil
	}

	mon.Start()
	p.logger.Load().Debug().Msg("Network monitor started")
	return nil
}

// handleNetworkChange is the network monitor callback, kept separate for synthetic deltas.
func (p *prog) handleNetworkChange(ctx context.Context, delta *netmon.ChangeDelta, isMajorChange bool) {
	// netmon v1.74.0 dispatches callbacks concurrently and caches only major
	// snapshots. Minor events refer to that cache through Old. The synthetic
	// delta.Major flag after a time jump does not change the cache.
	p.networkSourceMu.Lock()
	currentState := networkChangeCurrentStateFn(delta)
	current := networkSnapshotCurrent(delta, isMajorChange, currentState)
	if delta.TimeJumped {
		// A late callback still tells that the host woke, but the network of
		// the wake is the one that netmon holds now.
		wakeState := delta.New
		if !current {
			wakeState = currentState
		}
		p.noteHostWoke("netmon", 0, wakeState)
	}
	// Only a current callback becomes the baseline of the next diff. A late
	// callback that a newer snapshot replaced would hide the changes between
	// the callback before it and the callback after it.
	before := p.deltaBeforeState(delta, currentState, current, isMajorChange)
	changes := diffNetworkDelta(before, delta.New)
	if current && hasAddOrRemove(changes) {
		refreshInterfaceMeta()
	}
	// The noise test runs before any other work, because the point of the class
	// is that a storm starts no pfctl and no scutil process.
	if current && noiseDelta(before, delta.New, delta.TimeJumped, changes) {
		p.networkSourceMu.Unlock()
		p.noteNoiseDelta(changes)
		return
	}
	// A superseded snapshot never becomes a transition, so it takes no ID and
	// reports zero.
	var transitionID uint64
	changed := false
	changedInterface := ""
	sourceSnapshotAvailable := false
	before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6()
	after4, after6 := before4, before6
	// The newer callback of this epoch carries the same interface changes, so
	// this outcome reports at debug and stays out of the journal.
	outcome := transitionOutcomeSnapshotSuperseded
	defer func() {
		stateBefore, stateAfter := networkStateOrEmpty(before), networkStateOrEmpty(delta.New)
		networkTransitionEvent(outcome).Uint64("transition_id", transitionID).
			Bool("is_major_change", isMajorChange).Bool("changed", changed).
			Str("interface", changedInterface).
			Strs("changed_interfaces", changedInterfaceNames(changes)).
			Bool("time_jumped", delta.TimeJumped).
			Str("default_route_before", stateBefore.DefaultRouteInterface).
			Str("default_route_after", stateAfter.DefaultRouteInterface).
			Bool("have_v4_before", stateBefore.HaveV4).Bool("have_v4_after", stateAfter.HaveV4).
			Bool("have_v6_before", stateBefore.HaveV6).Bool("have_v6_after", stateAfter.HaveV6).
			Str("source_ipv4_before", before4.String()).Str("source_ipv6_before", before6.String()).
			Str("source_ipv4_after", after4.String()).
			Str("source_ipv6_after", after6.String()).
			Bool("source_snapshot_available", sourceSnapshotAvailable).Str("outcome", outcome).
			Msg(networkTransitionMessage)
	}()
	if !current {
		p.networkSourceMu.Unlock()
		return
	}
	p.flushNoiseSummary()
	outcome = transitionOutcomeIgnored
	transitionID = p.networkTransitionGen.Add(1)
	sourceState := delta.New
	// A late major callback and reordered minor callbacks share one cache
	// epoch. For these cases, source validity comes from the OS, not ordering.
	if delta.Monitor != nil && (!isMajorChange || (p.networkSourceEpoch == currentState && p.networkSourceState != currentState)) {
		fresh, err := readNetworkSourceStateFn()
		if err != nil || fresh == nil {
			if !p.networkSourceReadFailed {
				p.logger.Load().Warn().Uint64("transition_id", transitionID).
					Msg("Network source snapshot unavailable; source writes deferred")
			}
			p.networkSourceReadFailed = true
			sourceState = nil
		} else {
			snapshot := *fresh
			snapshot.DefaultRouteInterface = currentState.DefaultRouteInterface
			sourceState = &snapshot
			if p.networkSourceReadFailed {
				p.logger.Load().Warn().Uint64("transition_id", transitionID).Msg("Network source snapshot available again")
			}
			p.networkSourceReadFailed = false
		}
	}
	p.networkSourceState, p.networkSourceEpoch = sourceState, currentState
	sourceSnapshotAvailable = sourceState != nil
	validateDefaultLocalIPsFromDelta(ctrld.LoggerCtx(ctx, p.logger.Load()), sourceState, transitionID)
	after4, after6 = ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6()
	// The network of this callback belongs to the epoch that the lines above
	// store. A store after the unlock can overwrite the network of a newer
	// callback. The header render reads the hardware ports, so it runs after
	// the unlock and starts no process under the lock.
	p.lastNetworkState.Store(currentState)
	// HasIPv6 reads the flag that this callback stores, so one monitor serves
	// the whole process.
	ctrld.SetIPv6Available(currentState.HaveV6)
	p.networkSourceMu.Unlock()
	p.refreshLogHeader()
	describeInterfaceChanges(changes, interfaceMetaFor)
	logInterfaceChanges(transitionID, changes)

	p.handleDNS64NetworkChange(delta, isMajorChange)
	validIfaces := networkChangeValidInterfacesFn(ctrld.LoggerCtx(ctx, p.logger.Load()))

	activeInterfaceExists := false
	var changeIPs []netip.Prefix
	// Check each valid interface for changes
	for ifaceName := range validIfaces {
		oldIface, oldExists := delta.Old.Interface[ifaceName]
		newIface, newExists := delta.New.Interface[ifaceName]
		if !newExists {
			continue
		}

		oldIPs := delta.Old.InterfaceIPs[ifaceName]
		newIPs := delta.New.InterfaceIPs[ifaceName]

		// if a valid interface did not exist in old
		// check that its up and has usable IPs
		if !oldExists {
			// The interface is new (was not present in the old state).
			usableNewIPs := filterUsableIPs(newIPs)
			if newIface.IsUp() && len(usableNewIPs) > 0 {
				// The new interface is the active one, or the change ends as
				// no_active_interface and nothing reconciles.
				activeInterfaceExists = true
				changed = true
				changeIPs = usableNewIPs
				changedInterface = ifaceName
				p.logger.Load().Debug().
					Str("interface", ifaceName).
					Interface("new_ips", usableNewIPs).
					Msg("Interface newly appeared (was not present in old state)")
				break
			}
			continue
		}

		// Filter new IPs to only those that are usable.
		usableNewIPs := filterUsableIPs(newIPs)

		// Check if interface is up and has usable IPs.
		if newIface.IsUp() && len(usableNewIPs) > 0 {
			activeInterfaceExists = true
		}

		// Compare interface states and IPs (interfaceIPsEqual will itself filter the IPs).
		if !interfaceStatesEqual(&oldIface, &newIface) || !interfaceIPsEqual(oldIPs, newIPs) {
			if newIface.IsUp() && len(usableNewIPs) > 0 {
				changed = true
				changeIPs = usableNewIPs
				changedInterface = ifaceName
				p.logger.Load().Debug().
					Str("interface", ifaceName).
					Interface("old_ips", oldIPs).
					Interface("new_ips", usableNewIPs).
					Msg("Interface state or IPs changed")
				break
			}
		}
	}

	// if the default route changed, set changed to true
	if delta.New.DefaultRouteInterface != delta.Old.DefaultRouteInterface {
		changed = true
		p.logger.Load().Debug().Msgf("Default route changed from %s to %s", delta.Old.DefaultRouteInterface, delta.New.DefaultRouteInterface)
	}

	if !changed {
		p.logger.Load().Debug().Msg("Ignoring interface change - no valid interfaces affected")
		// Minor interface changes can still accompany pf/WFP or VPN DNS changes.
		// On macOS, bound the immediate full reconciliation so link-local-only
		// notification storms do not run pfctl/scutil work for every event.
		// Windows keeps the existing immediate behavior. Tunnel changes always
		// bypass the macOS limit, and delayed checks provide a trailing refresh.
		if dnsIntercept && p.dnsInterceptState != nil {
			networkChangeIgnoredInterceptFn(p, delta, time.Now())
		}
		return
	}

	if !activeInterfaceExists {
		outcome = "no_active_interface"
		p.logger.Load().Debug().Msg("No active interfaces found, skipping reinitialization")
		return
	}

	// An ignored event must not cancel this accepted transition. Only a newer
	// accepted transition can replace its source commit or debounced recovery.
	p.networkSourceMu.Lock()
	if p.networkAcceptedGen.Load() > transitionID {
		p.networkSourceMu.Unlock()
		outcome = "superseded"
		return
	}
	p.networkAcceptedGen.Store(transitionID)
	p.networkSourceMu.Unlock()

	p.logger.Load().Debug().Msg("Link state changed, re-bootstrapping")
	for _, uc := range p.cfg.Upstream {
		uc.ReBootstrap(ctrld.LoggerCtx(ctx, p.logger.Load()))
	}

	// Get IPs from default route interface in new state
	selfIP := networkChangeDefaultRouteIPFn(p)

	// Ensure that selfIP is an IPv4 address.
	// If defaultRouteIP mistakenly returns an IPv6 (such as a ULA), clear it
	if ip := net.ParseIP(selfIP); ip != nil && ip.To4() == nil {
		p.logger.Load().Debug().Msgf("defaultRouteIP returned a non-IPv4 address: %s, ignoring it", selfIP)
		selfIP = ""
	}
	// Route discovery can lag the delta; do not reintroduce an invalid source.
	if ip := net.ParseIP(selfIP); ip != nil && sourceInvalidReason(delta.New, ip) != "" {
		selfIP = ""
	}
	var ipv6 string

	if delta.New.DefaultRouteInterface != "" {
		p.logger.Load().Debug().Msgf("default route interface: %s, IPs: %v", delta.New.DefaultRouteInterface, delta.New.InterfaceIPs[delta.New.DefaultRouteInterface])
		for _, ip := range delta.New.InterfaceIPs[delta.New.DefaultRouteInterface] {
			if sourceInvalidReason(delta.New, net.ParseIP(ip.Addr().String())) != "" {
				continue
			}
			ipAddr, _ := netip.ParsePrefix(ip.String())
			addr := ipAddr.Addr()
			if selfIP == "" && addr.Is4() {
				p.logger.Load().Debug().Msgf("checking IP: %s", addr.String())
				if !addr.IsLoopback() && !addr.IsLinkLocalUnicast() {
					selfIP = addr.String()
				}
			}
			if addr.Is6() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() {
				ipv6 = addr.String()
			}
		}
	} else {
		// If no default route interface is set yet, use the changed IPs
		p.logger.Load().Debug().Msgf("no default route interface found, using changed IPs: %v", changeIPs)
		for _, ip := range changeIPs {
			ipAddr, _ := netip.ParsePrefix(ip.String())
			addr := ipAddr.Addr()
			if selfIP == "" && addr.Is4() {
				p.logger.Load().Debug().Msgf("checking IP: %s", addr.String())
				if !addr.IsLoopback() && !addr.IsLinkLocalUnicast() {
					selfIP = addr.String()
				}
			}
			if addr.Is6() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() {
				ipv6 = addr.String()
			}
		}
	}

	// An ignored event can change source validity without replacing recovery.
	// Recheck addresses against the newest state after blocking route lookup.
	p.networkSourceMu.Lock()
	if p.networkAcceptedGen.Load() != transitionID {
		p.networkSourceMu.Unlock()
		outcome = "superseded"
		return
	}
	latestState := p.sourceCommitState(delta)
	sourceSnapshotAvailable = latestState != nil
	validateDefaultLocalIPsFromDelta(ctrld.LoggerCtx(ctx, p.logger.Load()), latestState, transitionID)
	// Only keep candidate addresses that still exist on an up interface.
	if ip := net.ParseIP(selfIP); ip != nil && ip.To4() != nil && latestState != nil && sourceInvalidReason(latestState, ip) == "" {
		ctrld.SetDefaultLocalIPv4(ctrld.LoggerCtx(ctx, p.logger.Load()), ip)
		if !isMobile() && p.ciTable != nil {
			p.ciTable.SetSelfIP(selfIP)
		}
	}
	if ip := net.ParseIP(ipv6); ip != nil && latestState != nil && sourceInvalidReason(latestState, ip) == "" {
		ctrld.SetDefaultLocalIPv6(ctrld.LoggerCtx(ctx, p.logger.Load()), ip)
	}
	after4, after6 = ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6()
	p.networkSourceMu.Unlock()
	p.logger.Load().Debug().Msgf("Set default local IPv4: %s, IPv6: %s", selfIP, ipv6)

	outcome = "accepted"
	networkChangeReconcileFn(p, ctx, transitionID)
	p.logNetworkSnapshot("transition")
	if p.dnsConfig != nil {
		p.dnsConfig.noteActivity(networkEventsNowFn())
	}
}

// reconcileNetworkChange is the first DNS/PF mutation boundary after source updates.
func (p *prog) reconcileNetworkChange(ctx context.Context, transitionID uint64) {
	p.debounceRecovery(transitionID)

	// Drop stale firewall allowances from the previous network.
	p.firewallOnNetworkChange()

	// After network changes, verify our pf anchor is still active and
	// refresh VPN DNS state. Order matters: tunnel checks first (may rebuild
	// anchor), then VPN DNS refresh (updates exemptions in anchor), then
	// delayed re-checks for async VPN teardown.
	if dnsIntercept && p.dnsInterceptState != nil {
		if !p.pfStabilizing.Load() {
			p.ensurePFAnchorActive()
		}
		// Check tunnel interfaces unconditionally — it decides internally
		// whether to enter stabilization or rebuild immediately.
		p.checkTunnelInterfaceChanges()
		// Refresh VPN DNS routes — runs after tunnel checks so the anchor
		// rebuild includes current VPN DNS exemptions.
		if p.vpnDNS != nil {
			p.vpnDNS.Refresh(ctrld.LoggerCtx(ctx, p.logger.Load()), true)
		}
		// Preserve master VM/container forwarded-source reconciliation.
		p.reconcileForwardedSources()
		// Schedule delayed re-checks to catch async VPN teardown changes.
		p.scheduleDelayedRechecks()
	}
}

// handleDNSInterceptIgnoredNetworkChange runs the DNS-intercept work for a
// network delta that did not affect a usable interface. Keeping this path in a
// method lets tests exercise the callback wiring with synthetic deltas.
func (p *prog) handleDNSInterceptIgnoredNetworkChange(delta *netmon.ChangeDelta, now time.Time) {
	reconcileNow := false
	// Stabilization owns PF repair. Do not consume the next leading-edge slot
	// until an ignored delta can actually perform the corresponding PF check.
	if !p.pfStabilizing.Load() {
		reconcileNow = p.dnsInterceptIgnoredChangeReconcileDue(now)
		if reconcileNow {
			p.ensurePFAnchorActive()
		}
	}

	// Check tunnel interfaces unconditionally — it decides internally whether
	// to enter stabilization or rebuild immediately.
	tunnelChanged := p.checkTunnelInterfaceChanges()
	// Schedule delayed re-checks to catch async VPN teardown changes. These also
	// refresh the OS resolver and VPN DNS routes.
	p.scheduleDelayedRechecks()

	// Detect interface appearance/disappearance — hypervisors (Parallels,
	// VMware, VirtualBox) reload pf when creating/destroying virtual network
	// interfaces, which can corrupt pf's internal translation state. The rdr
	// rules survive in text form (watchdog says "intact") but stop evaluating.
	// Spawn an async monitor that probes pf interception with backoff and forces
	// a full pf reload if broken.
	if delta.Old != nil {
		changedAction := "removed"
		changedIface := interfaceOnlyIn(delta.Old.Interface, delta.New.Interface)
		if changedIface == "" {
			changedAction = "added"
			changedIface = interfaceOnlyIn(delta.New.Interface, delta.Old.Interface)
		}
		if changedIface != "" {
			journal(mainLog.Load().Info()).Str("interface", changedIface).
				Str("class", interfaceMetaFor(changedIface).Class).
				Str("action", changedAction).
				Msg("DNS intercept: interface appeared/disappeared — starting interception probe monitor")
			go p.pfInterceptMonitor()
			// A VM/container bridge appearing/disappearing changes the effective
			// forwarded-source set; an intact anchor/probe does not detect that.
			p.reconcileForwardedSources()
		}
	}

	// Refresh VPN DNS immediately for real tunnel changes even when the periodic
	// ignored-change reconciliation is currently rate-limited - but not while
	// stabilization owns pf. A refresh rebuilds the anchor, and these deltas arrive
	// exactly when a VPN is bringing its own ruleset up, which is the collision
	// stabilization is there to prevent. checkTunnelInterfaceChanges keeps the
	// observation pending, so the transition is retried rather than dropped.
	if p.vpnDNS != nil && (reconcileNow || tunnelChanged) && !p.pfStabilizing.Load() {
		p.vpnDNS.Refresh(ctrld.LoggerCtx(context.Background(), p.logger.Load()), true)
	}
}

// interfaceOnlyIn names one interface that have holds and lack does not. The
// loopback stays out, because it never appears and never goes away.
func interfaceOnlyIn(have, lack map[string]netmon.Interface) string {
	for name := range have {
		if name == "lo0" {
			continue
		}
		if _, exists := lack[name]; !exists {
			return name
		}
	}
	return ""
}

// interfaceStatesEqual compares two interface states
func interfaceStatesEqual(a, b *netmon.Interface) bool {
	if a == nil || b == nil {
		return a == b
	}
	return a.IsUp() == b.IsUp()
}

// filterUsableIPs is a helper that returns only "usable" IP prefixes,
// filtering out link-local, loopback, multicast, unspecified, broadcast, or CGNAT addresses.
func filterUsableIPs(prefixes []netip.Prefix) []netip.Prefix {
	var usable []netip.Prefix
	for _, p := range prefixes {
		addr := p.Addr()
		if addr.IsLinkLocalUnicast() ||
			addr.IsLoopback() ||
			addr.IsMulticast() ||
			addr.IsUnspecified() ||
			addr.IsLinkLocalMulticast() ||
			(addr.Is4() && addr.String() == "255.255.255.255") ||
			tsaddr.CGNATRange().Contains(addr) {
			continue
		}
		usable = append(usable, p)
	}
	return usable
}

// Modified interfaceIPsEqual compares only the usable (non-link local, non-loopback, etc.) IP addresses.
func interfaceIPsEqual(a, b []netip.Prefix) bool {
	aUsable := filterUsableIPs(a)
	bUsable := filterUsableIPs(b)
	if len(aUsable) != len(bUsable) {
		return false
	}

	aMap := make(map[string]bool)
	for _, ip := range aUsable {
		aMap[ip.String()] = true
	}
	for _, ip := range bUsable {
		if !aMap[ip.String()] {
			return false
		}
	}
	return true
}

var errOsHealthcheckSuppressed = errors.New("upstream os health check suppressed")

// upstreamFailureLog keeps one error line for each upstream of one recovery
// pass. The pass retries every two seconds until the upstream answers, and the
// first line already names the fault, so the failures that follow it go to
// debug.
type upstreamFailureLog struct{ reported bool }

// report logs one failed check of upstream.
func (l *upstreamFailureLog) report(p *prog, upstream string, uc *ctrld.UpstreamConfig, err error, duration time.Duration) {
	level := p.Error
	if l.reported {
		level = p.Debug
	}
	l.reported = true
	p.logUpstreamProbeFailure(upstream, uc, err, level, "Upstream check failed after %v", duration)
}

// checkUpstreamOnce sends a test query to the specified upstream.
// Returns nil if the upstream responds successfully.
func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig, failures *upstreamFailureLog) error {
	p.Debug().Msgf("Starting check for upstream: %s", upstream)

	resolver, err := ctrld.NewResolver(ctrld.LoggerCtx(context.Background(), p.logger.Load()), uc)
	if err != nil {
		p.logUpstreamProbeFailure(upstream, uc, err, p.Error, "Failed to create resolver")
		return err
	}

	timeout := 1000 * time.Millisecond
	if uc.Timeout > 0 {
		timeout = time.Millisecond * time.Duration(uc.Timeout)
	}
	p.Debug().Msgf("Timeout for upstream %s: %s", upstream, timeout)

	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	uc.ReBootstrap(ctrld.LoggerCtx(ctx, p.logger.Load()))
	p.Debug().Msgf("Rebootstrapping resolver for upstream: %s", upstream)

	start := time.Now()
	msg := uc.VerifyMsg()
	_, err = resolver.Resolve(ctx, msg)
	duration := time.Since(start)

	if err != nil {
		// Demote upstream.os check failures to debug while WFP loopback
		// protect is active: an external WFP block filter is interfering
		// with plain DNS so repeated failures here are expected. Other
		// upstreams keep error level so real outages stay visible.
		if upstream == upstreamOS && p.osHealthcheckSuppressed() {
			p.Debug().Err(err).Msgf("Upstream %s check failed after %v (WFP loopback protect active)", upstream, duration)
			return errOsHealthcheckSuppressed
		}
		// A no-route/network-unreachable failure means the endpoint's address
		// family is available locally but unroutable (e.g. an IPv6 DoH endpoint
		// while IPv6 is up but has no route). These repeat until the route
		// returns and are handled by bounded backoff in the recovery loop, so
		// keep them at debug to avoid sustained error-log spam.
		if ctrldnet.IsUnreachable(err) {
			p.Debug().Err(err).Msgf("Upstream %s check failed after %v (network unreachable)", upstream, duration)
			return err
		}
		failures.report(p, upstream, uc, err, duration)
		return err
	}
	p.Debug().Msgf("Upstream %s responded successfully in %v", upstream, duration)
	return nil
}

// handleRecovery orchestrates the recovery process by coordinating multiple smaller methods.
// It handles recovery cancellation logic, creates recovery context, prepares the system,
// waits for upstream recovery with timeout, and completes the recovery process.
// The method is designed to be called from a goroutine and handles different recovery reasons
// (network changes, regular failures, OS failures) with appropriate logic for each.
// recoveryDebounceWindow is the time to wait after the last network change
// before triggering handleRecovery. This coalesces rapid consecutive network
// changes (e.g., hotspot→LAN causing en1 drop + en0 pickup + en1 re-pickup)
// into a single recovery pass, avoiding the cancel-and-restart race that
// leaves DoH transports in a stale state.
const recoveryDebounceWindow = 500 * time.Millisecond

// recoveryResolverReason names this flow in the OS resolver events, so a
// reader tells a resolver read of the recovery from every other read.
const recoveryResolverReason = "recovery"

// debounceRecovery schedules a handleRecovery(NetworkChange) call after a debounce
// window. If called again before the window expires, the timer is reset so that
// recovery runs once with the final network state. All other state updates (IP,
// pf anchor, VPN DNS, tunnel checks) run immediately — only the recovery flow
// with its upstream probing and DHCP bypass logic is debounced.
func (p *prog) debounceRecovery(transitionID uint64) {
	p.networkSourceMu.Lock()
	defer p.networkSourceMu.Unlock()
	if p.networkAcceptedGen.Load() != transitionID {
		return
	}
	p.recoveryDebounceMu.Lock()
	defer p.recoveryDebounceMu.Unlock()
	if p.networkActivityClosed() {
		return
	}

	if p.recoveryDebounceTimer != nil {
		p.recoveryDebounceTimer.Stop()
		p.Debug().Msg("Recovery debounce: resetting timer (rapid network change)")
	}
	p.recoveryDebounceTimer = time.AfterFunc(recoveryDebounceWindow, func() {
		if !p.beginNetworkActivity() {
			return
		}
		defer p.netMonitorWG.Done()
		p.networkSourceMu.Lock()
		if p.networkAcceptedGen.Load() != transitionID {
			p.networkSourceMu.Unlock()
			return
		}
		p.recoveryDebounceMu.Lock()
		p.recoveryDebounceTimer = nil
		p.recoveryDebounceMu.Unlock()
		p.networkSourceMu.Unlock()
		handleRecoveryForTransitionFn(p, RecoveryReasonNetworkChange, transitionID)
	})
	p.Debug().Msg("Recovery debounce: scheduled (500ms window)")
}

// endRecovery closes one recovery pass. Support reads the end event and the
// snapshot that follows it to learn what the recovery left behind.
func (p *prog) endRecovery(diagnostic *recoveryDiagnostic, outcome, targetBefore string) {
	// A superseded pass changed nothing that it owns any more. Its successor
	// owns the loopback target and the bypass flag, so reading them here would
	// report the state of the successor.
	if outcome == recoveryOutcomeSuperseded {
		diagnostic.interceptTargetAction = interceptTargetActionUnchanged
		diagnostic.bypassActive = false
	} else {
		diagnostic.interceptTargetAction = interceptTargetAction(targetBefore, p.interceptTargetSnapshot())
		diagnostic.bypassActive = p.recoveryBypass.Load()
	}
	diagnostic.end(outcome)
	p.logNetworkSnapshot("recovery_end")
	// The resolver table of the host settles after the recovery, so the poll
	// stays fast until it does.
	if p.dnsConfig != nil {
		p.dnsConfig.noteActivity(networkEventsNowFn())
	}
}

// The actions that one recovery takes on the loopback DNS target.
const (
	interceptTargetActionUnchanged = "unchanged"
	interceptTargetActionRemoved   = "removed"
	interceptTargetActionSet       = "set"
)

// interceptTargetAction names what one recovery did to the loopback DNS
// target. The target value alone does not say whether this pass wrote it.
func interceptTargetAction(before, after string) string {
	switch {
	case before == after:
		return interceptTargetActionUnchanged
	case after == "":
		return interceptTargetActionRemoved
	default:
		return interceptTargetActionSet
	}
}

func (p *prog) handleRecovery(reason RecoveryReason) {
	p.handleRecoveryForTransition(reason, 0)
}

func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID uint64) {
	if !p.beginNetworkActivity() {
		return
	}
	defer p.netMonitorWG.Done()
	p.networkSourceMu.Lock()
	if transitionID != 0 && p.networkAcceptedGen.Load() != transitionID {
		p.networkSourceMu.Unlock()
		p.logger.Load().Debug().Uint64("transition_id", transitionID).Msg("Recovery skipped: transition superseded")
		return
	}
	// Admission and completion use the same pool. A failed OS-only policy
	// is not evidence that configured general DNS is unavailable. Check here,
	// rather than only at the query, because a queued trigger can arrive late.
	upstreams := p.buildRecoveryUpstreams(reason)
	upstreamNames := make([]string, 0, len(upstreams))
	for name := range upstreams {
		upstreamNames = append(upstreamNames, name)
	}
	slices.Sort(upstreamNames)
	if reason == RecoveryReasonOSFailure {
		if _, osOnly := upstreams[upstreamOS]; !osOnly {
			for _, name := range upstreamNames {
				if !p.um.isDown(name) {
					p.networkSourceMu.Unlock()
					p.refreshOSResolverAfterRecoverySkip(name)
					return
				}
			}
		}
	}
	recoveryCtx, gen, interceptRecovery, ok := p.beginRecovery(reason)
	p.networkSourceMu.Unlock()
	if !ok {
		p.logger.Load().Debug().Uint64("transition_id", transitionID).
			Uint64("recovery_generation", p.recoveryGen.Load()).
			Msg("Upstream recovery already in progress; skipping duplicate trigger")
		return
	}
	diagnostic := recoveryDiagnostic{logger: p.logger.Load(), transitionID: transitionID, generation: gen, reason: reason, started: time.Now()}
	journal(diagnostic.event(p.logger.Load().Info())).Bool("intercept", interceptRecovery).
		Strs("probe_upstreams", journalUpstreamNames(upstreamNames)).Msg("Recovery begin")
	p.logNetworkSnapshot("recovery_begin")
	targetBefore := p.interceptTargetSnapshot()
	outcome := recoveryOutcomeSuperseded
	defer func() { p.endRecovery(&diagnostic, outcome, targetBefore) }()
	// Clean every exit before the final snapshot, including cancellation before mutation.
	defer p.recoveryCanceledCleanup(gen)
	if recoveryCtx.Err() != nil {
		if p.recoveryGen.Load() == gen {
			outcome = recoveryOutcomeCanceled
		}
		return
	}
	if reason == RecoveryReasonNetworkChange {
		p.logger.Load().Debug().Msg("Network change recovery now owns shared recovery state")
	}

	// For network changes, force-reset all upstream transports synchronously.
	// The lazy ReBootstrap() called earlier in the network change callback only
	// sets a flag — the old transport's dead connections can still be used by
	// recovery probes, causing context deadline timeouts. ForceReBootstrap()
	// closes old connections and creates fresh transports so probes succeed on
	// first attempt. This runs outside recoveryCancelMu to avoid holding the
	// mutex during transport work.
	if reason == RecoveryReasonNetworkChange {
		loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load())
		for _, uc := range p.cfg.Upstream {
			if uc != nil {
				uc.ForceReBootstrap(loggerCtx)
			}
		}
		p.Info().Msg("Force-reset upstream transports for network change recovery")
	}

	if err := p.prepareForRecovery(reason, interceptRecovery, &diagnostic); err != nil {
		p.Error().Err(err).Msg("Failed to prepare for recovery")
		outcome = recoveryOutcomeFailed
		p.recoveryCanceledCleanup(gen)
		return
	}

	// Wait for the same upstream pool used to admit this recovery.
	recovered, err := waitForUpstreamRecoveryFn(p, recoveryCtx, upstreams, &diagnostic)
	if err != nil {
		if p.recoveryGen.Load() == gen {
			outcome = recoveryOutcomeCanceled
		}
		return
	}
	diagnostic.recoveredUpstream = recovered
	if !p.recoveryOwnsState(gen) {
		p.Debug().Msgf("Recovery generation %d was superseded after upstream success; skipping stale completion", gen)
		return
	}

	// The reset ends the outage, so the down time is read before it. The name
	// of the line is bounded, because an upstream key is operator text.
	journalName := journalUpstreamName(recovered)
	journal(p.Info()).Str("upstream", journalName).
		Int64("down_for_ms", p.um.downFor(recovered).Milliseconds()).
		Msgf("Upstream %q recovered; re-applying DNS settings", journalName)
	if err := p.completeRecoveryWork(reason, recovered, interceptRecovery); err != nil {
		p.Error().Err(err).Msg("Failed to complete recovery")
		p.recoveryCanceledCleanup(gen)
		return
	}
	if !p.completeRecoveryState(gen) {
		p.Debug().Msgf("Recovery generation %d was superseded during completion; preserving successor state", gen)
		return
	}

	outcome = recoveryOutcomeCompleted
	if interceptRecovery {
		journal(p.Info()).Msg("DNS intercept recovery complete: disabling DHCP bypass, resuming normal flow")
	}
	p.Info().Msgf("Recovery completed successfully for upstream %q", journalName)
}

// prepareForRecovery removes DNS settings and initializes OS resolver if needed.
func (p *prog) prepareForRecovery(reason RecoveryReason, interceptRecovery bool, diagnostic *recoveryDiagnostic) error {
	// In DNS intercept mode, don't tear down WFP/pf filters.
	// Instead, enable recovery bypass so proxy() forwards queries to
	// the OS/DHCP resolver. This handles captive portal authentication
	// without the overhead of filter teardown/rebuild.
	if interceptRecovery {
		journal(p.Info()).Msg("DNS intercept recovery: enabling DHCP bypass (filters stay active)")

		// Reinitialize OS resolver to discover DHCP servers on the new network.
		// This is critical for captive portals — we need the network's DNS servers
		// to resolve the auth page.
		p.Debug().Msg("DNS intercept recovery: discovering DHCP nameservers")
		loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load())
		// The effective list adds a synthetic public resolver when the network
		// gave none, so the journal reports the discovered list instead.
		resolverNameservers, systemNameservers := initializeOsResolverWithSystemNameserversFn(loggerCtx, true, recoveryResolverReason)
		diagnostic.dhcpServers = systemNameservers
		if len(systemNameservers) == 0 {
			journal(p.Warn()).Msg("DNS intercept recovery: no DHCP nameservers found")
		} else {
			journal(p.Info()).Strs("dhcp_servers", systemNameservers).
				Msgf("DNS intercept recovery: found DHCP nameservers: %v", systemNameservers)
		}

		ensureInterceptDNSTargetFn(p, systemNameservers)

		// Exempt DHCP nameservers from intercept filters so the OS resolver
		// can actually reach them on port 53. Without this, the WFP block
		// or pf redirect would intercept ctrld's own recovery queries.
		// The OS resolver queries the effective list, so the exemptions follow it.
		if len(resolverNameservers) > 0 {
			// Strip :53 port suffix if present (exemptVPNDNSServers expects bare IPs).
			var dhcpExemptions []vpnDNSExemption
			for _, s := range resolverNameservers {
				host := s
				if h, _, err := net.SplitHostPort(s); err == nil {
					host = h
				}
				dhcpExemptions = append(dhcpExemptions, vpnDNSExemption{Server: host})
			}
			p.Info().Msgf("DNS intercept recovery: exempting DHCP nameservers from filters: %v", resolverNameservers)
			if err := p.exemptVPNDNSServers(dhcpExemptions); err != nil {
				p.Warn().Err(err).Msg("DNS intercept recovery: failed to exempt DHCP nameservers — recovery queries may fail")
			}
		}

		return nil
	}

	// Traditional flow: remove DNS settings to expose DHCP nameservers
	// we do not want to restore any static DNS settings
	// we must try to get the DHCP values, any static DNS settings
	// will be appended to nameservers from the saved interface values
	recoveryResetDNSFn(p, false, false)

	// For an OS failure, reinitialize OS resolver nameservers immediately.
	if reason == RecoveryReasonOSFailure {
		if err := p.reinitializeOSResolver("OS resolver failure detected"); err != nil {
			return fmt.Errorf("failed to reinitialize OS resolver: %w", err)
		}
	}

	return nil
}

// reinitializeOSResolver reinitializes the OS resolver and logs the results.
func (p *prog) reinitializeOSResolver(message string) error {
	p.Debug().Msg(message)
	loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load())
	ns := ctrld.InitializeOsResolverWithReason(loggerCtx, true, recoveryResolverReason)
	if len(ns) == 0 {
		p.Warn().Msg("No nameservers found for OS resolver; using existing values")
	} else {
		journal(p.Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns)
	}
	return nil
}

// completeRecoveryWork performs owner-specific recovery work. Shared recovery
// flags are released separately by completeRecoveryState under the ownership lock.
func (p *prog) completeRecoveryWork(reason RecoveryReason, recovered string, interceptRecovery bool) error {
	p.um.reset(recovered)

	if interceptRecovery {
		// Refresh VPN DNS routes in case VPN state changed during recovery.
		if p.vpnDNS != nil {
			p.vpnDNS.Refresh(ctrld.LoggerCtx(context.Background(), p.logger.Load()), true)
		}
		if reason == RecoveryReasonNetworkChange {
			if err := p.reinitializeOSResolver("Network change detected during recovery"); err != nil {
				return fmt.Errorf("failed to reinitialize OS resolver during network change: %w", err)
			}
		}
		return nil
	}

	var systemNameservers []string
	if dnsIntercept {
		// Intercept was requested but was not active when recovery began. A
		// retry must use a completed raw discovery result, never nil.
		systemNameservers = p.systemNameserversForInterceptRetry()
	} else if reason == RecoveryReasonNetworkChange {
		if err := p.reinitializeOSResolver("Network change detected during recovery"); err != nil {
			return fmt.Errorf("failed to reinitialize OS resolver during network change: %w", err)
		}
	}

	// Apply our DNS settings back. The snapshot that follows the end event
	// reports the interfaces.
	p.setDNS(systemNameservers)
	return nil
}

// refreshOSResolverAfterRecoverySkip repairs stale OS resolver discovery without
// enabling bypass, changing host DNS, or clearing the monitor's failure state.
// Only a successful query establishes that the OS resolver recovered.
func (p *prog) refreshOSResolverAfterRecoverySkip(healthy string) {
	if !p.osRecoveryRefreshMu.TryLock() {
		return
	}
	defer p.osRecoveryRefreshMu.Unlock()

	now := networkEventsNowFn()
	if p.osRecoverySkipLogAt.IsZero() || healthy != p.osRecoverySkipUpstream || now.Sub(p.osRecoverySkipLogAt) >= 5*time.Minute {
		journal(p.Info()).Str("recovery_reason", recoveryReasonName(RecoveryReasonOSFailure)).
			Str("healthy_upstream", journalUpstreamName(healthy)).
			Msg("Recovery skipped: configured upstream is not marked down")
		p.osRecoverySkipLogAt = now
		p.osRecoverySkipUpstream = healthy
	}
	// Failed policy queries can arrive concurrently and at high volume. Retry
	// discovery on later traffic, but never run overlapping or per-query reads.
	if !p.osRecoveryRefreshAt.IsZero() && now.Sub(p.osRecoveryRefreshAt) < upstreamDownDelay {
		return
	}
	p.osRecoveryRefreshAt = now
	initializeOsResolverWithSystemNameserversFn(ctrld.LoggerCtx(context.Background(), p.logger.Load()), true, recoveryResolverReason)
}

// waitForUpstreamRecoveryFn is the seam of the upstream probe. A test drives
// the whole recovery flow without a query to a real upstream.
var waitForUpstreamRecoveryFn = (*prog).waitForUpstreamRecovery

var queryRecoveryFn = (*prog).handleRecovery

// waitForUpstreamRecovery checks the provided upstreams concurrently until one recovers.
// It returns the name of the recovered upstream or an error if the check times out.
func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string]*ctrld.UpstreamConfig, diagnostic *recoveryDiagnostic) (string, error) {
	recoveryCtx, cancel := context.WithCancel(ctx)
	defer cancel()

	recoveredCh := make(chan string, 1)
	var wg sync.WaitGroup

	p.Debug().Msgf("Starting upstream recovery check for %d upstreams", len(upstreams))
	defer func() { cancel(); wg.Wait() }()

	for name, uc := range upstreams {
		wg.Add(1)
		go func(name string, uc *ctrld.UpstreamConfig) {
			defer wg.Done()
			p.Debug().Msgf("Starting recovery check loop for upstream: %s", name)
			attempts := 0
			unreachableStreak := 0
			var failures upstreamFailureLog
			for {
				select {
				case <-recoveryCtx.Done():
					p.Debug().Msgf("Context canceled for upstream %s", name)
					return
				default:
					attempts++
					// The owning recovery resets the monitor after this probe succeeds.
					err := p.checkUpstreamOnce(name, uc, &failures)
					if err == nil || errors.Is(err, errOsHealthcheckSuppressed) {
						p.Debug().Msgf("Upstream %s recovered successfully", name)
						select {
						case recoveredCh <- name:
							p.Debug().Msgf("Sent recovery notification for upstream %s", name)
							cancel()
						default:
							p.Debug().Msg("Recovery channel full, another upstream already recovered")
						}
						return
					}
					diagnostic.failure(err)
					// Back off the retry cadence for an unroutable endpoint so a
					// host with IPv6 up but no route to the IPv6 DoH endpoint does
					// not re-bootstrap/re-check every checkUpstreamBackoffSleep and
					// spam the log. The backoff is bounded (checkUpstreamUnreachableBackoffMax)
					// so the endpoint is still re-probed and recovers when the route
					// returns; any other failure resets to the base cadence.
					sleep := checkUpstreamBackoffSleep
					if ctrldnet.IsUnreachable(err) {
						unreachableStreak++
						sleep = unreachableRecoveryBackoff(unreachableStreak)
						p.Debug().Msgf("Upstream %s unreachable (streak %d), backing off %s before retry", name, unreachableStreak, sleep)
					} else {
						unreachableStreak = 0
						p.Debug().Msgf("Upstream %s check failed, sleeping before retry", name)
					}
					if !sleepWithContext(recoveryCtx, sleep) {
						return
					}

					// if this is the upstreamOS and it's the 3rd attempt (or multiple of 3),
					// we should try to reinit the OS resolver to ensure we can recover
					if name == upstreamOS && attempts%3 == 0 {
						p.Debug().Msgf("UpstreamOS check failed on attempt %d, reinitializing OS resolver", attempts)
						ns := ctrld.InitializeOsResolverWithReason(ctrld.LoggerCtx(ctx, p.logger.Load()), true, recoveryResolverReason)
						if len(ns) == 0 {
							p.Warn().Msg("No nameservers found for OS resolver; using existing values")
						} else {
							journal(p.Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns)
						}
					}
				}
			}
		}(name, uc)
	}

	var recovered string
	select {
	case <-ctx.Done():
		return "", ctx.Err()
	default:
	}
	select {
	case recovered = <-recoveredCh:
		if err := ctx.Err(); err != nil {
			return "", err
		}
	case <-ctx.Done():
		return "", ctx.Err()
	}
	return recovered, nil
}

func sleepWithContext(ctx context.Context, d time.Duration) bool {
	timer := time.NewTimer(d)
	defer timer.Stop()
	select {
	case <-timer.C:
		return true
	case <-ctx.Done():
		return false
	}
}

// buildRecoveryUpstreams constructs the map of upstream configurations to test.
// OS failures use configured non-OS upstreams for both admission and recovery,
// excluding generated Internal Domain resolvers. OS-only configurations retain
// the OS recovery path. Other recovery reasons retain their configured pool.
func (p *prog) buildRecoveryUpstreams(reason RecoveryReason) map[string]*ctrld.UpstreamConfig {
	upstreams := make(map[string]*ctrld.UpstreamConfig)
	switch reason {
	case RecoveryReasonOSFailure:
		for k, uc := range p.cfg.Upstream {
			name := upstreamPrefix + k
			if uc != nil && uc.Type != ctrld.ResolverTypeOS && !isInternalDomainUpstream(name) {
				upstreams[name] = uc
			}
		}
		if len(upstreams) == 0 {
			upstreams[upstreamOS] = osUpstreamConfig
		}
	case RecoveryReasonNetworkChange, RecoveryReasonRegularFailure:
		// Use all configured upstreams except any OS type.
		for k, uc := range p.cfg.Upstream {
			if uc != nil && uc.Type != ctrld.ResolverTypeOS {
				upstreams[upstreamPrefix+k] = uc
			}
		}
		// An OS-only configuration still needs a recovery worker. Do not add
		// OS to mixed pools: their existing recovery boundaries stay intact.
		if len(upstreams) == 0 {
			upstreams[upstreamOS] = osUpstreamConfig
		}
	}
	return upstreams
}

// ValidateDefaultLocalIPsFromDelta checks if the default local IPv4 and IPv6 stored
// are still present in the new network state (provided by delta.New).
// If a stored default IP is no longer active, it resets that default (sets it to nil)
// so that it won't be used in subsequent custom dialer contexts.
// Deprecated: retained for callers of the existing exported cli API.
// The network monitor uses transition-aware validation instead.
func ValidateDefaultLocalIPsFromDelta(newState *netmon.State) {
	validateDefaultLocalIPsFromDelta(ctrld.LoggerCtx(context.Background(), mainLog.Load()), newState, 0)
}