mirror of
https://github.com/Control-D-Inc/ctrld.git
synced 2026-10-12 01:58:46 +02:00
Anthony Wong
8ae09ac98e
The journal records the network state of the host and each change of that state. The support team reads one file to see the interfaces, the routes, the resolvers, and the recoveries around an incident. The same file shows the health of the query path. A change that touches only AirDrop or virtual interfaces runs no DNS work.
2607 lines
94 KiB
Go
2607 lines
94 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"net/netip"
|
|
"os/exec"
|
|
"runtime"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/miekg/dns"
|
|
"golang.org/x/sync/errgroup"
|
|
"tailscale.com/net/netmon"
|
|
"tailscale.com/net/tsaddr"
|
|
|
|
"github.com/Control-D-Inc/ctrld"
|
|
"github.com/Control-D-Inc/ctrld/internal/controld"
|
|
"github.com/Control-D-Inc/ctrld/internal/dnscache"
|
|
ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
|
|
"github.com/Control-D-Inc/ctrld/internal/router"
|
|
)
|
|
|
|
const (
|
|
staleTTL = 60 * time.Second
|
|
localTTL = 3600 * time.Second
|
|
// zpaDNSEchoDomain is Zscaler Client Connector's fixed public health-check
|
|
// name. Client Connector must see the answer to it arrive on its own DNS
|
|
// path before it will synthesize addresses for Private Access applications.
|
|
zpaDNSEchoDomain = "dnsechotest.zscaler.com"
|
|
// zpaDNSEchoPolicyName is reported as the matched policy when the built-in
|
|
// ZPA DNS echo bypass routes a query, so log lines name the reason.
|
|
zpaDNSEchoPolicyName = "Built-in ZPA DNS echo bypass"
|
|
// noRuleMatched is the matchedRule placeholder used when no listener domain
|
|
// rule matched the query. It is compared, not only logged: the built-in ZPA
|
|
// echo bypass yields to an explicit domain rule for the same name.
|
|
noRuleMatched = "no rule"
|
|
// EDNS0_OPTION_MAC is dnsmasq EDNS0 code for adding mac option.
|
|
// https://thekelleys.org.uk/gitweb/?p=dnsmasq.git;a=blob;f=src/dns-protocol.h;h=76ac66a8c28317e9c121a74ab5fd0e20f6237dc8;hb=HEAD#l81
|
|
// This is also dns.EDNS0LOCALSTART, but define our own constant here for clarification.
|
|
EDNS0_OPTION_MAC = 0xFDE9
|
|
|
|
// selfUninstallMaxQueries is number of REFUSED queries seen before checking for self-uninstallation.
|
|
selfUninstallMaxQueries = 32
|
|
)
|
|
|
|
// zpaDNSEchoBypassEnabled limits the built-in ZPA DNS echo bypass to macOS,
|
|
// the only platform where pf interception is known to break Zscaler Private
|
|
// Access. Tests override it.
|
|
var zpaDNSEchoBypassEnabled = runtime.GOOS == "darwin"
|
|
|
|
// zpaDNSEchoBypassActive reports whether the built-in Zscaler DNS echo bypass
|
|
// applies. Like every other VPN accommodation it is skipped in
|
|
// --intercept-mode hard, where the operator has asked for all DNS to go
|
|
// through ctrld with no split routing.
|
|
func zpaDNSEchoBypassActive() bool {
|
|
return zpaDNSEchoBypassEnabled && dnsIntercept && !hardIntercept
|
|
}
|
|
|
|
var osUpstreamConfig = &ctrld.UpstreamConfig{
|
|
Name: "OS resolver",
|
|
Type: ctrld.ResolverTypeOS,
|
|
Timeout: 3000,
|
|
}
|
|
|
|
var privateUpstreamConfig = &ctrld.UpstreamConfig{
|
|
Name: "Private resolver",
|
|
Type: ctrld.ResolverTypePrivate,
|
|
Timeout: 2000,
|
|
}
|
|
|
|
var localUpstreamConfig = &ctrld.UpstreamConfig{
|
|
Name: "Local resolver",
|
|
Type: ctrld.ResolverTypeLocal,
|
|
Timeout: 2000,
|
|
}
|
|
|
|
// proxyRequest contains data for proxying a DNS query to upstream.
|
|
type proxyRequest struct {
|
|
msg *dns.Msg
|
|
ci *ctrld.ClientInfo
|
|
failoverRcodes []int
|
|
ufr *upstreamForResult
|
|
}
|
|
|
|
// proxyResponse contains data for proxying a DNS response from upstream.
|
|
type proxyResponse struct {
|
|
answer *dns.Msg
|
|
cached bool
|
|
clientInfo bool
|
|
upstream string
|
|
}
|
|
|
|
// upstreamForResult represents the result of processing rules for a request.
|
|
type upstreamForResult struct {
|
|
upstreams []string
|
|
matchedPolicy string
|
|
matchedNetwork string
|
|
matchedRule string
|
|
matched bool
|
|
srcAddr string
|
|
// zpaDNSEcho reports that the built-in Zscaler DNS echo bypass chose the
|
|
// upstreams. proxy() keeps this query out of the response cache so Client
|
|
// Connector's health check reaches the OS resolver on every poll.
|
|
zpaDNSEcho bool
|
|
}
|
|
|
|
func (p *prog) addCachedResponse(key dnscache.Key, answer *dns.Msg) {
|
|
ttl := ttlFromMsg(answer)
|
|
now := time.Now()
|
|
expired := now.Add(time.Duration(ttl) * time.Second)
|
|
if cachedTTL := p.cfg.Service.CacheTTLOverride; cachedTTL > 0 {
|
|
expired = now.Add(time.Duration(cachedTTL) * time.Second)
|
|
}
|
|
setCachedAnswerTTL(answer, now, expired)
|
|
p.cache.Add(key, dnscache.NewValue(answer, expired))
|
|
}
|
|
|
|
func (p *prog) cachedResponse(req *dns.Msg, upstream string, dns64Prefix netip.Prefix, dns64Active bool, now time.Time) (answer, stale *dns.Msg, hit, dns64Hit, dns64Bypass bool) {
|
|
if dns64Active {
|
|
if cachedValue := p.cache.Get(dns64CacheKey(req, upstream, dns64Prefix)); cachedValue != nil {
|
|
answer = cachedValue.Msg.Copy()
|
|
ctrld.SetCacheReply(answer, req, answer.Rcode)
|
|
if cachedValue.Expire.After(now) {
|
|
setCachedAnswerTTL(answer, now, cachedValue.Expire)
|
|
return answer, nil, true, true, false
|
|
}
|
|
stale = answer
|
|
}
|
|
}
|
|
|
|
cachedValue := p.cache.Get(dnscache.NewKey(req, upstream))
|
|
if cachedValue == nil {
|
|
return nil, stale, false, false, false
|
|
}
|
|
answer = cachedValue.Msg.Copy()
|
|
ctrld.SetCacheReply(answer, req, answer.Rcode)
|
|
if cachedValue.Expire.After(now) {
|
|
if dns64Eligible(req, answer) && dns64Active {
|
|
if stale == nil {
|
|
stale = answer
|
|
}
|
|
return nil, stale, false, false, true
|
|
}
|
|
setCachedAnswerTTL(answer, now, cachedValue.Expire)
|
|
return answer, stale, true, false, false
|
|
}
|
|
if stale == nil {
|
|
stale = answer
|
|
}
|
|
return nil, stale, false, false, false
|
|
}
|
|
|
|
func (p *prog) serveDNS(listenerNum string) error {
|
|
listenerConfig := p.cfg.Listener[listenerNum]
|
|
// make sure ip is allocated
|
|
if allocErr := p.allocateIP(listenerConfig.IP); allocErr != nil {
|
|
mainLog.Load().Error().Err(allocErr).Str("ip", listenerConfig.IP).Msg("serveUDP: failed to allocate listen ip")
|
|
return allocErr
|
|
}
|
|
|
|
handler := dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) {
|
|
p.sema.acquire()
|
|
defer p.sema.release()
|
|
if len(m.Question) == 0 {
|
|
answer := new(dns.Msg)
|
|
answer.SetRcode(m, dns.RcodeFormatError)
|
|
_ = w.WriteMsg(answer)
|
|
return
|
|
}
|
|
listenerConfig := p.cfg.Listener[listenerNum]
|
|
reqId := requestID()
|
|
ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, reqId)
|
|
if !listenerConfig.AllowWanClients && isWanClient(w.RemoteAddr()) && !isIPv6LoopbackListener(w.LocalAddr()) {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "query refused, listener does not allow WAN clients: %s", w.RemoteAddr().String())
|
|
answer := new(dns.Msg)
|
|
answer.SetRcode(m, dns.RcodeRefused)
|
|
_ = w.WriteMsg(answer)
|
|
return
|
|
}
|
|
go p.detectLoop(m)
|
|
q := m.Question[0]
|
|
domain := canonicalName(q.Name)
|
|
switch {
|
|
case domain == "":
|
|
answer := new(dns.Msg)
|
|
answer.SetRcode(m, dns.RcodeFormatError)
|
|
_ = w.WriteMsg(answer)
|
|
return
|
|
case domain == selfCheckInternalTestDomain:
|
|
answer := resolveInternalDomainTestQuery(ctx, domain, m)
|
|
_ = w.WriteMsg(answer)
|
|
return
|
|
}
|
|
|
|
// Interception probe: if we're expecting a probe query and this matches,
|
|
// signal the prober and respond NXDOMAIN. Used by both macOS pf probes
|
|
// (_pf-probe-*) and Windows NRPT probes (_nrpt-probe-*) to verify that
|
|
// DNS interception is actually routing queries to ctrld's listener.
|
|
if p.signalInterceptProbe(domain) {
|
|
answer := new(dns.Msg)
|
|
answer.SetRcode(m, dns.RcodeNameError) // NXDOMAIN
|
|
_ = w.WriteMsg(answer)
|
|
return
|
|
}
|
|
|
|
if _, ok := p.cacheFlushDomainsMap[domain]; ok && p.cache != nil {
|
|
p.cache.Purge()
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "received query %q, local cache is purged", domain)
|
|
}
|
|
remoteIP, _, _ := net.SplitHostPort(w.RemoteAddr().String())
|
|
ci := p.getClientInfo(remoteIP, m)
|
|
ci.ClientIDPref = p.cfg.Service.ClientIDPref
|
|
stripClientSubnet(m)
|
|
remoteAddr := spoofRemoteAddr(w.RemoteAddr(), ci)
|
|
fmtSrcToDest := fmtRemoteToLocal(listenerNum, ci.Hostname, remoteAddr.String())
|
|
t := time.Now()
|
|
ctrld.Log(ctx, mainLog.Load().Info(), "QUERY: %s: %s %s", fmtSrcToDest, dns.TypeToString[q.Qtype], domain)
|
|
ur := p.upstreamFor(ctx, listenerNum, listenerConfig, remoteAddr, ci.Mac, domain)
|
|
|
|
labelValues := make([]string, 0, len(statsQueriesCountLabels))
|
|
labelValues = append(labelValues, net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port)))
|
|
labelValues = append(labelValues, ci.IP)
|
|
labelValues = append(labelValues, ci.Mac)
|
|
labelValues = append(labelValues, ci.Hostname)
|
|
|
|
var answer *dns.Msg
|
|
if !ur.matched && listenerConfig.Restricted {
|
|
ctrld.Log(ctx, mainLog.Load().Info(), "query refused, %s does not match any network policy", remoteAddr.String())
|
|
answer = new(dns.Msg)
|
|
answer.SetRcode(m, dns.RcodeRefused)
|
|
labelValues = append(labelValues, "") // no upstream
|
|
} else {
|
|
var failoverRcode []int
|
|
if listenerConfig.Policy != nil {
|
|
failoverRcode = listenerConfig.Policy.FailoverRcodeNumbers
|
|
}
|
|
pr := p.proxy(ctx, &proxyRequest{
|
|
msg: m,
|
|
ci: ci,
|
|
failoverRcodes: failoverRcode,
|
|
ufr: ur,
|
|
})
|
|
go p.doSelfUninstall(pr.answer)
|
|
|
|
answer = pr.answer
|
|
rtt := time.Since(t)
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "received response of %d bytes in %s", answer.Len(), rtt)
|
|
upstream := pr.upstream
|
|
switch {
|
|
case pr.cached:
|
|
upstream = "cache"
|
|
case pr.clientInfo:
|
|
upstream = "client_info_table"
|
|
}
|
|
labelValues = append(labelValues, upstream)
|
|
}
|
|
labelValues = append(labelValues, dns.TypeToString[q.Qtype])
|
|
labelValues = append(labelValues, dns.RcodeToString[answer.Rcode])
|
|
// The grade of the window must not depend on the goroutine below.
|
|
p.health.countQuery()
|
|
go func() {
|
|
p.WithLabelValuesInc(statsQueriesCount, labelValues...)
|
|
p.WithLabelValuesInc(statsClientQueriesCount, []string{ci.IP, ci.Mac, ci.Hostname}...)
|
|
p.forceFetchingAPI(domain)
|
|
}()
|
|
if err := w.WriteMsg(answer); err != nil {
|
|
ctrld.Log(ctx, p.querySampler.event(sampleClassSendResponseFailed, "").Err(err), "serveDNS: failed to send DNS response to client")
|
|
}
|
|
})
|
|
|
|
g, ctx := errgroup.WithContext(context.Background())
|
|
for _, proto := range []string{"udp", "tcp"} {
|
|
proto := proto
|
|
if needLocalIPv6Listener(p.cfg.Service.InterceptMode) {
|
|
g.Go(func() error {
|
|
s, errCh := runDNSServer(net.JoinHostPort("::1", strconv.Itoa(listenerConfig.Port)), proto, handler)
|
|
defer s.Shutdown()
|
|
select {
|
|
case <-p.stopCh:
|
|
case <-ctx.Done():
|
|
case err := <-errCh:
|
|
// Local ipv6 listener should not terminate ctrld.
|
|
// It's a workaround for a quirk on Windows.
|
|
mainLog.Load().Warn().Err(err).Msg("local ipv6 listener failed")
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
// When we spawn a listener on 127.0.0.1, also spawn listeners on the RFC1918 addresses of the machine
|
|
// if explicitly set via setting rfc1918 flag, so ctrld could receive queries from LAN clients.
|
|
if needRFC1918Listeners(listenerConfig) {
|
|
g.Go(func() error {
|
|
for _, addr := range ctrld.Rfc1918Addresses() {
|
|
func() {
|
|
listenAddr := net.JoinHostPort(addr, strconv.Itoa(listenerConfig.Port))
|
|
s, errCh := runDNSServer(listenAddr, proto, handler)
|
|
defer s.Shutdown()
|
|
select {
|
|
case <-p.stopCh:
|
|
case <-ctx.Done():
|
|
case err := <-errCh:
|
|
// RFC1918 listener should not terminate ctrld.
|
|
// It's a workaround for a quirk on system with systemd-resolved.
|
|
mainLog.Load().Warn().Err(err).Msgf("could not listen on %s: %s", proto, listenAddr)
|
|
}
|
|
}()
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
g.Go(func() error {
|
|
addr := net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port))
|
|
s, errCh := runDNSServer(addr, proto, handler)
|
|
defer s.Shutdown()
|
|
|
|
p.started <- struct{}{}
|
|
|
|
select {
|
|
case <-p.stopCh:
|
|
case <-ctx.Done():
|
|
case err := <-errCh:
|
|
return err
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
return g.Wait()
|
|
}
|
|
|
|
// upstreamFor returns the list of upstreams for resolving the given domain,
|
|
// matching by policies defined in the listener config. The second return value
|
|
// reports whether the domain matches the policy.
|
|
//
|
|
// Though domain policy has higher priority than network policy, it is still
|
|
// processed later, because policy logging want to know whether a network rule
|
|
// is disregarded in favor of the domain level rule.
|
|
func (p *prog) upstreamFor(ctx context.Context, defaultUpstreamNum string, lc *ctrld.ListenerConfig, addr net.Addr, srcMac, domain string) (res *upstreamForResult) {
|
|
upstreams := []string{upstreamPrefix + defaultUpstreamNum}
|
|
matchedPolicy := "no policy"
|
|
matchedNetwork := "no network"
|
|
matchedRule := noRuleMatched
|
|
matched := false
|
|
zpaDNSEcho := false
|
|
res = &upstreamForResult{srcAddr: addr.String()}
|
|
|
|
defer func() {
|
|
// Zscaler Private Access: under macOS Intercept Mode, pf routes Client
|
|
// Connector's health-check query for zpaDNSEchoDomain into ctrld, which
|
|
// answers it from the Control D upstream. Client Connector never sees the
|
|
// answer on its own DNS path, so it keeps Private Access disabled.
|
|
//
|
|
// Route only that one exact name the way a Control D "bypass" rule does:
|
|
// an empty upstream list, which proxy() resolves through upstream.os —
|
|
// the system resolver set, which under Intercept Mode still includes
|
|
// Client Connector's own DNS servers. Matching on the name alone covers
|
|
// every record type, the same as a bypass rule.
|
|
//
|
|
// This runs after policy evaluation, and never touches `matched`. That
|
|
// flag is also the source authorization bit: serveDNS refuses a query on
|
|
// a Restricted listener when it is false, so forcing it true here would
|
|
// let an unauthorized client resolve this name through such a listener.
|
|
//
|
|
// The scope is one fixed public health-check name: every other domain,
|
|
// including all Private Access application domains, stays intercepted and
|
|
// filtered by Control D. --intercept-mode hard opts out, the same as it
|
|
// opts out of VPN DNS split routing.
|
|
if zpaDNSEchoBypassActive() && canonicalName(domain) == zpaDNSEchoDomain {
|
|
switch {
|
|
case matchedRule == noRuleMatched:
|
|
// The listener policy says nothing about this name, so the
|
|
// built-in route decides. Network- and MAC-policy targets route
|
|
// every name from a source and so state nothing about this one:
|
|
// override them, and the default upstream, and label the result
|
|
// so logs name the reason.
|
|
upstreams = nil
|
|
matchedPolicy = zpaDNSEchoPolicyName
|
|
matchedRule = zpaDNSEchoDomain
|
|
zpaDNSEcho = true
|
|
case len(upstreams) == 0:
|
|
// An explicit domain rule already selects the OS path: a rule
|
|
// with empty targets, which is how a Control D profile's bypass
|
|
// list reaches us (see the cfg.Listener rules built from
|
|
// resolverConfig.Exclude). That is the documented pre-fix
|
|
// workaround for this very bug, so keep the operator's own
|
|
// labels but still classify it as the health check — otherwise
|
|
// anyone who keeps the workaround after upgrading would route
|
|
// correctly and then have the answer served from our cache,
|
|
// losing the reconnect fix.
|
|
zpaDNSEcho = true
|
|
default:
|
|
// Any other explicit domain rule is a deliberate non-OS route
|
|
// for this exact name. That is operator intent, and the
|
|
// supported way to opt out without leaving Intercept Mode, so
|
|
// leave the policy's decision untouched.
|
|
}
|
|
}
|
|
res.upstreams = upstreams
|
|
res.matched = matched
|
|
res.matchedPolicy = matchedPolicy
|
|
res.matchedNetwork = matchedNetwork
|
|
res.matchedRule = matchedRule
|
|
res.zpaDNSEcho = zpaDNSEcho
|
|
}()
|
|
|
|
if lc.Policy == nil {
|
|
return
|
|
}
|
|
|
|
do := func(policyUpstreams []string) {
|
|
upstreams = append([]string(nil), policyUpstreams...)
|
|
}
|
|
|
|
var networkTargets []string
|
|
var sourceIP net.IP
|
|
switch addr := addr.(type) {
|
|
case *net.UDPAddr:
|
|
sourceIP = addr.IP
|
|
case *net.TCPAddr:
|
|
sourceIP = addr.IP
|
|
}
|
|
|
|
networkRules:
|
|
for _, rule := range lc.Policy.Networks {
|
|
for source, targets := range rule {
|
|
networkNum := strings.TrimPrefix(source, "network.")
|
|
nc := p.cfg.Network[networkNum]
|
|
if nc == nil {
|
|
continue
|
|
}
|
|
for _, ipNet := range nc.IPNets {
|
|
if ipNet.Contains(sourceIP) {
|
|
matchedPolicy = lc.Policy.Name
|
|
matchedNetwork = source
|
|
networkTargets = targets
|
|
matched = true
|
|
break networkRules
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
macRules:
|
|
for _, rule := range lc.Policy.Macs {
|
|
for source, targets := range rule {
|
|
if source != "" && (strings.EqualFold(source, srcMac) || wildcardMatches(strings.ToLower(source), strings.ToLower(srcMac))) {
|
|
matchedPolicy = lc.Policy.Name
|
|
matchedNetwork = source
|
|
networkTargets = targets
|
|
matched = true
|
|
break macRules
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, rule := range lc.Policy.Rules {
|
|
// There's only one entry per rule, config validation ensures this.
|
|
for source, targets := range rule {
|
|
if source == domain || wildcardMatches(source, domain) {
|
|
matchedPolicy = lc.Policy.Name
|
|
if len(networkTargets) > 0 {
|
|
matchedNetwork += " (unenforced)"
|
|
}
|
|
matchedRule = source
|
|
do(targets)
|
|
matched = true
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
if matched {
|
|
do(networkTargets)
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func (p *prog) proxyPrivatePtrLookup(ctx context.Context, msg *dns.Msg) *dns.Msg {
|
|
cDomainName := msg.Question[0].Name
|
|
locked := p.ptrLoopGuard.TryLock(cDomainName)
|
|
defer p.ptrLoopGuard.Unlock(cDomainName)
|
|
if !locked {
|
|
return nil
|
|
}
|
|
ip := ipFromARPA(cDomainName)
|
|
if name := p.ciTable.LookupHostname(ip.String(), ""); name != "" {
|
|
answer := new(dns.Msg)
|
|
answer.SetReply(msg)
|
|
answer.Compress = true
|
|
answer.Answer = []dns.RR{&dns.PTR{
|
|
Hdr: dns.RR_Header{
|
|
Name: msg.Question[0].Name,
|
|
Rrtype: dns.TypePTR,
|
|
Class: dns.ClassINET,
|
|
},
|
|
Ptr: dns.Fqdn(name),
|
|
}}
|
|
ctrld.Log(ctx, mainLog.Load().Info(), "private PTR lookup, using client info table")
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "client info: %v", ctrld.ClientInfo{
|
|
Mac: p.ciTable.LookupMac(ip.String()),
|
|
IP: ip.String(),
|
|
Hostname: name,
|
|
})
|
|
return answer
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (p *prog) proxyLanHostnameQuery(ctx context.Context, msg *dns.Msg) *dns.Msg {
|
|
q := msg.Question[0]
|
|
hostname := strings.TrimSuffix(q.Name, ".")
|
|
locked := p.lanLoopGuard.TryLock(hostname)
|
|
defer p.lanLoopGuard.Unlock(hostname)
|
|
if !locked {
|
|
return nil
|
|
}
|
|
if ip := p.ciTable.LookupIPByHostname(hostname, q.Qtype == dns.TypeAAAA); ip != nil {
|
|
answer := new(dns.Msg)
|
|
answer.SetReply(msg)
|
|
answer.Compress = true
|
|
switch {
|
|
case ip.Is4():
|
|
answer.Answer = []dns.RR{&dns.A{
|
|
Hdr: dns.RR_Header{
|
|
Name: msg.Question[0].Name,
|
|
Rrtype: dns.TypeA,
|
|
Class: dns.ClassINET,
|
|
Ttl: uint32(localTTL.Seconds()),
|
|
},
|
|
A: ip.AsSlice(),
|
|
}}
|
|
case ip.Is6():
|
|
answer.Answer = []dns.RR{&dns.AAAA{
|
|
Hdr: dns.RR_Header{
|
|
Name: msg.Question[0].Name,
|
|
Rrtype: dns.TypeAAAA,
|
|
Class: dns.ClassINET,
|
|
Ttl: uint32(localTTL.Seconds()),
|
|
},
|
|
AAAA: ip.AsSlice(),
|
|
}}
|
|
}
|
|
ctrld.Log(ctx, mainLog.Load().Info(), "lan hostname lookup, using client info table")
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "client info: %v", ctrld.ClientInfo{
|
|
Mac: p.ciTable.LookupMac(ip.String()),
|
|
IP: ip.String(),
|
|
Hostname: hostname,
|
|
})
|
|
return answer
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse {
|
|
// DNS intercept recovery bypass: forward all queries to OS/DHCP resolver.
|
|
// This runs when upstreams are unreachable (e.g., captive portal network)
|
|
// and allows the network's DNS to handle authentication pages.
|
|
//
|
|
// An Internal Domain with explicit resolvers does not take part. The bypass
|
|
// exists because general DNS is broken, which says nothing about the
|
|
// administrator's own resolvers: they may well be reachable, and sending a
|
|
// private name to the network's DNS is the disclosure the explicit
|
|
// selection exists to prevent. Such a query continues to the normal flow,
|
|
// where it reaches only its configured resolvers and fails if none answer.
|
|
if dnsIntercept && p.recoveryBypass.Load() && !internalDomainExplicitUpstreams(req.ufr.upstreams) {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "Recovery bypass active: forwarding to OS resolver")
|
|
resolver, err := ctrld.NewResolver(osUpstreamConfig)
|
|
if err == nil {
|
|
resolveCtx, cancel := osUpstreamConfig.Context(ctx)
|
|
defer cancel()
|
|
answer, _ := resolver.Resolve(resolveCtx, req.msg)
|
|
if answer != nil {
|
|
return &proxyResponse{answer: answer}
|
|
}
|
|
}
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "OS resolver failed during recovery bypass")
|
|
// Fall through to normal flow as last resort
|
|
}
|
|
|
|
var staleAnswer *dns.Msg
|
|
upstreams := req.ufr.upstreams
|
|
serveStaleCache := p.cache != nil && p.cfg.Service.CacheServeStale
|
|
upstreamConfigs := p.upstreamConfigsFromUpstreamNumbers(upstreams)
|
|
|
|
// Inverse queries must not be cached:
|
|
// https://www.rfc-editor.org/rfc/rfc1035#section-7.4
|
|
//
|
|
// Neither must the Zscaler health check. Its whole purpose is to be observed
|
|
// arriving on Client Connector's own DNS path, so it has to reach the OS
|
|
// resolver on every poll — a locally served answer looks like a successful
|
|
// lookup to us and like silence to Client Connector. This matters most on
|
|
// reconnect: an answer cached while ZPA was disconnected would otherwise
|
|
// outlive InitializeOsResolver() and keep Private Access disabled until the
|
|
// entry's TTL (or Service.CacheTTLOverride) expired.
|
|
cacheable := p.cache != nil && req.msg.Question[0].Qtype != dns.TypePTR && !req.ufr.zpaDNSEcho
|
|
if req.ufr.zpaDNSEcho {
|
|
serveStaleCache = false
|
|
}
|
|
|
|
if len(upstreamConfigs) == 0 {
|
|
upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig}
|
|
upstreams = []string{upstreamOS}
|
|
// For OS resolver, local addresses are ignored to prevent possible looping.
|
|
// However, on Active Directory Domain Controller, where it has local DNS server
|
|
// running and listening on local addresses, these local addresses must be used
|
|
// as nameservers, so queries for ADDC could be resolved as expected.
|
|
if p.isAdDomainQuery(req.msg) && p.hasLocalDNS {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(),
|
|
"AD domain query detected for %s in domain %s, using local DNS server",
|
|
req.msg.Question[0].Name, p.adDomain)
|
|
upstreamConfigs = []*ctrld.UpstreamConfig{localUpstreamConfig}
|
|
upstreams = []string{upstreamOSLocal}
|
|
}
|
|
}
|
|
|
|
res := &proxyResponse{}
|
|
|
|
// LAN/PTR lookup flow:
|
|
//
|
|
// 1. If there's matching rule, follow it.
|
|
// 2. Try from client info table.
|
|
// 3. Try private resolver.
|
|
// 4. Try remote upstream.
|
|
isLanOrPtrQuery := false
|
|
if req.ufr.matched {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "%s, %s, %s -> %v", req.ufr.matchedPolicy, req.ufr.matchedNetwork, req.ufr.matchedRule, upstreams)
|
|
} else {
|
|
switch {
|
|
case req.ufr.zpaDNSEcho:
|
|
// Reached when no policy matched the source, so the branch above did
|
|
// not log the bypass. Named explicitly rather than falling into the
|
|
// "no explicit policy matched" default.
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "%s, %s -> %v", req.ufr.matchedPolicy, req.ufr.matchedRule, upstreams)
|
|
case isSrvLanLookup(req.msg):
|
|
upstreams = []string{upstreamOS}
|
|
upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig}
|
|
ctx = ctrld.LanQueryCtx(ctx)
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "SRV record lookup, using upstreams: %v", upstreams)
|
|
case isPrivatePtrLookup(req.msg):
|
|
isLanOrPtrQuery = true
|
|
if answer := p.proxyPrivatePtrLookup(ctx, req.msg); answer != nil {
|
|
res.answer = answer
|
|
res.clientInfo = true
|
|
return res
|
|
}
|
|
upstreams, upstreamConfigs = p.upstreamsAndUpstreamConfigForPtr(upstreams, upstreamConfigs)
|
|
ctx = ctrld.LanQueryCtx(ctx)
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "private PTR lookup, using upstreams: %v", upstreams)
|
|
case isLanHostnameQuery(req.msg):
|
|
isLanOrPtrQuery = true
|
|
if answer := p.proxyLanHostnameQuery(ctx, req.msg); answer != nil {
|
|
res.answer = answer
|
|
res.clientInfo = true
|
|
return res
|
|
}
|
|
upstreams = []string{upstreamOS}
|
|
upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig}
|
|
ctx = ctrld.LanQueryCtx(ctx)
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "lan hostname lookup, using upstreams: %v", upstreams)
|
|
default:
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "no explicit policy matched, using default routing -> %v", upstreams)
|
|
}
|
|
}
|
|
|
|
if cacheable {
|
|
dns64Prefix, dns64Active := netip.Prefix{}, false
|
|
if req.msg.Question[0].Qtype == dns.TypeAAAA {
|
|
dns64Prefix, dns64Active = p.activeDNS64Prefix()
|
|
}
|
|
for _, upstream := range upstreams {
|
|
answer, stale, hit, dns64Hit, dns64Bypass := p.cachedResponse(req.msg, upstream, dns64Prefix, dns64Active, time.Now())
|
|
if stale != nil {
|
|
staleAnswer = stale
|
|
}
|
|
if dns64Bypass {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "dns64: bypassing cached empty-AAAA answer for synthesis")
|
|
}
|
|
if !hit {
|
|
continue
|
|
}
|
|
if dns64Hit {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "dns64: hit cached response variant")
|
|
} else {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "hit cached response")
|
|
}
|
|
p.health.countCacheHit()
|
|
res.answer = answer
|
|
res.cached = true
|
|
return res
|
|
}
|
|
}
|
|
|
|
// VPN DNS split routing (only in dns-intercept mode).
|
|
//
|
|
// An Internal Domain with explicit resolvers is skipped here: the
|
|
// administrator named the resolvers for that suffix, and VPN suffixes
|
|
// are auto-detected, so handing the query to a VPN DNS server would
|
|
// override an explicit selection with a discovered one.
|
|
//
|
|
// Internal Domains in OS-resolver mode are not skipped, they ask for the
|
|
// endpoint's default resolution, which under intercept mode includes
|
|
// the VPN's own resolver.
|
|
if dnsIntercept && p.vpnDNS != nil && len(req.msg.Question) > 0 && !internalDomainExplicitUpstreams(upstreams) {
|
|
domain := req.msg.Question[0].Name
|
|
if vpnServers := p.vpnDNS.UpstreamForDomain(domain); len(vpnServers) > 0 {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "VPN DNS route matched for domain %s, using servers: %v", domain, vpnServers)
|
|
|
|
var gotTransportFailure bool
|
|
for _, server := range vpnServers {
|
|
upstreamConfig := p.vpnDNS.upstreamConfigFor(server)
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "Querying VPN DNS server: %s", server)
|
|
|
|
dnsResolver, err := ctrld.NewResolver(upstreamConfig)
|
|
if err != nil {
|
|
ctrld.Log(ctx, mainLog.Load().Error().Err(err), "failed to create VPN DNS resolver")
|
|
continue
|
|
}
|
|
resolveCtx, cancel := upstreamConfig.Context(ctx)
|
|
answer, err := dnsResolver.Resolve(resolveCtx, req.msg)
|
|
cancel()
|
|
if answer != nil {
|
|
p.vpnDNS.VPNDNSReachable()
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "VPN DNS query successful")
|
|
if cacheable {
|
|
ttl := 60 * time.Second
|
|
if len(answer.Answer) > 0 {
|
|
ttl = time.Duration(answer.Answer[0].Header().Ttl) * time.Second
|
|
}
|
|
for _, upstream := range upstreams {
|
|
p.cache.Add(dnscache.NewKey(req.msg, upstream), dnscache.NewValue(answer, time.Now().Add(ttl)))
|
|
}
|
|
}
|
|
return &proxyResponse{answer: answer}
|
|
}
|
|
gotTransportFailure = true
|
|
ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "VPN DNS server %s failed", server)
|
|
}
|
|
|
|
// Explicit VPN DNS routes are authoritative for their suffix. If all
|
|
// routed servers fail at the transport layer while Windows is serving
|
|
// retained VPN DNS state, fail closed instead of leaking VPN/internal
|
|
// names to normal upstreams.
|
|
if gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, vpnServers) {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(),
|
|
"All VPN DNS servers had transport failures for %s; returning SERVFAIL while retained VPN DNS state is active", domain)
|
|
p.countFailedClientQuery(upstreams)
|
|
answer := new(dns.Msg)
|
|
answer.SetRcode(req.msg, dns.RcodeServerFailure)
|
|
return &proxyResponse{answer: answer}
|
|
}
|
|
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "All VPN DNS servers failed, falling back to normal upstreams")
|
|
}
|
|
}
|
|
|
|
// Domain-less VPN DNS fallback: when a query is going to upstream.os via a
|
|
// split-rule (matched policy) and we have VPN DNS servers with no associated
|
|
// domains, try those servers for this query. This handles cases like F5 VPN
|
|
// where the VPN doesn't advertise DNS search domains but its DNS servers
|
|
// know the internal zones referenced by split-rules (e.g., *.provisur.local).
|
|
// These servers are NOT used for general OS resolver queries to avoid
|
|
// polluting captive portal / DHCP flows.
|
|
if dnsIntercept && p.vpnDNS != nil && req.ufr.matched &&
|
|
len(upstreams) > 0 && upstreams[0] == upstreamOS &&
|
|
len(req.msg.Question) > 0 {
|
|
if dlServers := p.vpnDNS.DomainlessServers(); len(dlServers) > 0 {
|
|
domain := req.msg.Question[0].Name
|
|
ctrld.Log(ctx, mainLog.Load().Debug(),
|
|
"Split-rule query %s going to upstream.os, trying %d domain-less VPN DNS servers first: %v",
|
|
domain, len(dlServers), dlServers)
|
|
|
|
var gotDNSAnswer bool
|
|
var gotTransportFailure bool
|
|
for _, server := range dlServers {
|
|
upstreamCfg := p.vpnDNS.upstreamConfigFor(server)
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "Querying domain-less VPN DNS server: %s", server)
|
|
|
|
dnsResolver, err := ctrld.NewResolver(upstreamCfg)
|
|
if err != nil {
|
|
ctrld.Log(ctx, mainLog.Load().Error().Err(err), "failed to create domain-less VPN DNS resolver")
|
|
continue
|
|
}
|
|
resolveCtx, cancel := upstreamCfg.Context(ctx)
|
|
answer, err := dnsResolver.Resolve(resolveCtx, req.msg)
|
|
cancel()
|
|
if answer != nil {
|
|
gotDNSAnswer = true
|
|
p.vpnDNS.VPNDNSReachable()
|
|
}
|
|
if answer != nil && answer.Rcode == dns.RcodeSuccess {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(),
|
|
"Domain-less VPN DNS server %s answered %s successfully", server, domain)
|
|
return &proxyResponse{answer: answer}
|
|
}
|
|
if answer != nil {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(),
|
|
"Domain-less VPN DNS server %s returned %s for %s, trying next",
|
|
server, dns.RcodeToString[answer.Rcode], domain)
|
|
} else {
|
|
gotTransportFailure = true
|
|
ctrld.Log(ctx, mainLog.Load().Debug().Err(err),
|
|
"Domain-less VPN DNS server %s failed for %s", server, domain)
|
|
}
|
|
}
|
|
|
|
// If every domainless VPN DNS attempt failed before receiving a DNS
|
|
// packet while Windows is serving retained VPN DNS state, fail closed
|
|
// instead of asking LAN/public DNS about internal split-rule names and
|
|
// caching false negatives. Reachable negative DNS responses still fall
|
|
// through to the old OS fallback behavior below.
|
|
if !gotDNSAnswer && gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, dlServers) {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(),
|
|
"All domain-less VPN DNS servers had transport failures for %s; returning SERVFAIL while retained VPN DNS state is active", domain)
|
|
p.countFailedClientQuery(upstreams)
|
|
answer := new(dns.Msg)
|
|
answer.SetRcode(req.msg, dns.RcodeServerFailure)
|
|
return &proxyResponse{answer: answer}
|
|
}
|
|
|
|
ctrld.Log(ctx, mainLog.Load().Debug(),
|
|
"All domain-less VPN DNS servers failed for %s, falling back to OS resolver", domain)
|
|
}
|
|
}
|
|
|
|
resolve1 := func(upstream string, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) (*dns.Msg, error) {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "sending query to %s: %s", upstream, upstreamConfig.Name)
|
|
dnsResolver, err := ctrld.NewResolver(upstreamConfig)
|
|
if err != nil {
|
|
ctrld.Log(ctx, mainLog.Load().Error().Err(err), "failed to create resolver")
|
|
return nil, err
|
|
}
|
|
resolveCtx, cancel := upstreamConfig.Context(ctx)
|
|
defer cancel()
|
|
return dnsResolver.Resolve(resolveCtx, msg)
|
|
}
|
|
resolve := func(upstream string, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) *dns.Msg {
|
|
if upstreamConfig.UpstreamSendClientInfo() && req.ci != nil {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "including client info with the request")
|
|
ctx = context.WithValue(ctx, ctrld.ClientInfoCtxKey{}, req.ci)
|
|
}
|
|
answer, err := resolve1(upstream, upstreamConfig, msg)
|
|
|
|
// reset is not used here, because it also stops the failure count for
|
|
// one second, and the failures already in flight must still count.
|
|
if answer != nil {
|
|
p.um.noteSuccess(upstream)
|
|
return answer
|
|
}
|
|
|
|
// A failing minute must not push the state events out of the retained log.
|
|
// A transport error names the endpoint it failed to reach, which for a
|
|
// generated Internal Domain upstream is the organization's private
|
|
// resolver address. Report the classification through the sampler and
|
|
// keep the address-bearing error at debug.
|
|
if isInternalDomainUpstream(upstream) {
|
|
ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "failed to resolve query")
|
|
ctrld.Log(ctx, p.querySampler.event(sampleClassInternalDomain, upstream).Str("failure", internalDomainFailureReason(err)),
|
|
"failed to resolve query using an Internal Domain resolver")
|
|
} else {
|
|
ctrld.Log(ctx, p.querySampler.event(sampleClassResolveFailed, upstream).Err(err), "failed to resolve query")
|
|
}
|
|
|
|
// increase failure count when there is no answer
|
|
// rehardless of what kind of error we get
|
|
p.um.increaseFailureCount(upstream)
|
|
|
|
if err != nil {
|
|
// For timeout error (i.e: context deadline exceed), force re-bootstrapping.
|
|
var e net.Error
|
|
if errors.As(err, &e) && e.Timeout() {
|
|
upstreamConfig.ReBootstrap()
|
|
}
|
|
// For network error, turn ipv6 off if enabled.
|
|
if ctrld.HasIPv6() && (errUrlNetworkError(err) || errNetworkError(err)) {
|
|
ctrld.DisableIPv6()
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
for n, upstreamConfig := range upstreamConfigs {
|
|
if upstreamConfig == nil {
|
|
continue
|
|
}
|
|
logger := mainLog.Load().Debug().
|
|
Str("upstream", upstreamConfig.String()).
|
|
Str("query", req.msg.Question[0].Name).
|
|
Bool("is_ad_query", p.isAdDomainQuery(req.msg)).
|
|
Bool("is_lan_query", isLanOrPtrQuery)
|
|
|
|
if p.isLoop(upstreamConfig) {
|
|
ctrld.Log(ctx, logger, "DNS loop detected")
|
|
continue
|
|
}
|
|
answer := resolve(upstreams[n], upstreamConfig, req.msg)
|
|
if answer == nil {
|
|
if serveStaleCache && staleAnswer != nil {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "serving stale cached response")
|
|
p.health.countCacheHit()
|
|
now := time.Now()
|
|
setCachedAnswerTTL(staleAnswer, now, now.Add(staleTTL))
|
|
res.answer = staleAnswer
|
|
res.cached = true
|
|
return res
|
|
}
|
|
continue
|
|
}
|
|
// Reject an answer whose question does not match the request before it
|
|
// can be served or cached. A mismatched question means the upstream
|
|
// answered a different name/type than asked; caching it would poison
|
|
// the shared cache with wrong-domain records for the requested name.
|
|
// See github.com/Control-D-Inc/ctrld/issues/322.
|
|
if !sameQuestion(req.msg, answer) {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(),
|
|
"discarding answer from %s: question mismatch (asked %q, got %q)",
|
|
upstreams[n], questionString(req.msg), questionString(answer))
|
|
continue
|
|
}
|
|
// We are doing LAN/PTR lookup using private resolver, so always process next one.
|
|
// Except for the last, we want to send response instead of saying all upstream failed.
|
|
if answer.Rcode != dns.RcodeSuccess && isLanOrPtrQuery && n != len(upstreamConfigs)-1 {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "no response from %s, process to next upstream", upstreams[n])
|
|
continue
|
|
}
|
|
if answer.Rcode != dns.RcodeSuccess && len(upstreamConfigs) > 1 && containRcode(req.failoverRcodes, answer.Rcode) {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "failover rcode matched, process to next upstream")
|
|
continue
|
|
}
|
|
|
|
// set compression, as it is not set by default when unpacking
|
|
answer.Compress = true
|
|
|
|
if cacheable {
|
|
p.addCachedResponse(dnscache.NewKey(req.msg, upstreams[n]), answer)
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "add cached response")
|
|
}
|
|
hostname := ""
|
|
if req.ci != nil {
|
|
hostname = req.ci.Hostname
|
|
}
|
|
// DNS64 synthesis for IPv6-only networks without CLAT: applied to the
|
|
// policy-approved answer only, using the same upstream for the companion
|
|
// A resolution. No-op unless the network state requires it.
|
|
var synthesizedPrefix netip.Prefix
|
|
answer, synthesizedPrefix = p.maybeDNS64(ctx, req.msg, answer, func(aReq *dns.Msg) *dns.Msg {
|
|
key := dnscache.NewKey(aReq, upstreams[n])
|
|
if cacheable {
|
|
if cachedValue := p.cache.Get(key); cachedValue != nil {
|
|
now := time.Now()
|
|
if cachedValue.Expire.After(now) {
|
|
cached := cachedValue.Msg.Copy()
|
|
ctrld.SetCacheReply(cached, aReq, cached.Rcode)
|
|
setCachedAnswerTTL(cached, now, cachedValue.Expire)
|
|
return cached
|
|
}
|
|
}
|
|
}
|
|
resolved := resolve(upstreams[n], upstreamConfig, aReq)
|
|
if cacheable && resolved != nil && sameQuestion(aReq, resolved) {
|
|
p.addCachedResponse(key, resolved)
|
|
}
|
|
return resolved
|
|
})
|
|
if cacheable && synthesizedPrefix.IsValid() {
|
|
p.addCachedResponse(dns64CacheKey(req.msg, upstreams[n], synthesizedPrefix), answer)
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "dns64: add cached response variant")
|
|
}
|
|
ctrld.Log(ctx, mainLog.Load().Info(), "REPLY: %s -> %s (%s): %s", upstreams[n], req.ufr.srcAddr, hostname, dns.RcodeToString[answer.Rcode])
|
|
res.answer = answer
|
|
res.upstream = upstreamConfig.Endpoint
|
|
return res
|
|
}
|
|
ctrld.Log(ctx, p.querySampler.event(sampleClassAllEndpointsFailed, ""), "all %v endpoints failed", upstreams)
|
|
|
|
// An Internal Domain with explicit resolvers is served only by the
|
|
// configured resolvers. When all of them are unreachable the query fails
|
|
// here: the OS-resolver catch-all below would send a private name to a resolver
|
|
// that was not selected for it, and the recovery flow exists for the loss of
|
|
// general DNS, not for one unavailable internal server.
|
|
if internalDomainExplicitUpstreams(upstreams) {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolvers unreachable; not falling back")
|
|
} else if p.leakOnUpstreamFailure() {
|
|
if p.um.countHealthy(upstreams) == 0 {
|
|
p.recoveryCancelMu.Lock()
|
|
if p.recoveryCancel == nil {
|
|
var reason RecoveryReason
|
|
if upstreams[0] == upstreamOS {
|
|
reason = RecoveryReasonOSFailure
|
|
|
|
} else {
|
|
reason = RecoveryReasonRegularFailure
|
|
}
|
|
mainLog.Load().Debug().Msgf("No healthy upstreams, triggering recovery with reason: %v", reason)
|
|
go p.handleRecovery(reason)
|
|
} else {
|
|
mainLog.Load().Debug().Msg("Recovery already in progress; skipping duplicate trigger from down detection")
|
|
}
|
|
p.recoveryCancelMu.Unlock()
|
|
} else {
|
|
mainLog.Load().Debug().Msg("One upstream is down but at least one is healthy; skipping recovery trigger")
|
|
}
|
|
|
|
// attempt query to OS resolver while as a retry catch all
|
|
// we dont want this to happen if leakOnUpstreamFailure is false
|
|
if upstreams[0] != upstreamOS {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "attempting query to OS resolver as a retry catch all")
|
|
answer := resolve(upstreamOS, osUpstreamConfig, req.msg)
|
|
if answer != nil {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "OS resolver retry query successful")
|
|
res.answer = answer
|
|
res.upstream = osUpstreamConfig.Endpoint
|
|
return res
|
|
}
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "OS resolver retry query failed")
|
|
}
|
|
}
|
|
|
|
p.countFailedClientQuery(upstreams)
|
|
answer := new(dns.Msg)
|
|
answer.SetRcode(req.msg, dns.RcodeServerFailure)
|
|
res.answer = answer
|
|
return res
|
|
}
|
|
|
|
// countFailedClientQuery grades one client query that ended without an answer.
|
|
// A query that only Internal Domain resolvers serve stays out of the grade: an
|
|
// endpoint away from the organization network never reaches them, and that is
|
|
// not an outage of the query path.
|
|
func (p *prog) countFailedClientQuery(upstreams []string) {
|
|
if internalDomainExplicitUpstreams(upstreams) {
|
|
return
|
|
}
|
|
p.health.countFailedQuery()
|
|
}
|
|
|
|
func (p *prog) upstreamsAndUpstreamConfigForPtr(upstreams []string, upstreamConfigs []*ctrld.UpstreamConfig) ([]string, []*ctrld.UpstreamConfig) {
|
|
if len(p.localUpstreams) > 0 {
|
|
tmp := make([]string, 0, len(p.localUpstreams)+len(upstreams))
|
|
tmp = append(tmp, p.localUpstreams...)
|
|
tmp = append(tmp, upstreams...)
|
|
return tmp, p.upstreamConfigsFromUpstreamNumbers(tmp)
|
|
}
|
|
return append([]string{upstreamOS}, upstreams...), append([]*ctrld.UpstreamConfig{privateUpstreamConfig}, upstreamConfigs...)
|
|
}
|
|
|
|
func (p *prog) upstreamConfigsFromUpstreamNumbers(upstreams []string) []*ctrld.UpstreamConfig {
|
|
upstreamConfigs := make([]*ctrld.UpstreamConfig, 0, len(upstreams))
|
|
for _, upstream := range upstreams {
|
|
upstreamNum := strings.TrimPrefix(upstream, upstreamPrefix)
|
|
upstreamConfigs = append(upstreamConfigs, p.cfg.Upstream[upstreamNum])
|
|
}
|
|
return upstreamConfigs
|
|
}
|
|
|
|
func (p *prog) isAdDomainQuery(msg *dns.Msg) bool {
|
|
if p.adDomain == "" {
|
|
return false
|
|
}
|
|
cDomainName := canonicalName(msg.Question[0].Name)
|
|
return dns.IsSubDomain(p.adDomain, cDomainName)
|
|
}
|
|
|
|
// canonicalName returns canonical name from FQDN with "." trimmed.
|
|
func canonicalName(fqdn string) string {
|
|
q := strings.TrimSpace(fqdn)
|
|
q = strings.TrimSuffix(q, ".")
|
|
// https://datatracker.ietf.org/doc/html/rfc4343
|
|
q = strings.ToLower(q)
|
|
|
|
return q
|
|
}
|
|
|
|
// wildcardMatches reports whether string str matches the wildcard pattern in case-insensitive manner.
|
|
func wildcardMatches(wildcard, str string) bool {
|
|
// Wildcard match.
|
|
wildCardParts := strings.Split(strings.ToLower(wildcard), "*")
|
|
if len(wildCardParts) != 2 {
|
|
return false
|
|
}
|
|
|
|
str = strings.ToLower(str)
|
|
switch {
|
|
case len(wildCardParts[0]) > 0 && len(wildCardParts[1]) > 0:
|
|
// Domain must match both prefix and suffix.
|
|
return strings.HasPrefix(str, wildCardParts[0]) && strings.HasSuffix(str, wildCardParts[1])
|
|
|
|
case len(wildCardParts[1]) > 0:
|
|
// Only suffix must match.
|
|
return strings.HasSuffix(str, wildCardParts[1])
|
|
|
|
case len(wildCardParts[0]) > 0:
|
|
// Only prefix must match.
|
|
return strings.HasPrefix(str, wildCardParts[0])
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func fmtRemoteToLocal(listenerNum, hostname, remote string) string {
|
|
return fmt.Sprintf("%s (%s) -> listener.%s", remote, hostname, listenerNum)
|
|
}
|
|
|
|
func requestID() string {
|
|
b := make([]byte, 3) // 6 chars
|
|
if _, err := rand.Read(b); err != nil {
|
|
panic(err)
|
|
}
|
|
return hex.EncodeToString(b)
|
|
}
|
|
|
|
func containRcode(rcodes []int, rcode int) bool {
|
|
for i := range rcodes {
|
|
if rcodes[i] == rcode {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// sameQuestion reports whether the upstream answer echoes the request's
|
|
// question. A well-behaved resolver always copies the question section from
|
|
// the query (RFC 1035 section 4.1.2); names are compared case-insensitively
|
|
// because DNS names are case-insensitive. A mismatch means the upstream
|
|
// answered a different name/type than asked - malformed or malicious - and the
|
|
// answer must not be served or cached, or it would poison the shared cache with
|
|
// wrong-domain records. See github.com/Control-D-Inc/ctrld/issues/322.
|
|
func sameQuestion(req, answer *dns.Msg) bool {
|
|
if req == nil || answer == nil {
|
|
return false
|
|
}
|
|
if len(req.Question) == 0 || len(answer.Question) == 0 {
|
|
return false
|
|
}
|
|
rq, aq := req.Question[0], answer.Question[0]
|
|
return rq.Qtype == aq.Qtype && rq.Qclass == aq.Qclass && strings.EqualFold(rq.Name, aq.Name)
|
|
}
|
|
|
|
// questionString renders a message's first question as "name/type" for logging.
|
|
func questionString(msg *dns.Msg) string {
|
|
if msg == nil || len(msg.Question) == 0 {
|
|
return "<none>"
|
|
}
|
|
q := msg.Question[0]
|
|
return q.Name + "/" + dns.TypeToString[q.Qtype]
|
|
}
|
|
|
|
func setCachedAnswerTTL(answer *dns.Msg, now, expiredTime time.Time) {
|
|
ttlSecs := expiredTime.Sub(now).Seconds()
|
|
if ttlSecs < 0 {
|
|
return
|
|
}
|
|
|
|
ttl := uint32(ttlSecs)
|
|
for _, rr := range answer.Answer {
|
|
rr.Header().Ttl = ttl
|
|
}
|
|
for _, rr := range answer.Ns {
|
|
rr.Header().Ttl = ttl
|
|
}
|
|
for _, rr := range answer.Extra {
|
|
if rr.Header().Rrtype != dns.TypeOPT {
|
|
rr.Header().Ttl = ttl
|
|
}
|
|
}
|
|
}
|
|
|
|
func ttlFromMsg(msg *dns.Msg) uint32 {
|
|
for _, rr := range msg.Answer {
|
|
return rr.Header().Ttl
|
|
}
|
|
for _, rr := range msg.Ns {
|
|
return rr.Header().Ttl
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func needLocalIPv6Listener(interceptMode string) bool {
|
|
if !ctrldnet.SupportsIPv6ListenLocal() {
|
|
mainLog.Load().Debug().Msg("IPv6 listener: not needed — SupportsIPv6ListenLocal() is false")
|
|
return false
|
|
}
|
|
// On Windows, there's no easy way for disabling/removing IPv6 DNS resolver, so we check whether we can
|
|
// listen on ::1, then spawn a listener for receiving DNS requests.
|
|
if runtime.GOOS == "windows" {
|
|
mainLog.Load().Debug().Msg("IPv6 listener: enabled (Windows)")
|
|
return true
|
|
}
|
|
// macOS: IPv6 DNS is blocked at the pf level (not intercepted). The [::1] listener
|
|
// is not needed — macOS falls back to IPv4 DNS automatically. See #507 and
|
|
// docs/pf-dns-intercept.md for why IPv6 interception on macOS is not feasible
|
|
// (sendmsg EINVAL from ::1 to global unicast, nat-on-lo0 doesn't fire for route-to).
|
|
if runtime.GOOS == "darwin" {
|
|
mainLog.Load().Debug().Msg("IPv6 listener: not needed (macOS — IPv6 DNS blocked at pf, fallback to IPv4)")
|
|
return false
|
|
}
|
|
mainLog.Load().Debug().Str("os", runtime.GOOS).Str("interceptMode", interceptMode).Msg("IPv6 listener: not needed")
|
|
return false
|
|
}
|
|
|
|
// ipAndMacFromMsg extracts IP and MAC information included in a DNS message, if any.
|
|
func ipAndMacFromMsg(msg *dns.Msg) (string, string) {
|
|
ip, mac := "", ""
|
|
if opt := msg.IsEdns0(); opt != nil {
|
|
for _, s := range opt.Option {
|
|
switch e := s.(type) {
|
|
case *dns.EDNS0_LOCAL:
|
|
if e.Code == EDNS0_OPTION_MAC {
|
|
mac = net.HardwareAddr(e.Data).String()
|
|
}
|
|
case *dns.EDNS0_SUBNET:
|
|
if len(e.Address) > 0 && !e.Address.IsLoopback() {
|
|
ip = e.Address.String()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return ip, mac
|
|
}
|
|
|
|
// stripClientSubnet removes EDNS0_SUBNET from DNS message if the IP is RFC1918 or loopback address,
|
|
// passing them to upstream is pointless, these cannot be used by anything on the WAN.
|
|
func stripClientSubnet(msg *dns.Msg) {
|
|
if opt := msg.IsEdns0(); opt != nil {
|
|
opts := make([]dns.EDNS0, 0, len(opt.Option))
|
|
for _, s := range opt.Option {
|
|
if e, ok := s.(*dns.EDNS0_SUBNET); ok && (e.Address.IsPrivate() || e.Address.IsLoopback()) {
|
|
continue
|
|
}
|
|
opts = append(opts, s)
|
|
}
|
|
if len(opts) != len(opt.Option) {
|
|
opt.Option = opts
|
|
}
|
|
}
|
|
}
|
|
|
|
func spoofRemoteAddr(addr net.Addr, ci *ctrld.ClientInfo) net.Addr {
|
|
if ci != nil && ci.IP != "" {
|
|
switch addr := addr.(type) {
|
|
case *net.UDPAddr:
|
|
udpAddr := &net.UDPAddr{
|
|
IP: net.ParseIP(ci.IP),
|
|
Port: addr.Port,
|
|
Zone: addr.Zone,
|
|
}
|
|
return udpAddr
|
|
case *net.TCPAddr:
|
|
udpAddr := &net.TCPAddr{
|
|
IP: net.ParseIP(ci.IP),
|
|
Port: addr.Port,
|
|
Zone: addr.Zone,
|
|
}
|
|
return udpAddr
|
|
}
|
|
}
|
|
return addr
|
|
}
|
|
|
|
// runDNSServer starts a DNS server for given address and network,
|
|
// with the given handler. It ensures the server has started listening.
|
|
// Any error will be reported to the caller via returned channel.
|
|
//
|
|
// It's the caller responsibility to call Shutdown to close the server.
|
|
func runDNSServer(addr, network string, handler dns.Handler) (*dns.Server, <-chan error) {
|
|
s := &dns.Server{
|
|
Addr: addr,
|
|
Net: network,
|
|
Handler: handler,
|
|
}
|
|
|
|
startedCh := make(chan struct{})
|
|
s.NotifyStartedFunc = func() { sync.OnceFunc(func() { close(startedCh) })() }
|
|
|
|
errCh := make(chan error)
|
|
go func() {
|
|
defer close(errCh)
|
|
if err := s.ListenAndServe(); err != nil {
|
|
s.NotifyStartedFunc()
|
|
mainLog.Load().Error().Err(err).Msgf("could not listen and serve on: %s", s.Addr)
|
|
errCh <- err
|
|
}
|
|
}()
|
|
<-startedCh
|
|
return s, errCh
|
|
}
|
|
|
|
func (p *prog) getClientInfo(remoteIP string, msg *dns.Msg) *ctrld.ClientInfo {
|
|
ci := &ctrld.ClientInfo{}
|
|
if p.appCallback != nil {
|
|
ci.IP = p.appCallback.LanIp()
|
|
ci.Mac = p.appCallback.MacAddress()
|
|
ci.Hostname = p.appCallback.HostName()
|
|
ci.Self = true
|
|
return ci
|
|
}
|
|
ci.IP, ci.Mac = ipAndMacFromMsg(msg)
|
|
switch {
|
|
case ci.IP != "" && ci.Mac != "":
|
|
// Nothing to do.
|
|
case ci.IP == "" && ci.Mac != "":
|
|
// Have MAC, no IP.
|
|
ci.IP = p.ciTable.LookupIP(ci.Mac)
|
|
case ci.IP == "" && ci.Mac == "":
|
|
// Have nothing, use remote IP then lookup MAC.
|
|
ci.IP = remoteIP
|
|
fallthrough
|
|
case ci.IP != "" && ci.Mac == "":
|
|
// Have IP, no MAC.
|
|
ci.Mac = p.ciTable.LookupMac(ci.IP)
|
|
}
|
|
|
|
// If MAC is still empty here, that mean the requests are made from virtual interface,
|
|
// like VPN/Wireguard clients, so we use ci.IP as hostname to distinguish those clients.
|
|
if ci.Mac == "" {
|
|
if hostname := p.ciTable.LookupHostname(ci.IP, ""); hostname != "" {
|
|
ci.Hostname = hostname
|
|
} else {
|
|
// Only use IP as hostname for IPv4 clients.
|
|
// For Android devices, when it joins the network, it uses ctrld to resolve
|
|
// its private DNS once and never reaches ctrld again. For each time, it uses
|
|
// a different IPv6 address, which causes hundreds/thousands different client
|
|
// IDs created for the same device, which is pointless.
|
|
//
|
|
// TODO(cuonglm): investigate whether this can be a false positive for other clients?
|
|
if !ctrldnet.IsIPv6(ci.IP) {
|
|
ci.Hostname = ci.IP
|
|
p.ciTable.StoreVPNClient(ci)
|
|
}
|
|
}
|
|
} else {
|
|
ci.Hostname = p.ciTable.LookupHostname(ci.IP, ci.Mac)
|
|
}
|
|
|
|
if ci.IP == "" {
|
|
mainLog.Load().Debug().Msgf("client info entry with empty IP address: %v", ci)
|
|
} else {
|
|
ci.Self = p.queryFromSelf(ci.IP)
|
|
}
|
|
|
|
// In DNS intercept mode, ALL queries are from the local machine — pf/WFP
|
|
// intercepts outbound DNS and redirects to ctrld. The source IP may be a
|
|
// virtual interface (Tailscale, VPN) that has no ARP/MAC entry, causing
|
|
// missing x-cd-mac, x-cd-host, and x-cd-os headers. Force Self=true and
|
|
// populate from the primary physical interface info.
|
|
if dnsIntercept && !ci.Self {
|
|
ci.Self = true
|
|
}
|
|
|
|
// If this is a query from self, but ci.IP is not loopback IP,
|
|
// try using hostname mapping for lookback IP if presents.
|
|
if ci.Self {
|
|
if name := p.ciTable.LocalHostname(); name != "" {
|
|
ci.Hostname = name
|
|
}
|
|
// If MAC is still empty (e.g., query arrived via virtual interface IP
|
|
// like Tailscale), fall back to the loopback MAC mapping which addSelf()
|
|
// populates from the primary physical interface.
|
|
if ci.Mac == "" {
|
|
if mac := p.ciTable.LookupMac("127.0.0.1"); mac != "" {
|
|
ci.Mac = mac
|
|
}
|
|
}
|
|
}
|
|
p.spoofLoopbackIpInClientInfo(ci)
|
|
return ci
|
|
}
|
|
|
|
// spoofLoopbackIpInClientInfo replaces loopback IPs in client info.
|
|
//
|
|
// - Preference IPv4.
|
|
// - Preference RFC1918.
|
|
func (p *prog) spoofLoopbackIpInClientInfo(ci *ctrld.ClientInfo) {
|
|
if ip := net.ParseIP(ci.IP); ip == nil || !ip.IsLoopback() {
|
|
return
|
|
}
|
|
if ip := p.ciTable.LookupRFC1918IPv4(ci.Mac); ip != "" {
|
|
ci.IP = ip
|
|
}
|
|
}
|
|
|
|
// doSelfUninstall performs self-uninstall if these condition met:
|
|
//
|
|
// - There is only 1 ControlD upstream in-use.
|
|
// - Number of refused queries seen so far equals to selfUninstallMaxQueries.
|
|
// - The cdUID is deleted.
|
|
func (p *prog) doSelfUninstall(answer *dns.Msg) {
|
|
if !p.canSelfUninstall.Load() || answer == nil || answer.Rcode != dns.RcodeRefused {
|
|
return
|
|
}
|
|
|
|
p.selfUninstallMu.Lock()
|
|
defer p.selfUninstallMu.Unlock()
|
|
if p.checkingSelfUninstall {
|
|
return
|
|
}
|
|
|
|
logger := mainLog.Load().With().Str("mode", "self-uninstall").Logger()
|
|
if p.refusedQueryCount > selfUninstallMaxQueries {
|
|
p.checkingSelfUninstall = true
|
|
|
|
req := &controld.ResolverConfigRequest{
|
|
RawUID: cdUID,
|
|
Version: rootCmd.Version,
|
|
Metadata: ctrld.SystemMetadataRuntime(context.Background()),
|
|
}
|
|
_, err := controld.FetchResolverConfig(context.Background(), req, cdDev)
|
|
logger.Debug().Msg("maximum number of refused queries reached, checking device status")
|
|
selfUninstallCheck(err, p, logger)
|
|
|
|
if err != nil {
|
|
logger.Warn().Err(err).Msg("could not fetch resolver config")
|
|
}
|
|
// Cool-of period to prevent abusing the API.
|
|
go p.selfUninstallCoolOfPeriod()
|
|
return
|
|
}
|
|
p.refusedQueryCount++
|
|
}
|
|
|
|
// selfUninstallCoolOfPeriod waits for 30 minutes before
|
|
// calling API again for checking ControlD device status.
|
|
func (p *prog) selfUninstallCoolOfPeriod() {
|
|
t := time.NewTimer(time.Minute * 30)
|
|
defer t.Stop()
|
|
<-t.C
|
|
p.selfUninstallMu.Lock()
|
|
p.checkingSelfUninstall = false
|
|
p.refusedQueryCount = 0
|
|
p.selfUninstallMu.Unlock()
|
|
}
|
|
|
|
// forceFetchingAPI sends signal to force syncing API config if run in cd mode,
|
|
// and the domain == "cdUID.verify.controld.com"
|
|
func (p *prog) forceFetchingAPI(domain string) {
|
|
if cdUID == "" {
|
|
return
|
|
}
|
|
resolverID, parent, _ := strings.Cut(domain, ".")
|
|
if resolverID != cdUID {
|
|
return
|
|
}
|
|
switch {
|
|
case cdDev && parent == "verify.controld.dev":
|
|
// match ControlD dev
|
|
case parent == "verify.controld.com":
|
|
// match ControlD
|
|
default:
|
|
return
|
|
}
|
|
_ = p.apiForceReloadGroup.DoChan("force_sync_api", func() (interface{}, error) {
|
|
p.apiForceReloadCh <- struct{}{}
|
|
// Wait here to prevent abusing API if we are flooded.
|
|
time.Sleep(timeDurationOrDefault(p.cfg.Service.ForceRefetchWaitTime, 30) * time.Second)
|
|
return nil, nil
|
|
})
|
|
}
|
|
|
|
// timeDurationOrDefault returns time duration value from n if not nil.
|
|
// Otherwise, it returns time duration value defaultN.
|
|
func timeDurationOrDefault(n *int, defaultN int) time.Duration {
|
|
if n != nil && *n > 0 {
|
|
return time.Duration(*n)
|
|
}
|
|
return time.Duration(defaultN)
|
|
}
|
|
|
|
// queryFromSelf reports whether the input IP is from device running ctrld.
|
|
func (p *prog) queryFromSelf(ip string) bool {
|
|
if val, ok := p.queryFromSelfMap.Load(ip); ok {
|
|
return val.(bool)
|
|
}
|
|
netIP, err := netip.ParseAddr(ip)
|
|
if err != nil {
|
|
mainLog.Load().Debug().Err(err).Msgf("could not parse IP: %q", ip)
|
|
return false
|
|
}
|
|
|
|
regularIPs, loopbackIPs, err := netmon.LocalAddresses()
|
|
if err != nil {
|
|
mainLog.Load().Warn().Err(err).Msg("could not get local addresses")
|
|
return false
|
|
}
|
|
for _, localIP := range slices.Concat(regularIPs, loopbackIPs) {
|
|
if localIP.Compare(netIP) == 0 {
|
|
p.queryFromSelfMap.Store(ip, true)
|
|
return true
|
|
}
|
|
}
|
|
p.queryFromSelfMap.Store(ip, false)
|
|
return false
|
|
}
|
|
|
|
// needRFC1918Listeners reports whether ctrld need to spawn listener for RFC 1918 addresses.
|
|
// This is helpful for non-desktop platforms to receive queries from LAN clients.
|
|
func needRFC1918Listeners(lc *ctrld.ListenerConfig) bool {
|
|
return rfc1918 && lc.IP == "127.0.0.1" && lc.Port == 53
|
|
}
|
|
|
|
// ipFromARPA parses a FQDN arpa domain and return the IP address if valid.
|
|
func ipFromARPA(arpa string) net.IP {
|
|
if arpa, ok := strings.CutSuffix(arpa, ".in-addr.arpa."); ok {
|
|
if ptrIP := net.ParseIP(arpa); ptrIP != nil {
|
|
return net.IP{ptrIP[15], ptrIP[14], ptrIP[13], ptrIP[12]}
|
|
}
|
|
}
|
|
if arpa, ok := strings.CutSuffix(arpa, ".ip6.arpa."); ok {
|
|
l := net.IPv6len * 2
|
|
base := 16
|
|
ip := make(net.IP, net.IPv6len)
|
|
for i := 0; i < l && arpa != ""; i++ {
|
|
idx := strings.LastIndexByte(arpa, '.')
|
|
off := idx + 1
|
|
if idx == -1 {
|
|
idx = 0
|
|
off = 0
|
|
} else if idx == len(arpa)-1 {
|
|
return nil
|
|
}
|
|
n, err := strconv.ParseUint(arpa[off:], base, 8)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
b := byte(n)
|
|
ii := i / 2
|
|
if i&1 == 1 {
|
|
b |= ip[ii] << 4
|
|
}
|
|
ip[ii] = b
|
|
arpa = arpa[:idx]
|
|
}
|
|
return ip
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// isPrivatePtrLookup reports whether DNS message is an PTR query for LAN/CGNAT network.
|
|
func isPrivatePtrLookup(m *dns.Msg) bool {
|
|
if m == nil || len(m.Question) == 0 {
|
|
return false
|
|
}
|
|
q := m.Question[0]
|
|
if ip := ipFromARPA(q.Name); ip != nil {
|
|
if addr, ok := netip.AddrFromSlice(ip); ok {
|
|
return addr.IsPrivate() ||
|
|
addr.IsLoopback() ||
|
|
addr.IsLinkLocalUnicast() ||
|
|
tsaddr.CGNATRange().Contains(addr) ||
|
|
isServiceContinuityAddr(addr)
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// isLanHostnameQuery reports whether DNS message is an A/AAAA query with LAN hostname.
|
|
func isLanHostnameQuery(m *dns.Msg) bool {
|
|
if m == nil || len(m.Question) == 0 {
|
|
return false
|
|
}
|
|
q := m.Question[0]
|
|
switch q.Qtype {
|
|
case dns.TypeA, dns.TypeAAAA:
|
|
default:
|
|
return false
|
|
}
|
|
return isLanHostname(q.Name)
|
|
}
|
|
|
|
// isSrvLanLookup reports whether DNS message is an SRV query of a LAN hostname.
|
|
func isSrvLanLookup(m *dns.Msg) bool {
|
|
if m == nil || len(m.Question) == 0 {
|
|
return false
|
|
}
|
|
q := m.Question[0]
|
|
return q.Qtype == dns.TypeSRV && isLanHostname(q.Name)
|
|
}
|
|
|
|
// isLanHostname reports whether name is a LAN hostname.
|
|
func isLanHostname(name string) bool {
|
|
name = strings.TrimSuffix(name, ".")
|
|
return !strings.Contains(name, ".") ||
|
|
strings.HasSuffix(name, ".domain") ||
|
|
strings.HasSuffix(name, ".lan") ||
|
|
strings.HasSuffix(name, ".local")
|
|
}
|
|
|
|
// ipv4ServiceContinuityPrefix is the RFC 7335 IPv4 Service Continuity Prefix
|
|
// (192.0.0.0/29), used by the CLAT in 464XLAT/DS-Lite transition setups. On such
|
|
// networks (common on IPv6-only cellular carriers and iPhone hotspots) the local
|
|
// machine's DNS queries reach ctrld with a source in this range (e.g. 192.0.0.2),
|
|
// so they must be treated as local, not WAN. Go's netip.IsPrivate does not cover
|
|
// this range — the same reason the CGNAT range is special-cased below. See #552.
|
|
var ipv4ServiceContinuityPrefix = netip.MustParsePrefix("192.0.0.0/29")
|
|
|
|
// isServiceContinuityAddr reports whether ip is in the RFC 7335 IPv4 Service
|
|
// Continuity Prefix (464XLAT/DS-Lite CLAT).
|
|
func isServiceContinuityAddr(ip netip.Addr) bool {
|
|
return ipv4ServiceContinuityPrefix.Contains(ip)
|
|
}
|
|
|
|
// isWanClient reports whether the input is a WAN address.
|
|
func isWanClient(na net.Addr) bool {
|
|
var ip netip.Addr
|
|
if ap, err := netip.ParseAddrPort(na.String()); err == nil {
|
|
ip = ap.Addr()
|
|
}
|
|
return !ip.IsLoopback() &&
|
|
!ip.IsPrivate() &&
|
|
!ip.IsLinkLocalUnicast() &&
|
|
!ip.IsLinkLocalMulticast() &&
|
|
!tsaddr.CGNATRange().Contains(ip) &&
|
|
!isServiceContinuityAddr(ip)
|
|
}
|
|
|
|
// isIPv6LoopbackListener reports whether the listener address is [::1].
|
|
// The [::1] listener only serves locally-redirected traffic (via pf on macOS
|
|
// or system DNS on Windows), so queries arriving on it are always from this
|
|
// machine — even when the source IP is a global IPv6 address (pf preserves the
|
|
// original source IP during rdr).
|
|
func isIPv6LoopbackListener(na net.Addr) bool {
|
|
if ap, err := netip.ParseAddrPort(na.String()); err == nil {
|
|
return ap.Addr() == netip.IPv6Loopback()
|
|
}
|
|
return false
|
|
}
|
|
|
|
// resolveInternalDomainTestQuery resolves internal test domain query, returning the answer to the caller.
|
|
func resolveInternalDomainTestQuery(ctx context.Context, domain string, m *dns.Msg) *dns.Msg {
|
|
ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain test query")
|
|
|
|
q := m.Question[0]
|
|
answer := new(dns.Msg)
|
|
rrStr := fmt.Sprintf("%s A %s", domain, net.IPv4zero)
|
|
if q.Qtype == dns.TypeAAAA {
|
|
rrStr = fmt.Sprintf("%s AAAA %s", domain, net.IPv6zero)
|
|
}
|
|
rr, err := dns.NewRR(rrStr)
|
|
if err == nil {
|
|
answer.Answer = append(answer.Answer, rr)
|
|
}
|
|
answer.SetReply(m)
|
|
return answer
|
|
}
|
|
|
|
// FlushDNSCache flushes the DNS cache on macOS.
|
|
func FlushDNSCache() error {
|
|
// if not macOS, return
|
|
if runtime.GOOS != "darwin" {
|
|
return nil
|
|
}
|
|
|
|
// Flush the DNS cache via mDNSResponder.
|
|
// This is typically needed on modern macOS systems.
|
|
if out, err := exec.Command("killall", "-HUP", "mDNSResponder").CombinedOutput(); err != nil {
|
|
return fmt.Errorf("failed to flush mDNSResponder: %w, output: %s", err, string(out))
|
|
}
|
|
|
|
// Optionally, flush the directory services cache.
|
|
if out, err := exec.Command("dscacheutil", "-flushcache").CombinedOutput(); err != nil {
|
|
return fmt.Errorf("failed to flush dscacheutil: %w, output: %s", err, string(out))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// monitorNetworkChanges starts monitoring for network interface changes
|
|
func (p *prog) monitorNetworkChanges() error {
|
|
mon, err := newNetworkChangeMonitorFn(func(format string, args ...any) {
|
|
// Always fetch the latest logger (and inject the prefix)
|
|
mainLog.Load().Printf("netmon: "+format, args...)
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("creating network monitor: %w", err)
|
|
}
|
|
|
|
mon.RegisterChangeCallback(func(delta *netmon.ChangeDelta) {
|
|
p.handleNetworkChange(delta, mon.IsMajorChangeFrom(delta.Old, delta.New))
|
|
})
|
|
|
|
mon.Start()
|
|
mainLog.Load().Debug().Msg("Network monitor started")
|
|
return nil
|
|
}
|
|
|
|
// handleNetworkChange is the network monitor callback, kept separate for synthetic deltas.
|
|
func (p *prog) handleNetworkChange(delta *netmon.ChangeDelta, isMajorChange bool) {
|
|
// netmon v1.74.0 dispatches callbacks concurrently and caches only major
|
|
// snapshots. Minor events refer to that cache through Old. The synthetic
|
|
// delta.Major flag after a time jump does not change the cache.
|
|
p.networkSourceMu.Lock()
|
|
currentState := networkChangeCurrentStateFn(delta)
|
|
current := networkSnapshotCurrent(delta, isMajorChange, currentState)
|
|
if delta.TimeJumped {
|
|
// A late callback still tells that the host woke, but the network of
|
|
// the wake is the one that netmon holds now.
|
|
wakeState := delta.New
|
|
if !current {
|
|
wakeState = currentState
|
|
}
|
|
p.noteHostWoke("netmon", 0, wakeState)
|
|
}
|
|
// Only a current callback becomes the baseline of the next diff. A late
|
|
// callback that a newer snapshot replaced would hide the changes between
|
|
// the callback before it and the callback after it.
|
|
before := p.deltaBeforeState(delta, currentState, current, isMajorChange)
|
|
changes := diffNetworkDelta(before, delta.New)
|
|
if current && hasAddOrRemove(changes) {
|
|
refreshInterfaceMeta()
|
|
}
|
|
// The noise test runs before any other work, because the point of the class
|
|
// is that a storm starts no pfctl and no scutil process.
|
|
if current && noiseDelta(before, delta.New, delta.TimeJumped, changes) {
|
|
p.networkSourceMu.Unlock()
|
|
p.noteNoiseDelta(changes)
|
|
return
|
|
}
|
|
// A superseded snapshot never becomes a transition, so it takes no ID and
|
|
// reports zero.
|
|
var transitionID uint64
|
|
changed := false
|
|
changedInterface := ""
|
|
sourceSnapshotAvailable := false
|
|
before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6()
|
|
after4, after6 := before4, before6
|
|
// The newer callback of this epoch carries the same interface changes, so
|
|
// this outcome reports at debug and stays out of the journal.
|
|
outcome := transitionOutcomeSnapshotSuperseded
|
|
defer func() {
|
|
stateBefore, stateAfter := networkStateOrEmpty(before), networkStateOrEmpty(delta.New)
|
|
networkTransitionEvent(outcome).Uint64("transition_id", transitionID).
|
|
Bool("is_major_change", isMajorChange).Bool("changed", changed).
|
|
Str("interface", changedInterface).
|
|
Strs("changed_interfaces", changedInterfaceNames(changes)).
|
|
Bool("time_jumped", delta.TimeJumped).
|
|
Str("default_route_before", stateBefore.DefaultRouteInterface).
|
|
Str("default_route_after", stateAfter.DefaultRouteInterface).
|
|
Bool("have_v4_before", stateBefore.HaveV4).Bool("have_v4_after", stateAfter.HaveV4).
|
|
Bool("have_v6_before", stateBefore.HaveV6).Bool("have_v6_after", stateAfter.HaveV6).
|
|
Str("source_ipv4_before", before4.String()).Str("source_ipv6_before", before6.String()).
|
|
Str("source_ipv4_after", after4.String()).
|
|
Str("source_ipv6_after", after6.String()).
|
|
Bool("source_snapshot_available", sourceSnapshotAvailable).Str("outcome", outcome).
|
|
Msg(networkTransitionMessage)
|
|
}()
|
|
if !current {
|
|
p.networkSourceMu.Unlock()
|
|
return
|
|
}
|
|
p.flushNoiseSummary()
|
|
outcome = transitionOutcomeIgnored
|
|
transitionID = p.networkTransitionGen.Add(1)
|
|
sourceState := delta.New
|
|
// A late major callback and reordered minor callbacks share one cache
|
|
// epoch. For these cases, source validity comes from the OS, not ordering.
|
|
if delta.Monitor != nil && (!isMajorChange || (p.networkSourceEpoch == currentState && p.networkSourceState != currentState)) {
|
|
fresh, err := readNetworkSourceStateFn()
|
|
if err != nil || fresh == nil {
|
|
if !p.networkSourceReadFailed {
|
|
mainLog.Load().Warn().Uint64("transition_id", transitionID).
|
|
Msg("Network source snapshot unavailable; source writes deferred")
|
|
}
|
|
p.networkSourceReadFailed = true
|
|
sourceState = nil
|
|
} else {
|
|
snapshot := *fresh
|
|
snapshot.DefaultRouteInterface = currentState.DefaultRouteInterface
|
|
sourceState = &snapshot
|
|
if p.networkSourceReadFailed {
|
|
mainLog.Load().Warn().Uint64("transition_id", transitionID).Msg("Network source snapshot available again")
|
|
}
|
|
p.networkSourceReadFailed = false
|
|
}
|
|
}
|
|
p.networkSourceState, p.networkSourceEpoch = sourceState, currentState
|
|
sourceSnapshotAvailable = sourceState != nil
|
|
validateDefaultLocalIPsFromDelta(sourceState, transitionID)
|
|
after4, after6 = ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6()
|
|
// The network of this callback belongs to the epoch that the lines above
|
|
// store. A store after the unlock can overwrite the network of a newer
|
|
// callback. The header render reads the hardware ports, so it runs after
|
|
// the unlock and starts no process under the lock.
|
|
p.lastNetworkState.Store(currentState)
|
|
// HasIPv6 reads the flag that this callback stores, so one monitor serves
|
|
// the whole process.
|
|
ctrld.SetIPv6Available(currentState.HaveV6)
|
|
p.networkSourceMu.Unlock()
|
|
p.refreshLogHeader()
|
|
describeInterfaceChanges(changes, interfaceMetaFor)
|
|
logInterfaceChanges(transitionID, changes)
|
|
|
|
p.handleDNS64NetworkChange(delta, isMajorChange)
|
|
validIfaces := networkChangeValidInterfacesFn()
|
|
|
|
activeInterfaceExists := false
|
|
var changeIPs []netip.Prefix
|
|
// Check each valid interface for changes
|
|
for ifaceName := range validIfaces {
|
|
oldIface, oldExists := delta.Old.Interface[ifaceName]
|
|
newIface, newExists := delta.New.Interface[ifaceName]
|
|
if !newExists {
|
|
continue
|
|
}
|
|
|
|
oldIPs := delta.Old.InterfaceIPs[ifaceName]
|
|
newIPs := delta.New.InterfaceIPs[ifaceName]
|
|
|
|
// if a valid interface did not exist in old
|
|
// check that its up and has usable IPs
|
|
if !oldExists {
|
|
// The interface is new (was not present in the old state).
|
|
usableNewIPs := filterUsableIPs(newIPs)
|
|
if newIface.IsUp() && len(usableNewIPs) > 0 {
|
|
// The new interface is the active one, or the change ends as
|
|
// no_active_interface and nothing reconciles.
|
|
activeInterfaceExists = true
|
|
changed = true
|
|
changeIPs = usableNewIPs
|
|
changedInterface = ifaceName
|
|
mainLog.Load().Debug().
|
|
Str("interface", ifaceName).
|
|
Interface("new_ips", usableNewIPs).
|
|
Msg("Interface newly appeared (was not present in old state)")
|
|
break
|
|
}
|
|
continue
|
|
}
|
|
|
|
// Filter new IPs to only those that are usable.
|
|
usableNewIPs := filterUsableIPs(newIPs)
|
|
|
|
// Check if interface is up and has usable IPs.
|
|
if newIface.IsUp() && len(usableNewIPs) > 0 {
|
|
activeInterfaceExists = true
|
|
}
|
|
|
|
// Compare interface states and IPs (interfaceIPsEqual will itself filter the IPs).
|
|
if !interfaceStatesEqual(&oldIface, &newIface) || !interfaceIPsEqual(oldIPs, newIPs) {
|
|
if newIface.IsUp() && len(usableNewIPs) > 0 {
|
|
changed = true
|
|
changeIPs = usableNewIPs
|
|
changedInterface = ifaceName
|
|
mainLog.Load().Debug().
|
|
Str("interface", ifaceName).
|
|
Interface("old_ips", oldIPs).
|
|
Interface("new_ips", usableNewIPs).
|
|
Msg("Interface state or IPs changed")
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// if the default route changed, set changed to true
|
|
if delta.New.DefaultRouteInterface != delta.Old.DefaultRouteInterface {
|
|
changed = true
|
|
mainLog.Load().Debug().Msgf("Default route changed from %s to %s", delta.Old.DefaultRouteInterface, delta.New.DefaultRouteInterface)
|
|
}
|
|
|
|
if !changed {
|
|
mainLog.Load().Debug().Msg("Ignoring interface change - no valid interfaces affected")
|
|
// Minor interface changes can still accompany pf/WFP or VPN DNS changes.
|
|
// On macOS, bound the immediate full reconciliation so link-local-only
|
|
// notification storms do not run pfctl/scutil work for every event.
|
|
// Windows keeps the existing immediate behavior. Tunnel changes always
|
|
// bypass the macOS limit, and delayed checks provide a trailing refresh.
|
|
if dnsIntercept && p.dnsInterceptState != nil {
|
|
networkChangeIgnoredInterceptFn(p, delta, time.Now())
|
|
}
|
|
return
|
|
}
|
|
|
|
if !activeInterfaceExists {
|
|
outcome = "no_active_interface"
|
|
mainLog.Load().Debug().Msg("No active interfaces found, skipping reinitialization")
|
|
return
|
|
}
|
|
|
|
// An ignored event must not cancel this accepted transition. Only a newer
|
|
// accepted transition can replace its source commit or debounced recovery.
|
|
p.networkSourceMu.Lock()
|
|
if p.networkAcceptedGen.Load() > transitionID {
|
|
p.networkSourceMu.Unlock()
|
|
outcome = "superseded"
|
|
return
|
|
}
|
|
p.networkAcceptedGen.Store(transitionID)
|
|
p.networkSourceMu.Unlock()
|
|
|
|
mainLog.Load().Debug().Msg("Link state changed, re-bootstrapping")
|
|
for _, uc := range p.cfg.Upstream {
|
|
uc.ReBootstrap()
|
|
}
|
|
|
|
// Get IPs from default route interface in new state
|
|
selfIP := networkChangeDefaultRouteIPFn()
|
|
|
|
// Ensure that selfIP is an IPv4 address.
|
|
// If defaultRouteIP mistakenly returns an IPv6 (such as a ULA), clear it
|
|
if ip := net.ParseIP(selfIP); ip != nil && ip.To4() == nil {
|
|
mainLog.Load().Debug().Msgf("defaultRouteIP returned a non-IPv4 address: %s, ignoring it", selfIP)
|
|
selfIP = ""
|
|
}
|
|
// Route discovery can lag the delta; do not reintroduce an invalid source.
|
|
if ip := net.ParseIP(selfIP); ip != nil && sourceInvalidReason(delta.New, ip) != "" {
|
|
selfIP = ""
|
|
}
|
|
var ipv6 string
|
|
|
|
if delta.New.DefaultRouteInterface != "" {
|
|
mainLog.Load().Debug().Msgf("default route interface: %s, IPs: %v", delta.New.DefaultRouteInterface, delta.New.InterfaceIPs[delta.New.DefaultRouteInterface])
|
|
for _, ip := range delta.New.InterfaceIPs[delta.New.DefaultRouteInterface] {
|
|
if sourceInvalidReason(delta.New, net.ParseIP(ip.Addr().String())) != "" {
|
|
continue
|
|
}
|
|
ipAddr, _ := netip.ParsePrefix(ip.String())
|
|
addr := ipAddr.Addr()
|
|
if selfIP == "" && addr.Is4() {
|
|
mainLog.Load().Debug().Msgf("checking IP: %s", addr.String())
|
|
if !addr.IsLoopback() && !addr.IsLinkLocalUnicast() {
|
|
selfIP = addr.String()
|
|
}
|
|
}
|
|
if addr.Is6() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() {
|
|
ipv6 = addr.String()
|
|
}
|
|
}
|
|
} else {
|
|
// If no default route interface is set yet, use the changed IPs
|
|
mainLog.Load().Debug().Msgf("no default route interface found, using changed IPs: %v", changeIPs)
|
|
for _, ip := range changeIPs {
|
|
ipAddr, _ := netip.ParsePrefix(ip.String())
|
|
addr := ipAddr.Addr()
|
|
if selfIP == "" && addr.Is4() {
|
|
mainLog.Load().Debug().Msgf("checking IP: %s", addr.String())
|
|
if !addr.IsLoopback() && !addr.IsLinkLocalUnicast() {
|
|
selfIP = addr.String()
|
|
}
|
|
}
|
|
if addr.Is6() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() {
|
|
ipv6 = addr.String()
|
|
}
|
|
}
|
|
}
|
|
|
|
// An ignored event can change source validity without replacing recovery.
|
|
// Recheck addresses against the newest state after blocking route lookup.
|
|
p.networkSourceMu.Lock()
|
|
if p.networkAcceptedGen.Load() != transitionID {
|
|
p.networkSourceMu.Unlock()
|
|
outcome = "superseded"
|
|
return
|
|
}
|
|
latestState := p.sourceCommitState(delta)
|
|
sourceSnapshotAvailable = latestState != nil
|
|
validateDefaultLocalIPsFromDelta(latestState, transitionID)
|
|
// Only keep candidate addresses that still exist on an up interface.
|
|
if ip := net.ParseIP(selfIP); ip != nil && ip.To4() != nil && latestState != nil && sourceInvalidReason(latestState, ip) == "" {
|
|
ctrld.SetDefaultLocalIPv4(ip)
|
|
if !isMobile() && p.ciTable != nil {
|
|
p.ciTable.SetSelfIP(selfIP)
|
|
}
|
|
}
|
|
if ip := net.ParseIP(ipv6); ip != nil && latestState != nil && sourceInvalidReason(latestState, ip) == "" {
|
|
ctrld.SetDefaultLocalIPv6(ip)
|
|
}
|
|
after4, after6 = ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6()
|
|
p.networkSourceMu.Unlock()
|
|
mainLog.Load().Debug().Msgf("Set default local IPv4: %s, IPv6: %s", selfIP, ipv6)
|
|
|
|
outcome = "accepted"
|
|
networkChangeReconcileFn(p, transitionID)
|
|
p.logNetworkSnapshot("transition")
|
|
if p.dnsConfig != nil {
|
|
p.dnsConfig.noteActivity(networkEventsNowFn())
|
|
}
|
|
}
|
|
|
|
// reconcileNetworkChange is the first DNS/PF mutation boundary after source updates.
|
|
func (p *prog) reconcileNetworkChange(transitionID uint64) {
|
|
// we only trigger recovery flow for network changes on non router devices
|
|
if router.Name() == "" {
|
|
p.debounceRecovery(transitionID)
|
|
}
|
|
|
|
// After network changes, verify our pf anchor is still active and
|
|
// refresh VPN DNS state. Order matters: tunnel checks first (may rebuild
|
|
// anchor), then VPN DNS refresh (updates exemptions in anchor), then
|
|
// delayed re-checks for async VPN teardown.
|
|
if dnsIntercept && p.dnsInterceptState != nil {
|
|
if !p.pfStabilizing.Load() {
|
|
p.ensurePFAnchorActive()
|
|
}
|
|
// Check tunnel interfaces unconditionally — it decides internally
|
|
// whether to enter stabilization or rebuild immediately.
|
|
p.checkTunnelInterfaceChanges()
|
|
// Refresh VPN DNS routes — runs after tunnel checks so the anchor
|
|
// rebuild includes current VPN DNS exemptions.
|
|
if p.vpnDNS != nil {
|
|
p.vpnDNS.Refresh(true)
|
|
}
|
|
// Schedule delayed re-checks to catch async VPN teardown changes.
|
|
p.scheduleDelayedRechecks()
|
|
}
|
|
}
|
|
|
|
// handleDNSInterceptIgnoredNetworkChange runs the DNS-intercept work for a
|
|
// network delta that did not affect a usable interface. Keeping this path in a
|
|
// method lets tests exercise the callback wiring with synthetic deltas.
|
|
func (p *prog) handleDNSInterceptIgnoredNetworkChange(delta *netmon.ChangeDelta, now time.Time) {
|
|
reconcileNow := false
|
|
// Stabilization owns PF repair. Do not consume the next leading-edge slot
|
|
// until an ignored delta can actually perform the corresponding PF check.
|
|
if !p.pfStabilizing.Load() {
|
|
reconcileNow = p.dnsInterceptIgnoredChangeReconcileDue(now)
|
|
if reconcileNow {
|
|
p.ensurePFAnchorActive()
|
|
}
|
|
}
|
|
|
|
// Check tunnel interfaces unconditionally — it decides internally whether
|
|
// to enter stabilization or rebuild immediately.
|
|
tunnelChanged := p.checkTunnelInterfaceChanges()
|
|
// Schedule delayed re-checks to catch async VPN teardown changes. These also
|
|
// refresh the OS resolver and VPN DNS routes.
|
|
p.scheduleDelayedRechecks()
|
|
|
|
// Detect interface appearance/disappearance — hypervisors (Parallels,
|
|
// VMware, VirtualBox) reload pf when creating/destroying virtual network
|
|
// interfaces, which can corrupt pf's internal translation state. The rdr
|
|
// rules survive in text form (watchdog says "intact") but stop evaluating.
|
|
// Spawn an async monitor that probes pf interception with backoff and forces
|
|
// a full pf reload if broken.
|
|
if delta.Old != nil {
|
|
changedAction := "removed"
|
|
changedIface := interfaceOnlyIn(delta.Old.Interface, delta.New.Interface)
|
|
if changedIface == "" {
|
|
changedAction = "added"
|
|
changedIface = interfaceOnlyIn(delta.New.Interface, delta.Old.Interface)
|
|
}
|
|
if changedIface != "" {
|
|
journal(mainLog.Load().Info()).Str("interface", changedIface).
|
|
Str("class", interfaceMetaFor(changedIface).Class).
|
|
Str("action", changedAction).
|
|
Msg("DNS intercept: interface appeared/disappeared — starting interception probe monitor")
|
|
go p.pfInterceptMonitor()
|
|
}
|
|
}
|
|
|
|
// Refresh VPN DNS immediately for real tunnel changes even when the periodic
|
|
// ignored-change reconciliation is currently rate-limited - but not while
|
|
// stabilization owns pf. A refresh rebuilds the anchor, and these deltas arrive
|
|
// exactly when a VPN is bringing its own ruleset up, which is the collision
|
|
// stabilization is there to prevent. checkTunnelInterfaceChanges keeps the
|
|
// observation pending, so the transition is retried rather than dropped.
|
|
if p.vpnDNS != nil && (reconcileNow || tunnelChanged) && !p.pfStabilizing.Load() {
|
|
p.vpnDNS.Refresh(true)
|
|
}
|
|
}
|
|
|
|
// interfaceOnlyIn names one interface that have holds and lack does not. The
|
|
// loopback stays out, because it never appears and never goes away.
|
|
func interfaceOnlyIn(have, lack map[string]netmon.Interface) string {
|
|
for name := range have {
|
|
if name == "lo0" {
|
|
continue
|
|
}
|
|
if _, exists := lack[name]; !exists {
|
|
return name
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// interfaceStatesEqual compares two interface states
|
|
func interfaceStatesEqual(a, b *netmon.Interface) bool {
|
|
if a == nil || b == nil {
|
|
return a == b
|
|
}
|
|
return a.IsUp() == b.IsUp()
|
|
}
|
|
|
|
// filterUsableIPs is a helper that returns only "usable" IP prefixes,
|
|
// filtering out link-local, loopback, multicast, unspecified, broadcast, or CGNAT addresses.
|
|
func filterUsableIPs(prefixes []netip.Prefix) []netip.Prefix {
|
|
var usable []netip.Prefix
|
|
for _, p := range prefixes {
|
|
addr := p.Addr()
|
|
if addr.IsLinkLocalUnicast() ||
|
|
addr.IsLoopback() ||
|
|
addr.IsMulticast() ||
|
|
addr.IsUnspecified() ||
|
|
addr.IsLinkLocalMulticast() ||
|
|
(addr.Is4() && addr.String() == "255.255.255.255") ||
|
|
tsaddr.CGNATRange().Contains(addr) {
|
|
continue
|
|
}
|
|
usable = append(usable, p)
|
|
}
|
|
return usable
|
|
}
|
|
|
|
// Modified interfaceIPsEqual compares only the usable (non-link local, non-loopback, etc.) IP addresses.
|
|
func interfaceIPsEqual(a, b []netip.Prefix) bool {
|
|
aUsable := filterUsableIPs(a)
|
|
bUsable := filterUsableIPs(b)
|
|
if len(aUsable) != len(bUsable) {
|
|
return false
|
|
}
|
|
|
|
aMap := make(map[string]bool)
|
|
for _, ip := range aUsable {
|
|
aMap[ip.String()] = true
|
|
}
|
|
for _, ip := range bUsable {
|
|
if !aMap[ip.String()] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
var errOsHealthcheckSuppressed = errors.New("upstream os health check suppressed")
|
|
|
|
// upstreamFailureLog keeps one error line for each upstream of one recovery
|
|
// pass. The pass retries every two seconds until the upstream answers, and the
|
|
// first line already names the fault, so the failures that follow it go to
|
|
// debug.
|
|
type upstreamFailureLog struct{ reported bool }
|
|
|
|
// report logs one failed check of upstream.
|
|
func (l *upstreamFailureLog) report(upstream string, uc *ctrld.UpstreamConfig, err error, duration time.Duration) {
|
|
level := mainLog.Load().Error
|
|
if l.reported {
|
|
level = mainLog.Load().Debug
|
|
}
|
|
l.reported = true
|
|
logUpstreamProbeFailure(isGeneratedInternalDomainUpstreamRef(upstream, uc), uc, err, level,
|
|
"Upstream %s check failed after %v", upstream, duration)
|
|
}
|
|
|
|
// checkUpstreamOnce sends a test query to the specified upstream.
|
|
// Returns nil if the upstream responds successfully.
|
|
func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig, failures *upstreamFailureLog) error {
|
|
mainLog.Load().Debug().Msgf("Starting check for upstream: %s", upstream)
|
|
|
|
resolver, err := ctrld.NewResolver(uc)
|
|
if err != nil {
|
|
logUpstreamProbeFailure(isGeneratedInternalDomainUpstreamRef(upstream, uc), uc, err,
|
|
mainLog.Load().Error, "Failed to create resolver for upstream %s", upstream)
|
|
return err
|
|
}
|
|
|
|
timeout := 1000 * time.Millisecond
|
|
if uc.Timeout > 0 {
|
|
timeout = time.Millisecond * time.Duration(uc.Timeout)
|
|
}
|
|
mainLog.Load().Debug().Msgf("Timeout for upstream %s: %s", upstream, timeout)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
defer cancel()
|
|
|
|
uc.ReBootstrap()
|
|
mainLog.Load().Debug().Msgf("Rebootstrapping resolver for upstream: %s", upstream)
|
|
|
|
start := time.Now()
|
|
msg := uc.VerifyMsg()
|
|
_, err = resolver.Resolve(ctx, msg)
|
|
duration := time.Since(start)
|
|
|
|
if err != nil {
|
|
// Demote upstream.os check failures to debug while WFP loopback
|
|
// protect is active: an external WFP block filter is interfering
|
|
// with plain DNS so repeated failures here are expected. Other
|
|
// upstreams keep error level so real outages stay visible.
|
|
if upstream == upstreamOS && p.osHealthcheckSuppressed() {
|
|
mainLog.Load().Debug().Err(err).Msgf("Upstream %s check failed after %v (WFP loopback protect active)", upstream, duration)
|
|
return errOsHealthcheckSuppressed
|
|
}
|
|
// A no-route/network-unreachable failure means the endpoint's address
|
|
// family is available locally but unroutable (e.g. an IPv6 DoH endpoint
|
|
// while IPv6 is up but has no route). These repeat until the route
|
|
// returns and are handled by bounded backoff in the recovery loop, so
|
|
// keep them at debug to avoid sustained error-log spam.
|
|
if ctrldnet.IsUnreachable(err) {
|
|
mainLog.Load().Debug().Err(err).Msgf("Upstream %s check failed after %v (network unreachable)", upstream, duration)
|
|
return err
|
|
}
|
|
failures.report(upstream, uc, err, duration)
|
|
return err
|
|
}
|
|
mainLog.Load().Debug().Msgf("Upstream %s responded successfully in %v", upstream, duration)
|
|
return nil
|
|
}
|
|
|
|
// recoveryDebounceWindow is the time to wait after the last network change
|
|
// before triggering handleRecovery. This coalesces rapid consecutive network
|
|
// changes (e.g., hotspot→LAN causing en1 drop + en0 pickup + en1 re-pickup)
|
|
// into a single recovery pass, avoiding the cancel-and-restart race that
|
|
// leaves DoH transports in a stale state.
|
|
const recoveryDebounceWindow = 500 * time.Millisecond
|
|
|
|
// recoveryResolverReason names this flow in the OS resolver events, so a
|
|
// reader tells a resolver read of the recovery from every other read.
|
|
const recoveryResolverReason = "recovery"
|
|
|
|
// debounceRecovery schedules a handleRecovery(NetworkChange) call after a debounce
|
|
// window. If called again before the window expires, the timer is reset so that
|
|
// recovery runs once with the final network state. All other state updates (IP,
|
|
// pf anchor, VPN DNS, tunnel checks) run immediately — only the recovery flow
|
|
// with its upstream probing and DHCP bypass logic is debounced.
|
|
func (p *prog) debounceRecovery(transitionID uint64) {
|
|
p.networkSourceMu.Lock()
|
|
defer p.networkSourceMu.Unlock()
|
|
if p.networkAcceptedGen.Load() != transitionID {
|
|
return
|
|
}
|
|
p.recoveryDebounceMu.Lock()
|
|
defer p.recoveryDebounceMu.Unlock()
|
|
|
|
if p.recoveryDebounceTimer != nil {
|
|
p.recoveryDebounceTimer.Stop()
|
|
mainLog.Load().Debug().Msg("Recovery debounce: resetting timer (rapid network change)")
|
|
}
|
|
p.recoveryDebounceTimer = time.AfterFunc(recoveryDebounceWindow, func() {
|
|
p.networkSourceMu.Lock()
|
|
if p.networkAcceptedGen.Load() != transitionID {
|
|
p.networkSourceMu.Unlock()
|
|
return
|
|
}
|
|
p.recoveryDebounceMu.Lock()
|
|
p.recoveryDebounceTimer = nil
|
|
p.recoveryDebounceMu.Unlock()
|
|
p.networkSourceMu.Unlock()
|
|
handleRecoveryForTransitionFn(p, RecoveryReasonNetworkChange, transitionID)
|
|
})
|
|
mainLog.Load().Debug().Msg("Recovery debounce: scheduled (500ms window)")
|
|
}
|
|
|
|
// endRecovery closes one recovery pass. Support reads the end event and the
|
|
// snapshot that follows it to learn what the recovery left behind.
|
|
func (p *prog) endRecovery(diagnostic *recoveryDiagnostic, outcome, targetBefore string) {
|
|
// A superseded pass changed nothing that it owns any more. Its successor
|
|
// owns the loopback target and the bypass flag, so reading them here would
|
|
// report the state of the successor.
|
|
if outcome == recoveryOutcomeSuperseded {
|
|
diagnostic.interceptTargetAction = interceptTargetActionUnchanged
|
|
diagnostic.bypassActive = false
|
|
} else {
|
|
diagnostic.interceptTargetAction = interceptTargetAction(targetBefore, p.interceptTargetSnapshot())
|
|
diagnostic.bypassActive = p.recoveryBypass.Load()
|
|
}
|
|
diagnostic.end(outcome)
|
|
p.logNetworkSnapshot("recovery_end")
|
|
// The resolver table of the host settles after the recovery, so the poll
|
|
// stays fast until it does.
|
|
if p.dnsConfig != nil {
|
|
p.dnsConfig.noteActivity(networkEventsNowFn())
|
|
}
|
|
}
|
|
|
|
// The actions that one recovery takes on the loopback DNS target.
|
|
const (
|
|
interceptTargetActionUnchanged = "unchanged"
|
|
interceptTargetActionRemoved = "removed"
|
|
interceptTargetActionSet = "set"
|
|
)
|
|
|
|
// interceptTargetAction names what one recovery did to the loopback DNS
|
|
// target. The target value alone does not say whether this pass wrote it.
|
|
func interceptTargetAction(before, after string) string {
|
|
switch {
|
|
case before == after:
|
|
return interceptTargetActionUnchanged
|
|
case after == "":
|
|
return interceptTargetActionRemoved
|
|
default:
|
|
return interceptTargetActionSet
|
|
}
|
|
}
|
|
|
|
// handleRecovery performs a unified recovery by removing DNS settings,
|
|
// canceling existing recovery checks for network changes, but coalescing duplicate
|
|
// upstream failure recoveries, waiting for recovery to complete (using a cancellable context without timeout),
|
|
// and then re-applying the DNS settings.
|
|
func (p *prog) handleRecovery(reason RecoveryReason) {
|
|
p.handleRecoveryForTransition(reason, 0)
|
|
}
|
|
|
|
func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID uint64) {
|
|
p.networkSourceMu.Lock()
|
|
if transitionID != 0 && p.networkAcceptedGen.Load() != transitionID {
|
|
p.networkSourceMu.Unlock()
|
|
mainLog.Load().Debug().Uint64("transition_id", transitionID).Msg("Recovery skipped: transition superseded")
|
|
return
|
|
}
|
|
recoveryCtx, gen, interceptRecovery, ok := p.beginRecovery(reason)
|
|
p.networkSourceMu.Unlock()
|
|
if !ok {
|
|
mainLog.Load().Debug().Uint64("transition_id", transitionID).
|
|
Uint64("recovery_generation", p.recoveryGen.Load()).
|
|
Msg("Upstream recovery already in progress; skipping duplicate trigger")
|
|
return
|
|
}
|
|
diagnostic := recoveryDiagnostic{transitionID: transitionID, generation: gen, reason: reason, started: time.Now()}
|
|
journal(diagnostic.event(mainLog.Load().Info())).Bool("intercept", interceptRecovery).Msg("Recovery begin")
|
|
p.logNetworkSnapshot("recovery_begin")
|
|
targetBefore := p.interceptTargetSnapshot()
|
|
outcome := recoveryOutcomeSuperseded
|
|
defer func() { p.endRecovery(&diagnostic, outcome, targetBefore) }()
|
|
if reason == RecoveryReasonNetworkChange {
|
|
mainLog.Load().Debug().Msg("Network change recovery now owns shared recovery state")
|
|
}
|
|
|
|
// For network changes, force-reset all upstream transports synchronously.
|
|
// The lazy ReBootstrap() called earlier in the network change callback only
|
|
// sets a flag — the old transport's dead connections can still be used by
|
|
// recovery probes, causing context deadline timeouts. ForceReBootstrap()
|
|
// closes old connections and creates fresh transports so probes succeed on
|
|
// first attempt.
|
|
if reason == RecoveryReasonNetworkChange {
|
|
for _, uc := range p.cfg.Upstream {
|
|
if uc != nil {
|
|
uc.ForceReBootstrap()
|
|
}
|
|
}
|
|
mainLog.Load().Info().Msg("Force-reset upstream transports for network change recovery")
|
|
}
|
|
|
|
// In DNS intercept mode, don't tear down WFP/pf filters.
|
|
// Instead, enable recovery bypass so proxy() forwards queries to
|
|
// the OS/DHCP resolver. This handles captive portal authentication
|
|
// without the overhead of filter teardown/rebuild.
|
|
if interceptRecovery {
|
|
journal(mainLog.Load().Info()).Msg("DNS intercept recovery: enabling DHCP bypass (filters stay active)")
|
|
|
|
// Reinitialize OS resolver to discover DHCP servers on the new network.
|
|
mainLog.Load().Debug().Msg("DNS intercept recovery: discovering DHCP nameservers")
|
|
// The effective list adds a synthetic public resolver when the network
|
|
// gave none, so the journal reports the discovered list instead.
|
|
resolverNameservers, systemNameservers := initializeOsResolverWithSystemNameserversFn(true, recoveryResolverReason)
|
|
diagnostic.dhcpServers = systemNameservers
|
|
if len(systemNameservers) == 0 {
|
|
journal(mainLog.Load().Warn()).Msg("DNS intercept recovery: no DHCP nameservers found")
|
|
} else {
|
|
journal(mainLog.Load().Info()).Strs("dhcp_servers", systemNameservers).
|
|
Msgf("DNS intercept recovery: found DHCP nameservers: %v", systemNameservers)
|
|
}
|
|
|
|
// If the new network provides no usable IPv4 DNS (e.g. IPv6-only
|
|
// tethering with 464XLAT), macOS cannot emit DNS queries at all and
|
|
// pf has nothing to intercept. Ensure a loopback DNS target exists
|
|
// so the OS keeps sending queries to ctrld's listener (issue #533).
|
|
ensureInterceptDNSTargetFn(p, systemNameservers)
|
|
|
|
// Exempt DHCP nameservers from intercept filters so the OS resolver
|
|
// can actually reach them on port 53.
|
|
// The OS resolver queries the effective list, so the exemptions follow it.
|
|
if len(resolverNameservers) > 0 {
|
|
// Build exemptions without an Interface — DHCP servers are not VPN-specific,
|
|
// so they only generate group-scoped pf rules (ctrld process only).
|
|
exemptions := make([]vpnDNSExemption, 0, len(resolverNameservers))
|
|
for _, s := range resolverNameservers {
|
|
host := s
|
|
if h, _, err := net.SplitHostPort(s); err == nil {
|
|
host = h
|
|
}
|
|
exemptions = append(exemptions, vpnDNSExemption{Server: host})
|
|
}
|
|
mainLog.Load().Info().Msgf("DNS intercept recovery: exempting DHCP nameservers from filters: %v", exemptions)
|
|
if err := p.exemptVPNDNSServers(exemptions); err != nil {
|
|
mainLog.Load().Warn().Err(err).Msg("DNS intercept recovery: failed to exempt DHCP nameservers — recovery queries may fail")
|
|
}
|
|
}
|
|
} else {
|
|
// Traditional flow: remove DNS settings to expose DHCP nameservers
|
|
recoveryResetDNSFn(p, false, false)
|
|
|
|
// For an OS failure, reinitialize OS resolver nameservers immediately.
|
|
if reason == RecoveryReasonOSFailure {
|
|
mainLog.Load().Debug().Msg("OS resolver failure detected; reinitializing OS resolver nameservers")
|
|
ns := ctrld.InitializeOsResolverWithReason(true, recoveryResolverReason)
|
|
if len(ns) == 0 {
|
|
mainLog.Load().Warn().Msg("No nameservers found for OS resolver; using existing values")
|
|
} else {
|
|
journal(mainLog.Load().Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Build upstream map based on the recovery reason.
|
|
upstreams := p.buildRecoveryUpstreams(reason)
|
|
|
|
// Wait indefinitely until one of the upstreams recovers.
|
|
recovered, err := waitForUpstreamRecoveryFn(p, recoveryCtx, upstreams, &diagnostic)
|
|
if err != nil {
|
|
if p.recoveryOwnsState(gen) {
|
|
outcome = recoveryOutcomeCanceled
|
|
}
|
|
p.recoveryCanceledCleanup(gen)
|
|
return
|
|
}
|
|
diagnostic.recoveredUpstream = recovered
|
|
if !p.recoveryOwnsState(gen) {
|
|
mainLog.Load().Debug().Msgf("Recovery generation %d was superseded after upstream success; skipping stale completion", gen)
|
|
return
|
|
}
|
|
// The reset ends the outage, so the down time is read before it. The name
|
|
// of the line is bounded, because an upstream key is operator text.
|
|
journalName := journalUpstreamName(recovered)
|
|
journal(mainLog.Load().Info()).Str("upstream", journalName).
|
|
Int64("down_for_ms", p.um.downFor(recovered).Milliseconds()).
|
|
Msgf("Upstream %q recovered; re-applying DNS settings", journalName)
|
|
|
|
// Reset the upstream failure count and down state while this generation
|
|
// still owns recovery completion.
|
|
p.um.reset(recovered)
|
|
|
|
if interceptRecovery {
|
|
// Refresh VPN DNS routes in case VPN state changed during recovery.
|
|
if p.vpnDNS != nil {
|
|
p.vpnDNS.Refresh(true)
|
|
}
|
|
|
|
// Reinitialize OS resolver for the recovered state.
|
|
if reason == RecoveryReasonNetworkChange {
|
|
ns := ctrld.InitializeOsResolverWithReason(true, recoveryResolverReason)
|
|
if len(ns) == 0 {
|
|
mainLog.Load().Warn().Msg("No nameservers found for OS resolver during network-change recovery; using existing values")
|
|
} else {
|
|
journal(mainLog.Load().Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns)
|
|
}
|
|
}
|
|
} else {
|
|
var systemNameservers []string
|
|
if dnsIntercept {
|
|
// Intercept was requested but no interceptor was active when recovery
|
|
// began. Rediscover on every recovery reason before retrying setDNS;
|
|
// passing nil could make a successful retry install a loopback target
|
|
// on a healthy DHCP network.
|
|
systemNameservers = systemNameserversForInterceptRetry()
|
|
} else if reason == RecoveryReasonNetworkChange {
|
|
ns := ctrld.InitializeOsResolverWithReason(true, recoveryResolverReason)
|
|
if len(ns) == 0 {
|
|
mainLog.Load().Warn().Msg("No nameservers found for OS resolver during network-change recovery; using existing values")
|
|
} else {
|
|
journal(mainLog.Load().Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns)
|
|
}
|
|
}
|
|
|
|
// Apply our DNS settings back. The snapshot that follows the end event
|
|
// reports the interfaces.
|
|
p.setDNS(systemNameservers)
|
|
}
|
|
|
|
if !p.completeRecovery(gen) {
|
|
mainLog.Load().Debug().Msgf("Recovery generation %d was superseded during completion; preserving successor state", gen)
|
|
return
|
|
}
|
|
outcome = recoveryOutcomeCompleted
|
|
if interceptRecovery {
|
|
journal(mainLog.Load().Info()).Msg("DNS intercept recovery complete: disabling DHCP bypass, resuming normal flow")
|
|
}
|
|
}
|
|
|
|
// waitForUpstreamRecoveryFn is the seam of the upstream probe. A test drives
|
|
// the whole recovery flow without a query to a real upstream.
|
|
var waitForUpstreamRecoveryFn = (*prog).waitForUpstreamRecovery
|
|
|
|
// waitForUpstreamRecovery checks the provided upstreams concurrently until one recovers.
|
|
// It returns the name of the recovered upstream or an error if the check times out.
|
|
func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string]*ctrld.UpstreamConfig, diagnostic *recoveryDiagnostic) (string, error) {
|
|
recoveryCtx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
|
|
recoveredCh := make(chan string, 1)
|
|
var wg sync.WaitGroup
|
|
|
|
mainLog.Load().Debug().Msgf("Starting upstream recovery check for %d upstreams", len(upstreams))
|
|
|
|
for name, uc := range upstreams {
|
|
wg.Add(1)
|
|
go func(name string, uc *ctrld.UpstreamConfig) {
|
|
defer wg.Done()
|
|
mainLog.Load().Debug().Msgf("Starting recovery check loop for upstream: %s", name)
|
|
attempts := 0
|
|
unreachableStreak := 0
|
|
var failures upstreamFailureLog
|
|
for {
|
|
select {
|
|
case <-recoveryCtx.Done():
|
|
mainLog.Load().Debug().Msgf("Context canceled for upstream %s", name)
|
|
return
|
|
default:
|
|
attempts++
|
|
// checkUpstreamOnce will reset any failure counters on success.
|
|
err := p.checkUpstreamOnce(name, uc, &failures)
|
|
if err == nil || errors.Is(err, errOsHealthcheckSuppressed) {
|
|
mainLog.Load().Debug().Msgf("Upstream %s recovered successfully", name)
|
|
select {
|
|
case recoveredCh <- name:
|
|
mainLog.Load().Debug().Msgf("Sent recovery notification for upstream %s", name)
|
|
cancel()
|
|
default:
|
|
mainLog.Load().Debug().Msg("Recovery channel full, another upstream already recovered")
|
|
}
|
|
return
|
|
}
|
|
diagnostic.failure(err)
|
|
// Back off the retry cadence for an unroutable endpoint so a
|
|
// host with IPv6 up but no route to the IPv6 DoH endpoint does
|
|
// not re-bootstrap/re-check every checkUpstreamBackoffSleep and
|
|
// spam the log. The backoff is bounded (checkUpstreamUnreachableBackoffMax)
|
|
// so the endpoint is still re-probed and recovers when the route
|
|
// returns; any other failure resets to the base cadence.
|
|
sleep := checkUpstreamBackoffSleep
|
|
if ctrldnet.IsUnreachable(err) {
|
|
unreachableStreak++
|
|
sleep = unreachableRecoveryBackoff(unreachableStreak)
|
|
mainLog.Load().Debug().Msgf("Upstream %s unreachable (streak %d), backing off %s before retry", name, unreachableStreak, sleep)
|
|
} else {
|
|
unreachableStreak = 0
|
|
mainLog.Load().Debug().Msgf("Upstream %s check failed, sleeping before retry", name)
|
|
}
|
|
if !sleepWithContext(recoveryCtx, sleep) {
|
|
return
|
|
}
|
|
|
|
// if this is the upstreamOS and it's the 3rd attempt (or multiple of 3),
|
|
// we should try to reinit the OS resolver to ensure we can recover
|
|
if name == upstreamOS && attempts%3 == 0 {
|
|
mainLog.Load().Debug().Msgf("UpstreamOS check failed on attempt %d, reinitializing OS resolver", attempts)
|
|
ns := ctrld.InitializeOsResolverWithReason(true, recoveryResolverReason)
|
|
if len(ns) == 0 {
|
|
mainLog.Load().Warn().Msg("No nameservers found for OS resolver; using existing values")
|
|
} else {
|
|
journal(mainLog.Load().Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}(name, uc)
|
|
}
|
|
|
|
var recovered string
|
|
select {
|
|
case <-ctx.Done():
|
|
return "", ctx.Err()
|
|
default:
|
|
}
|
|
select {
|
|
case recovered = <-recoveredCh:
|
|
if err := ctx.Err(); err != nil {
|
|
return "", err
|
|
}
|
|
case <-ctx.Done():
|
|
return "", ctx.Err()
|
|
}
|
|
wg.Wait()
|
|
return recovered, nil
|
|
}
|
|
|
|
func sleepWithContext(ctx context.Context, d time.Duration) bool {
|
|
timer := time.NewTimer(d)
|
|
defer timer.Stop()
|
|
select {
|
|
case <-timer.C:
|
|
return true
|
|
case <-ctx.Done():
|
|
return false
|
|
}
|
|
}
|
|
|
|
// buildRecoveryUpstreams constructs the map of upstream configurations to test.
|
|
// For OS failures we supply the manual OS resolver upstream configuration.
|
|
// For network change or regular failure we use the upstreams defined in p.cfg (ignoring OS).
|
|
func (p *prog) buildRecoveryUpstreams(reason RecoveryReason) map[string]*ctrld.UpstreamConfig {
|
|
upstreams := make(map[string]*ctrld.UpstreamConfig)
|
|
switch reason {
|
|
case RecoveryReasonOSFailure:
|
|
upstreams[upstreamOS] = osUpstreamConfig
|
|
case RecoveryReasonNetworkChange, RecoveryReasonRegularFailure:
|
|
// Use all configured upstreams except any OS type.
|
|
for k, uc := range p.cfg.Upstream {
|
|
if uc.Type != ctrld.ResolverTypeOS {
|
|
upstreams[upstreamPrefix+k] = uc
|
|
}
|
|
}
|
|
}
|
|
return upstreams
|
|
}
|
|
|
|
// ValidateDefaultLocalIPsFromDelta checks if the default local IPv4 and IPv6 stored
|
|
// are still present in the new network state (provided by delta.New).
|
|
// If a stored default IP is no longer active, it resets that default (sets it to nil)
|
|
// so that it won't be used in subsequent custom dialer contexts.
|
|
// Deprecated: retained for callers of the existing exported cli API.
|
|
// The network monitor uses transition-aware validation instead.
|
|
func ValidateDefaultLocalIPsFromDelta(newState *netmon.State) {
|
|
validateDefaultLocalIPsFromDelta(newState, 0)
|
|
}
|