diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index ff55343..59bff9f 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -37,6 +37,17 @@ const ( // localTTL is the TTL for local network responses // Longer TTL for local queries reduces unnecessary repeated lookups localTTL = 3600 * time.Second + // zpaDNSEchoDomain is Zscaler Client Connector's fixed public health-check + // name. Client Connector must see the answer to it arrive on its own DNS + // path before it will synthesize addresses for Private Access applications. + zpaDNSEchoDomain = "dnsechotest.zscaler.com" + // zpaDNSEchoPolicyName is reported as the matched policy when the built-in + // ZPA DNS echo bypass routes a query, so log lines name the reason. + zpaDNSEchoPolicyName = "Built-in ZPA DNS echo bypass" + // noRuleMatched is the matchedRule placeholder used when no listener domain + // rule matched the query. It is compared, not only logged: the built-in ZPA + // echo bypass yields to an explicit domain rule for the same name. + noRuleMatched = "no rule" // EDNS0_OPTION_MAC is dnsmasq EDNS0 code for adding mac option. // https://thekelleys.org.uk/gitweb/?p=dnsmasq.git;a=blob;f=src/dns-protocol.h;h=76ac66a8c28317e9c121a74ab5fd0e20f6237dc8;hb=HEAD#l81 @@ -49,6 +60,19 @@ const ( selfUninstallMaxQueries = 32 ) +// zpaDNSEchoBypassEnabled limits the built-in ZPA DNS echo bypass to macOS, +// the only platform where pf interception is known to break Zscaler Private +// Access. Tests override it. +var zpaDNSEchoBypassEnabled = runtime.GOOS == "darwin" + +// zpaDNSEchoBypassActive reports whether the built-in Zscaler DNS echo bypass +// applies. Like every other VPN accommodation it is skipped in +// --intercept-mode hard, where the operator has asked for all DNS to go +// through ctrld with no split routing. +func zpaDNSEchoBypassActive() bool { + return zpaDNSEchoBypassEnabled && dnsIntercept && !hardIntercept +} + // osUpstreamConfig defines the default OS resolver configuration // This is used as a fallback when all configured upstreams fail var osUpstreamConfig = &ctrld.UpstreamConfig{ @@ -96,6 +120,8 @@ type upstreamForResult struct { matchedRule string matched bool srcAddr string + // zpaDNSEcho keeps the health check on the OS path and out of the cache. + zpaDNSEcho bool } func (p *prog) addCachedResponse(key dnscache.Key, answer *dns.Msg) { @@ -477,6 +503,58 @@ func (p *prog) upstreamForWithConfig(ctx context.Context, req *upstreamForReques // Default upstreams upstreams := []string{upstreamPrefix + req.DefaultUpstreamNum} res = &upstreamForResult{srcAddr: req.Addr.String()} + defer func() { + // Zscaler Private Access: under macOS Intercept Mode, pf routes Client + // Connector's health-check query for zpaDNSEchoDomain into ctrld, which + // answers it from the Control D upstream. Client Connector never sees the + // answer on its own DNS path, so it keeps Private Access disabled. + // + // Route only that one exact name the way a Control D "bypass" rule does: + // an empty upstream list, which proxy() resolves through upstream.os — + // the system resolver set, which under Intercept Mode still includes + // Client Connector's own DNS servers. Matching on the name alone covers + // every record type, the same as a bypass rule. + // + // This runs after policy evaluation, and never touches `matched`. That + // flag is also the source authorization bit: serveDNS refuses a query on + // a Restricted listener when it is false, so forcing it true here would + // let an unauthorized client resolve this name through such a listener. + // + // The scope is one fixed public health-check name: every other domain, + // including all Private Access application domains, stays intercepted and + // filtered by Control D. --intercept-mode hard opts out, the same as it + // opts out of VPN DNS split routing. + if zpaDNSEchoBypassActive() && canonicalName(req.Domain) == zpaDNSEchoDomain { + switch { + case res.matchedRule == noRuleMatched: + // The listener policy says nothing about this name, so the + // built-in route decides. Network- and MAC-policy targets route + // every name from a source and so state nothing about this one: + // override them, and the default upstream, and label the result + // so logs name the reason. + res.upstreams = nil + res.matchedPolicy = zpaDNSEchoPolicyName + res.matchedRule = zpaDNSEchoDomain + res.zpaDNSEcho = true + case len(res.upstreams) == 0: + // An explicit domain rule already selects the OS path: a rule + // with empty targets, which is how a Control D profile's bypass + // list reaches us (see the cfg.Listener rules built from + // resolverConfig.Exclude). That is the documented pre-fix + // workaround for this very bug, so keep the operator's own + // labels but still classify it as the health check — otherwise + // anyone who keeps the workaround after upgrading would route + // correctly and then have the answer served from our cache, + // losing the reconnect fix. + res.zpaDNSEcho = true + default: + // Any other explicit domain rule is a deliberate non-OS route + // for this exact name. That is operator intent, and the + // supported way to opt out without leaving Intercept Mode, so + // leave the policy's decision untouched. + } + } + }() // If no policy, return default upstreams if req.ListenerConfig.Policy == nil { @@ -484,7 +562,7 @@ func (p *prog) upstreamForWithConfig(ctx context.Context, req *upstreamForReques res.matched = false res.matchedPolicy = "no policy" res.matchedNetwork = "no network" - res.matchedRule = "no rule" + res.matchedRule = noRuleMatched return } @@ -620,6 +698,9 @@ func (p *prog) handleSpecialQueryTypes(ctx *context.Context, req *proxyRequest, } switch { + case req.ufr.zpaDNSEcho: + ctrld.Log(*ctx, p.Debug(), "%s, %s -> %v", req.ufr.matchedPolicy, req.ufr.matchedRule, *upstreams) + return nil case isSrvLanLookup(req.msg): *upstreams = []string{upstreamOS} *upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig} @@ -811,8 +892,8 @@ func (p *prog) initializeUpstreams(req *proxyRequest) ([]string, []*ctrld.Upstre // Skips cache checking if caching is disabled or the request is a PTR query. // Iterates through the provided upstreams to find a cached response using the checkCache method. func (p *prog) tryCache(ctx context.Context, req *proxyRequest, upstreams []string) *proxyResponse { - if p.cache == nil || req.msg.Question[0].Qtype == dns.TypePTR { // https://www.rfc-editor.org/rfc/rfc1035#section-7.4 - ctrld.Log(ctx, p.Debug(), "Cache disabled or PTR query, skipping cache lookup") + if p.cache == nil || req.msg.Question[0].Qtype == dns.TypePTR || req.ufr.zpaDNSEcho { // https://www.rfc-editor.org/rfc/rfc1035#section-7.4 + ctrld.Log(ctx, p.Debug(), "Cache disabled, PTR query, or ZPA DNS echo, skipping cache lookup") return nil } @@ -869,6 +950,9 @@ func (p *prog) checkCache(ctx context.Context, req *proxyRequest, upstream strin // updateCache updates the DNS response cache with the given request, response, TTL, and upstream information. func (p *prog) updateCache(ctx context.Context, req *proxyRequest, answer *dns.Msg, upstream string) { + if req.ufr.zpaDNSEcho { + return + } p.addCachedResponse(dnscache.NewKey(req.msg, upstream), answer) ctrld.Log(ctx, p.Debug(), "Added cached response") } @@ -938,7 +1022,9 @@ func (p *prog) prepareSuccessResponse(ctx context.Context, req *proxyRequest, an answer.Compress = true - if p.cache != nil && req.msg.Question[0].Qtype != dns.TypePTR { + // The echo must reach Client Connector on every poll, including reconnects. + cacheable := p.cache != nil && req.msg.Question[0].Qtype != dns.TypePTR && !req.ufr.zpaDNSEcho + if cacheable { ctrld.Log(ctx, p.Debug(), "Updating cache with successful response") p.updateCache(ctx, req, answer, upstream) } @@ -948,7 +1034,7 @@ func (p *prog) prepareSuccessResponse(ctx context.Context, req *proxyRequest, an var synthesizedPrefix netip.Prefix answer, synthesizedPrefix = p.maybeDNS64(ctx, req.msg, answer, func(aReq *dns.Msg) *dns.Msg { key := dnscache.NewKey(aReq, upstream) - if p.cache != nil { + if cacheable { if cachedValue := p.cache.Get(key); cachedValue != nil { now := time.Now() if cachedValue.Expire.After(now) { @@ -963,12 +1049,12 @@ func (p *prog) prepareSuccessResponse(ctx context.Context, req *proxyRequest, an aProxyReq := *req aProxyReq.msg = aReq resolved := p.queryUpstream(ctx, &aProxyReq, upstream, upstreamConfig) - if p.cache != nil && resolved != nil && sameQuestion(aReq, resolved) { + if cacheable && resolved != nil && sameQuestion(aReq, resolved) { p.addCachedResponse(key, resolved) } return resolved }) - if p.cache != nil && synthesizedPrefix.IsValid() { + if cacheable && synthesizedPrefix.IsValid() { p.addCachedResponse(dns64CacheKey(req.msg, upstream, synthesizedPrefix), answer) ctrld.Log(ctx, p.Debug(), "DNS64: added cached response variant") } @@ -991,7 +1077,7 @@ func (p *prog) prepareSuccessResponse(ctx context.Context, req *proxyRequest, an // It returns a successful proxyResponse if any upstream processes the request successfully, or nil otherwise. // The function supports "serve stale" for cache by utilizing cached responses when upstreams fail. func (p *prog) tryUpstreams(ctx context.Context, req *proxyRequest, upstreams []string, upstreamConfigs []*ctrld.UpstreamConfig) *proxyResponse { - serveStaleCache := p.cache != nil && p.cfg.Service.CacheServeStale + serveStaleCache := p.cache != nil && p.cfg.Service.CacheServeStale && !req.ufr.zpaDNSEcho req.upstreamConfigs = upstreamConfigs ctrld.Log(ctx, p.Debug(), "Trying %d upstreams", len(upstreamConfigs)) diff --git a/cmd/cli/dns_proxy_zpa_test.go b/cmd/cli/dns_proxy_zpa_test.go new file mode 100644 index 0000000..a373779 --- /dev/null +++ b/cmd/cli/dns_proxy_zpa_test.go @@ -0,0 +1,386 @@ +package cli + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ctrld "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/dnscache" + "github.com/Control-D-Inc/ctrld/testhelper" +) + +// withZPADNSEchoBypass sets the three gates that guard the built-in Zscaler +// DNS echo bypass: the macOS-only platform gate, Intercept Mode, and the +// hard-mode opt-out. +func withZPADNSEchoBypass(t *testing.T, onDarwin, intercept, hard bool) { + t.Helper() + oldBypass, oldIntercept, oldHard := zpaDNSEchoBypassEnabled, dnsIntercept, hardIntercept + zpaDNSEchoBypassEnabled = onDarwin + dnsIntercept = intercept + hardIntercept = hard + t.Cleanup(func() { + zpaDNSEchoBypassEnabled = oldBypass + dnsIntercept = oldIntercept + hardIntercept = oldHard + }) +} + +// listenerWithRules builds a listener whose policy carries only domain rules. +func listenerWithRules(rules ...ctrld.Rule) *ctrld.ListenerConfig { + return &ctrld.ListenerConfig{ + IP: "127.0.0.1", + Port: 53, + Policy: &ctrld.ListenerPolicyConfig{ + Name: "Test Policy", + Rules: rules, + }, + } +} + +// Test_prog_upstreamFor_ZPADNSEchoBypass pins the routing decision for issue +// 601: under macOS Intercept Mode the exact Zscaler health-check name — and +// nothing else — is routed like a Control D bypass rule, i.e. with an empty +// upstream list that proxy() then resolves through upstream.os. +func Test_prog_upstreamFor_ZPADNSEchoBypass(t *testing.T) { + cfg := testhelper.SampleConfig(t) + p := &prog{cfg: cfg} + p.logger.Store(mainLog.Load()) + for _, nc := range p.cfg.Network { + for _, cidr := range nc.Cidrs { + _, ipNet, err := net.ParseCIDR(cidr) + require.NoError(t, err) + nc.IPNets = append(nc.IPNets, ipNet) + } + } + + // A listener whose policy routes the echo name to a specific non-OS + // upstream. That is a deliberate route for this exact name, so it wins. + lcWithEchoRule := listenerWithRules(ctrld.Rule{zpaDNSEchoDomain: []string{"upstream.2"}}) + lcWithZscalerWildcardRule := listenerWithRules(ctrld.Rule{"*.zscaler.com": []string{"upstream.2"}}) + + // Empty targets are how a Control D profile's bypass list reaches ctrld + // (cfg.Listener rules built from resolverConfig.Exclude). This is the + // documented pre-fix workaround, so it must keep routing to the OS path AND + // stay classified as the health check, or upgrading a machine that still + // carries the workaround would lose the no-cache behavior. + lcWithEchoBypassRule := listenerWithRules(ctrld.Rule{zpaDNSEchoDomain: []string{}}) + lcWithZscalerWildcardBypassRule := listenerWithRules(ctrld.Rule{"*.zscaler.com": []string{}}) + + // network.0 targets, i.e. what an un-bypassed query from 192.168.0.1 gets. + notBypassed := []string{"upstream.1", "upstream.0"} + const ( + inNetwork0 = "192.168.0.1:0" // matches network.0, so listener.0's policy matches + outOfPolicy = "10.0.0.1:0" // matches no configured network + ) + + tests := []struct { + name string + onDarwin bool + intercept bool + hard bool + lc *ctrld.ListenerConfig + defaultNum string + srcAddr string + domain string + wantUpstreams []string + wantMatched bool + wantZPAEcho bool + }{ + {"exact echo name routes to the OS resolver path", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, nil, true, true}, + {"echo name is matched case-insensitively and with a trailing dot", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "DNSEchoTest.ZScaler.COM.", nil, true, true}, + {"echo name is bypassed on a listener with no policy", true, true, false, p.cfg.Listener["1"], "1", inNetwork0, zpaDNSEchoDomain, nil, false, true}, + + // The bypass must not double as source authorization. serveDNS refuses a + // query on a Restricted listener when matched is false, so an + // unauthorized source must stay unmatched even for this name — while + // still being routed to the OS resolver path if the listener does serve + // it. See TestZPADNSEchoBypassDoesNotAuthorizeRestrictedListener. + {"unauthorized source is still unmatched", true, true, false, p.cfg.Listener["0"], "0", outOfPolicy, zpaDNSEchoDomain, nil, false, true}, + + // An explicit domain rule routing the name to a specific upstream + // outranks the built-in route; network/MAC policy targets, which say + // nothing about this name, do not. + {"explicit domain rule for the name wins", true, true, false, lcWithEchoRule, "0", inNetwork0, zpaDNSEchoDomain, []string{"upstream.2"}, true, false}, + {"explicit wildcard rule covering the name wins", true, true, false, lcWithZscalerWildcardRule, "0", inNetwork0, zpaDNSEchoDomain, []string{"upstream.2"}, true, false}, + + // An explicit rule that itself selects the OS path (empty targets, i.e. + // the Control D bypass-list shape) keeps its own labels but is still the + // health check, so proxy() keeps it out of the cache. + {"existing exact bypass rule stays classified as the health check", true, true, false, lcWithEchoBypassRule, "0", inNetwork0, zpaDNSEchoDomain, []string{}, true, true}, + {"existing wildcard bypass rule stays classified as the health check", true, true, false, lcWithZscalerWildcardBypassRule, "0", inNetwork0, zpaDNSEchoDomain, []string{}, true, true}, + + // Gating: the bypass exists only where the problem does. + {"not bypassed off macOS", false, true, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, notBypassed, true, false}, + {"not bypassed outside Intercept Mode", true, false, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, notBypassed, true, false}, + {"not bypassed in hard intercept mode", true, true, true, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, notBypassed, true, false}, + + // Scope: only the one exact name, so no neighbouring name loses filtering. + {"subdomain of the echo name is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "foo." + zpaDNSEchoDomain, notBypassed, true, false}, + {"echo name as a prefix of another domain is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain + ".evil.example", notBypassed, true, false}, + {"name that merely ends with the echo name is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "evil" + zpaDNSEchoDomain, notBypassed, true, false}, + {"parent Zscaler domain is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "zscaler.com", notBypassed, true, false}, + {"ordinary domain is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "example.com", notBypassed, true, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + withZPADNSEchoBypass(t, tc.onDarwin, tc.intercept, tc.hard) + addr, err := net.ResolveUDPAddr("udp", tc.srcAddr) + require.NoError(t, err) + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + + ufr := p.upstreamFor(ctx, tc.defaultNum, tc.lc, addr, "", tc.domain) + + assert.Equal(t, tc.wantUpstreams, ufr.upstreams) + assert.Equal(t, tc.wantMatched, ufr.matched, "matched is the source-authorization bit") + assert.Equal(t, tc.wantZPAEcho, ufr.zpaDNSEcho) + if tc.wantZPAEcho { + // An empty upstream list is the whole mechanism: it is what + // proxy() turns into upstream.os. Guard the translation step + // that makes that substitution fire. + assert.Empty(t, p.upstreamConfigsFromUpstreamNumbers(ufr.upstreams)) + if tc.lc.Policy == nil || len(tc.lc.Policy.Rules) == 0 { + // Only relabelled when the built-in route is the one that + // made the decision; an operator's own matching rule keeps + // its own labels. + assert.Equal(t, zpaDNSEchoPolicyName, ufr.matchedPolicy) + assert.Equal(t, zpaDNSEchoDomain, ufr.matchedRule) + } + } + }) + } +} + +// TestZPADNSEchoBypassDoesNotAuthorizeRestrictedListener pins the serveDNS +// consequence of keeping `matched` out of the bypass: a Restricted listener +// answers only sources that match its policy, and the built-in echo route must +// not become a hole in that for every macOS DNS-intercept instance. +func TestZPADNSEchoBypassDoesNotAuthorizeRestrictedListener(t *testing.T) { + withZPADNSEchoBypass(t, true, true, false) + + cfg := testhelper.SampleConfig(t) + p := &prog{cfg: cfg} + p.logger.Store(mainLog.Load()) + for _, nc := range p.cfg.Network { + for _, cidr := range nc.Cidrs { + _, ipNet, err := net.ParseCIDR(cidr) + require.NoError(t, err) + nc.IPNets = append(nc.IPNets, ipNet) + } + } + lc := p.cfg.Listener["0"] + lc.Restricted = true + + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + for _, tc := range []struct { + name string + srcAddr string + wantRefused bool + }{ + {"source outside every network policy is refused", "10.0.0.1:0", true}, + {"source inside network.0 is served", "192.168.0.1:0", false}, + } { + t.Run(tc.name, func(t *testing.T) { + addr, err := net.ResolveUDPAddr("udp", tc.srcAddr) + require.NoError(t, err) + ufr := p.upstreamFor(ctx, "0", lc, addr, "", zpaDNSEchoDomain) + + // The route itself is unconditional; only the authorization differs. + require.True(t, ufr.zpaDNSEcho) + // This is the exact condition serveDNS uses to answer REFUSED. + assert.Equal(t, tc.wantRefused, !ufr.matched && lc.Restricted) + }) + } +} + +// osResolverStub stands in for the OS resolver during a proxy() test. It +// answers NOERROR for anything, and records what it was actually asked, so a +// test can tell an answer that came off the wire from one served locally. +type osResolverStub struct { + addr string + + mu sync.Mutex + queries []string +} + +// startOSResolverStub points osUpstreamConfig at a local DNS server for the +// duration of the test. It uses the legacy (plain UDP) resolver type because +// ResolverTypeOS resolves through process-wide state in the ctrld package that +// a test in this package cannot address. +func startOSResolverStub(t *testing.T) *osResolverStub { + t.Helper() + + pc, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + require.NoError(t, err) + + stub := &osResolverStub{addr: pc.LocalAddr().String()} + srv := &dns.Server{PacketConn: pc, Net: "udp"} + srv.Handler = dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) { + stub.mu.Lock() + stub.queries = append(stub.queries, dns.TypeToString[m.Question[0].Qtype]+" "+m.Question[0].Name) + stub.mu.Unlock() + answer := new(dns.Msg) + answer.SetReply(m) + _ = w.WriteMsg(answer) + }) + + started := make(chan struct{}) + srv.NotifyStartedFunc = func() { close(started) } + go func() { _ = srv.ActivateAndServe() }() + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the OS resolver stub to start") + } + + old := osUpstreamConfig + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "OS resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: stub.addr, + Timeout: 2000, + } + t.Cleanup(func() { + osUpstreamConfig = old + _ = srv.Shutdown() + }) + return stub +} + +func (s *osResolverStub) received() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.queries...) +} + +// TestZPADNSEchoBypassReachesOSResolverUncached is the regression test for the +// two ways this route could look correct while still leaving ZPA broken. +// +// 1. It must not be served from ctrld's response cache. Client Connector +// enables Private Access only after it observes the echo answer arrive on +// its own DNS path; an answer we serve locally is silence to it. The nasty +// case is an entry cached while ZPA was disconnected, which would otherwise +// outlive InitializeOsResolver() on reconnect and keep Private Access +// disabled for the entry's whole TTL. +// 2. It must apply to every record type, not just A, matching the Control D +// bypass folder the customer confirmed as a workaround. +// +// It runs against three listener shapes, because the profile bypass list the +// customer is using today arrives as a listener domain rule with empty targets +// (see cfg.Listener rules built from resolverConfig.Exclude). Such a rule +// routes to the OS path on its own, so it would silently keep the cache — a +// machine upgraded while still carrying the workaround must get the fix too. +// +// A pre-existing upstream.os entry with a distinguishable rcode is seeded +// before every query, so a locally served answer is detectable. +func TestZPADNSEchoBypassReachesOSResolverUncached(t *testing.T) { + oldDNS64 := dns64NetworkClassFn + dns64NetworkClassFn = func() (bool, bool, error) { return true, false, nil } + t.Cleanup(func() { dns64NetworkClassFn = oldDNS64 }) + + listeners := []struct { + name string + rules []ctrld.Rule + }{ + {"no-rule", nil}, + {"exact-bypass-rule", []ctrld.Rule{{zpaDNSEchoDomain: []string{}}}}, + {"wildcard-bypass-rule", []ctrld.Rule{{"*.zscaler.com": []string{}}}}, + } + qtypes := []uint16{dns.TypeA, dns.TypeAAAA, dns.TypeTXT, dns.TypeHTTPS, dns.TypeMX} + for _, lcCase := range listeners { + for _, qtype := range qtypes { + for _, bypass := range []bool{true, false} { + name := lcCase.name + "/" + dns.TypeToString[qtype] + if bypass { + name += "/bypass" + } else { + name += "/no-bypass" + } + t.Run(name, func(t *testing.T) { + withZPADNSEchoBypass(t, bypass, true, false) + stub := startOSResolverStub(t) + + cfg := testhelper.SampleConfig(t) + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load())} + p.logger.Store(mainLog.Load()) + for _, nc := range p.cfg.Network { + for _, cidr := range nc.Cidrs { + _, ipNet, err := net.ParseCIDR(cidr) + require.NoError(t, err) + nc.IPNets = append(nc.IPNets, ipNet) + } + } + cache, err := dnscache.NewLRUCache(64) + require.NoError(t, err) + p.cache = cache + + lc := p.cfg.Listener["0"] + if lcCase.rules != nil { + lc = listenerWithRules(lcCase.rules...) + } + + msg := newDnsMsgWithHostname(zpaDNSEchoDomain+".", qtype) + // Every candidate upstream has a fresh REFUSED answer + // waiting. The stub answers NOERROR, so the rcode says + // whether we went to the wire or were served from cache. + seedCache := func() { + for _, upstream := range []string{upstreamOS, "upstream.0", "upstream.1", "upstream.2"} { + cached := new(dns.Msg) + cached.SetRcode(msg, dns.RcodeRefused) + p.cache.Add(dnscache.NewKey(msg, upstream), dnscache.NewValue(cached, time.Now().Add(time.Hour))) + } + } + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + require.NoError(t, err) + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + + // Query twice. The second pass is what a Client Connector + // poll after a ZPA reconnect looks like: if the first answer + // had been cached, this one would never reach the resolver. + for pass := 1; pass <= 2; pass++ { + seedCache() + ufr := p.upstreamFor(ctx, "0", lc, addr, "", zpaDNSEchoDomain) + assert.Equal(t, bypass, ufr.zpaDNSEcho) + res := p.proxy(ctx, &proxyRequest{msg: msg, ufr: ufr}) + + require.NotNil(t, res) + require.NotNil(t, res.answer) + if bypass { + assert.False(t, res.cached, "pass %d: the echo query must not be answered from cache", pass) + assert.Equal(t, dns.RcodeSuccess, res.answer.Rcode, + "pass %d: %s echo query must be answered by the OS resolver", pass, dns.TypeToString[qtype]) + } else { + assert.True(t, res.cached, "pass %d: an ordinary query still uses the cache", pass) + assert.Equal(t, dns.RcodeRefused, res.answer.Rcode, + "pass %d: %s echo query must not reach the OS resolver when the bypass is off", pass, dns.TypeToString[qtype]) + } + } + + got := stub.received() + want := dns.TypeToString[qtype] + " " + zpaDNSEchoDomain + "." + if bypass { + assert.Equal(t, []string{want, want}, got, + "the OS resolver must see the echo query on every poll") + // Write side: nothing may be stored under the OS + // upstream either, or the next poll would be local. + cached := p.cache.Get(dnscache.NewKey(msg, upstreamOS)) + require.NotNil(t, cached, "seeded entry disappeared") + assert.Equal(t, dns.RcodeRefused, cached.Msg.Rcode, + "the echo answer must not be written to the cache") + } else { + assert.Empty(t, got, "the OS resolver must not be queried when the bypass is off") + } + }) + } + } + } +} diff --git a/docs/dns-intercept-mode.md b/docs/dns-intercept-mode.md index 0545b85..738f384 100644 --- a/docs/dns-intercept-mode.md +++ b/docs/dns-intercept-mode.md @@ -483,6 +483,7 @@ exemption). | No VPN | None | ✅ All traffic | N/A | | Split DNS (Tailscale non-exit) | ✅ VPN interface | ✅ Non-VPN domains | ✅ Via MagicDNS | | Exit mode (Tailscale exit node) | ❌ None | ✅ All traffic | ✅ Via ctrld split routing | +| Zscaler Private Access | None | ✅ All traffic except one health-check name | ✅ Via Client Connector | | Windscribe | None (different flow) | ✅ All traffic | N/A | | Hard intercept | None | ✅ All traffic | ❌ Not forwarded | @@ -524,6 +525,71 @@ F5 BIG-IP APM VPN is a known source of DNS conflicts with ctrld (Support ticket - F5's relay proxy (`F5FltSrv`) performs similar functions to ctrld — they are in direct conflict when both active - The seemingly random failure pattern is caused by timing-dependent race conditions between ctrld's watchdog, F5's DNS enforcement, and (optionally) endpoint security inspection +### Zscaler Private Access (macOS) + +Zscaler Client Connector will not synthesize addresses for Private Access +applications until it has confirmed that DNS answers reach it on its own DNS +path. It confirms this by resolving one fixed public name, +`dnsechotest.zscaler.com`, and observing the answer. + +**How the conflict manifests:** + +1. `--intercept-mode dns` is active, so pf redirects port 53 to ctrld +2. Client Connector's query for `dnsechotest.zscaler.com` is redirected into ctrld +3. ctrld answers it from the Control D upstream, so Client Connector never sees the + answer arrive on its own DNS path +4. Client Connector treats domain validation as failed and keeps Private Access + disabled — while ordinary Control D DNS stays perfectly healthy, which is why the + symptom looks like "ZPA is broken" rather than "DNS is broken" + +**How ctrld handles it:** + +On macOS in `dns` mode, ctrld routes that one exact name the same way a Control D +"bypass" rule does: with an empty upstream list, which resolves through +`upstream.os` — the system resolver set, which under Intercept Mode still includes +Client Connector's own DNS servers. Because the decision is made on the name alone, +it covers every record type. + +The answer is **not cached**. Client Connector enables Private Access only after it +observes the echo answer arrive on its own DNS path, so an answer ctrld served from +its cache reads as silence to Client Connector — the query has to reach the OS +resolver on every poll. This matters most on reconnect: an answer cached while ZPA +was disconnected would otherwise outlive `InitializeOsResolver()` and keep Private +Access disabled for the rest of that entry's TTL (or `cache_ttl_override`). + +The scope is deliberately narrow — a single fixed public health-check name: + +- Private Access **application** domains are not special-cased. Once Client Connector + has validated its DNS path it answers them itself, so they never need a ctrld + routing exception; anything it does not claim stays on Control D. +- No interface, resolver range, or pf rule is exempted. There is no ZPA interface + detection, no CGNAT probing, and no passthrough rule. +- Everything else, including every other `zscaler.com` name, stays intercepted and + filtered by Control D. +- `--intercept-mode hard` opts out, the same as it opts out of VPN DNS split + routing: all DNS goes through ctrld and ZPA is not accommodated. +- The route does **not** authorize the query. A `Restricted` listener still answers + only sources that match its policy, so the built-in route is not a hole in + source authorization. +- An explicit listener **domain rule** that routes this name to a specific upstream + outranks the built-in route, and is the supported way to opt out without leaving + Intercept Mode. Network- and MAC-policy targets do not: they route every name + from a source and so state nothing about this one. +- A rule that itself selects the OS path — empty targets, which is how a Control D + profile's bypass list reaches ctrld — keeps its own routing and labels but is still + treated as the health check, so it gets the no-cache behavior too. A machine + upgraded while still carrying the bypass-folder workaround therefore gets the full + fix rather than just the routing half. + +Adding `dnsechotest.zscaler.com` to a Control D bypass folder produces the same +routing and remains a valid workaround on releases without this fix. Disabling +Intercept Mode entirely (`ctrld start --intercept-mode off`) also restores ZPA, at +the cost of interface-based DNS handling. + +**Status:** the routing decision is confirmed by a customer running the bypass-folder +equivalent. It has not yet been verified against a real ZPA tenant with a compiled +build; see the implementation tracker for the outstanding smoke test. + ### Cisco AnyConnect Cisco AnyConnect exhibits similar DNS override behavior. `--intercept-mode dns` mode prevents the conflict by operating at the packet filter level rather than competing for interface DNS settings.