ctrld/cmd/cli/dns_proxy_zpa_test.go

package cli

import (
	"context"
	"net"
	"sync"
	"testing"
	"time"

	"github.com/miekg/dns"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/Control-D-Inc/ctrld"
	"github.com/Control-D-Inc/ctrld/internal/dnscache"
	"github.com/Control-D-Inc/ctrld/testhelper"
)

// withZPADNSEchoBypass sets the three gates that guard the built-in Zscaler
// DNS echo bypass: the macOS-only platform gate, Intercept Mode, and the
// hard-mode opt-out.
func withZPADNSEchoBypass(t *testing.T, onDarwin, intercept, hard bool) {
	t.Helper()
	oldBypass, oldIntercept, oldHard := zpaDNSEchoBypassEnabled, dnsIntercept, hardIntercept
	zpaDNSEchoBypassEnabled = onDarwin
	dnsIntercept = intercept
	hardIntercept = hard
	t.Cleanup(func() {
		zpaDNSEchoBypassEnabled = oldBypass
		dnsIntercept = oldIntercept
		hardIntercept = oldHard
	})
}

// listenerWithRules builds a listener whose policy carries only domain rules.
func listenerWithRules(rules ...ctrld.Rule) *ctrld.ListenerConfig {
	return &ctrld.ListenerConfig{
		IP:   "127.0.0.1",
		Port: 53,
		Policy: &ctrld.ListenerPolicyConfig{
			Name:  "Test Policy",
			Rules: rules,
		},
	}
}

// Test_prog_upstreamFor_ZPADNSEchoBypass pins the routing decision for issue
// 601: under macOS Intercept Mode the exact Zscaler health-check name — and
// nothing else — is routed like a Control D bypass rule, i.e. with an empty
// upstream list that proxy() then resolves through upstream.os.
func Test_prog_upstreamFor_ZPADNSEchoBypass(t *testing.T) {
	cfg := testhelper.SampleConfig(t)
	p := &prog{cfg: cfg}
	p.logger.Store(mainLog.Load())
	for _, nc := range p.cfg.Network {
		for _, cidr := range nc.Cidrs {
			_, ipNet, err := net.ParseCIDR(cidr)
			require.NoError(t, err)
			nc.IPNets = append(nc.IPNets, ipNet)
		}
	}

	// A listener whose policy routes the echo name to a specific non-OS
	// upstream. That is a deliberate route for this exact name, so it wins.
	lcWithEchoRule := listenerWithRules(ctrld.Rule{zpaDNSEchoDomain: []string{"upstream.2"}})
	lcWithZscalerWildcardRule := listenerWithRules(ctrld.Rule{"*.zscaler.com": []string{"upstream.2"}})

	// Empty targets are how a Control D profile's bypass list reaches ctrld
	// (cfg.Listener rules built from resolverConfig.Exclude). This is the
	// documented pre-fix workaround, so it must keep routing to the OS path AND
	// stay classified as the health check, or upgrading a machine that still
	// carries the workaround would lose the no-cache behavior.
	lcWithEchoBypassRule := listenerWithRules(ctrld.Rule{zpaDNSEchoDomain: []string{}})
	lcWithZscalerWildcardBypassRule := listenerWithRules(ctrld.Rule{"*.zscaler.com": []string{}})

	// network.0 targets, i.e. what an un-bypassed query from 192.168.0.1 gets.
	notBypassed := []string{"upstream.1", "upstream.0"}
	const (
		inNetwork0  = "192.168.0.1:0" // matches network.0, so listener.0's policy matches
		outOfPolicy = "10.0.0.1:0"    // matches no configured network
	)

	tests := []struct {
		name          string
		onDarwin      bool
		intercept     bool
		hard          bool
		lc            *ctrld.ListenerConfig
		defaultNum    string
		srcAddr       string
		domain        string
		wantUpstreams []string
		wantMatched   bool
		wantZPAEcho   bool
	}{
		{"exact echo name routes to the OS resolver path", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, nil, true, true},
		{"echo name is matched case-insensitively and with a trailing dot", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "DNSEchoTest.ZScaler.COM.", nil, true, true},
		{"echo name is bypassed on a listener with no policy", true, true, false, p.cfg.Listener["1"], "1", inNetwork0, zpaDNSEchoDomain, nil, false, true},

		// The bypass must not double as source authorization. serveDNS refuses a
		// query on a Restricted listener when matched is false, so an
		// unauthorized source must stay unmatched even for this name — while
		// still being routed to the OS resolver path if the listener does serve
		// it. See TestZPADNSEchoBypassDoesNotAuthorizeRestrictedListener.
		{"unauthorized source is still unmatched", true, true, false, p.cfg.Listener["0"], "0", outOfPolicy, zpaDNSEchoDomain, nil, false, true},

		// An explicit domain rule routing the name to a specific upstream
		// outranks the built-in route; network/MAC policy targets, which say
		// nothing about this name, do not.
		{"explicit domain rule for the name wins", true, true, false, lcWithEchoRule, "0", inNetwork0, zpaDNSEchoDomain, []string{"upstream.2"}, true, false},
		{"explicit wildcard rule covering the name wins", true, true, false, lcWithZscalerWildcardRule, "0", inNetwork0, zpaDNSEchoDomain, []string{"upstream.2"}, true, false},

		// An explicit rule that itself selects the OS path (empty targets, i.e.
		// the Control D bypass-list shape) keeps its own labels but is still the
		// health check, so proxy() keeps it out of the cache.
		{"existing exact bypass rule stays classified as the health check", true, true, false, lcWithEchoBypassRule, "0", inNetwork0, zpaDNSEchoDomain, []string{}, true, true},
		{"existing wildcard bypass rule stays classified as the health check", true, true, false, lcWithZscalerWildcardBypassRule, "0", inNetwork0, zpaDNSEchoDomain, []string{}, true, true},

		// Gating: the bypass exists only where the problem does.
		{"not bypassed off macOS", false, true, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, notBypassed, true, false},
		{"not bypassed outside Intercept Mode", true, false, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, notBypassed, true, false},
		{"not bypassed in hard intercept mode", true, true, true, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, notBypassed, true, false},

		// Scope: only the one exact name, so no neighbouring name loses filtering.
		{"subdomain of the echo name is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "foo." + zpaDNSEchoDomain, notBypassed, true, false},
		{"echo name as a prefix of another domain is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain + ".evil.example", notBypassed, true, false},
		{"name that merely ends with the echo name is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "evil" + zpaDNSEchoDomain, notBypassed, true, false},
		{"parent Zscaler domain is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "zscaler.com", notBypassed, true, false},
		{"ordinary domain is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "example.com", notBypassed, true, false},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			withZPADNSEchoBypass(t, tc.onDarwin, tc.intercept, tc.hard)
			addr, err := net.ResolveUDPAddr("udp", tc.srcAddr)
			require.NoError(t, err)
			ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())

			ufr := p.upstreamFor(ctx, tc.defaultNum, tc.lc, addr, "", tc.domain)

			assert.Equal(t, tc.wantUpstreams, ufr.upstreams)
			assert.Equal(t, tc.wantMatched, ufr.matched, "matched is the source-authorization bit")
			assert.Equal(t, tc.wantZPAEcho, ufr.zpaDNSEcho)
			if tc.wantZPAEcho {
				// An empty upstream list is the whole mechanism: it is what
				// proxy() turns into upstream.os. Guard the translation step
				// that makes that substitution fire.
				assert.Empty(t, p.upstreamConfigsFromUpstreamNumbers(ufr.upstreams))
				if tc.lc.Policy == nil || len(tc.lc.Policy.Rules) == 0 {
					// Only relabelled when the built-in route is the one that
					// made the decision; an operator's own matching rule keeps
					// its own labels.
					assert.Equal(t, zpaDNSEchoPolicyName, ufr.matchedPolicy)
					assert.Equal(t, zpaDNSEchoDomain, ufr.matchedRule)
				}
			}
		})
	}
}

// TestZPADNSEchoBypassDoesNotAuthorizeRestrictedListener pins the serveDNS
// consequence of keeping `matched` out of the bypass: a Restricted listener
// answers only sources that match its policy, and the built-in echo route must
// not become a hole in that for every macOS DNS-intercept instance.
func TestZPADNSEchoBypassDoesNotAuthorizeRestrictedListener(t *testing.T) {
	withZPADNSEchoBypass(t, true, true, false)

	cfg := testhelper.SampleConfig(t)
	p := &prog{cfg: cfg}
	p.logger.Store(mainLog.Load())
	for _, nc := range p.cfg.Network {
		for _, cidr := range nc.Cidrs {
			_, ipNet, err := net.ParseCIDR(cidr)
			require.NoError(t, err)
			nc.IPNets = append(nc.IPNets, ipNet)
		}
	}
	lc := p.cfg.Listener["0"]
	lc.Restricted = true

	ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
	for _, tc := range []struct {
		name        string
		srcAddr     string
		wantRefused bool
	}{
		{"source outside every network policy is refused", "10.0.0.1:0", true},
		{"source inside network.0 is served", "192.168.0.1:0", false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			addr, err := net.ResolveUDPAddr("udp", tc.srcAddr)
			require.NoError(t, err)
			ufr := p.upstreamFor(ctx, "0", lc, addr, "", zpaDNSEchoDomain)

			// The route itself is unconditional; only the authorization differs.
			require.True(t, ufr.zpaDNSEcho)
			// This is the exact condition serveDNS uses to answer REFUSED.
			assert.Equal(t, tc.wantRefused, !ufr.matched && lc.Restricted)
		})
	}
}

// osResolverStub stands in for the OS resolver during a proxy() test. It
// answers NOERROR for anything, and records what it was actually asked, so a
// test can tell an answer that came off the wire from one served locally.
type osResolverStub struct {
	addr string

	mu      sync.Mutex
	queries []string
}

// startOSResolverStub points osUpstreamConfig at a local DNS server for the
// duration of the test. It uses the legacy (plain UDP) resolver type because
// ResolverTypeOS resolves through process-wide state in the ctrld package that
// a test in this package cannot address.
func startOSResolverStub(t *testing.T) *osResolverStub {
	t.Helper()

	pc, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)})
	require.NoError(t, err)

	stub := &osResolverStub{addr: pc.LocalAddr().String()}
	srv := &dns.Server{PacketConn: pc, Net: "udp"}
	srv.Handler = dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) {
		stub.mu.Lock()
		stub.queries = append(stub.queries, dns.TypeToString[m.Question[0].Qtype]+" "+m.Question[0].Name)
		stub.mu.Unlock()
		answer := new(dns.Msg)
		answer.SetReply(m)
		_ = w.WriteMsg(answer)
	})

	started := make(chan struct{})
	srv.NotifyStartedFunc = func() { close(started) }
	go func() { _ = srv.ActivateAndServe() }()
	select {
	case <-started:
	case <-time.After(5 * time.Second):
		t.Fatal("timed out waiting for the OS resolver stub to start")
	}

	old := osUpstreamConfig
	osUpstreamConfig = &ctrld.UpstreamConfig{
		Name:     "OS resolver",
		Type:     ctrld.ResolverTypeLegacy,
		Endpoint: stub.addr,
		Timeout:  2000,
	}
	t.Cleanup(func() {
		osUpstreamConfig = old
		_ = srv.Shutdown()
	})
	return stub
}

func (s *osResolverStub) received() []string {
	s.mu.Lock()
	defer s.mu.Unlock()
	return append([]string(nil), s.queries...)
}

// TestZPADNSEchoBypassReachesOSResolverUncached is the regression test for the
// two ways this route could look correct while still leaving ZPA broken.
//
//  1. It must not be served from ctrld's response cache. Client Connector
//     enables Private Access only after it observes the echo answer arrive on
//     its own DNS path; an answer we serve locally is silence to it. The nasty
//     case is an entry cached while ZPA was disconnected, which would otherwise
//     outlive InitializeOsResolver() on reconnect and keep Private Access
//     disabled for the entry's whole TTL.
//  2. It must apply to every record type, not just A, matching the Control D
//     bypass folder the customer confirmed as a workaround.
//
// It runs against three listener shapes, because the profile bypass list the
// customer is using today arrives as a listener domain rule with empty targets
// (see cfg.Listener rules built from resolverConfig.Exclude). Such a rule
// routes to the OS path on its own, so it would silently keep the cache — a
// machine upgraded while still carrying the workaround must get the fix too.
//
// A pre-existing upstream.os entry with a distinguishable rcode is seeded
// before every query, so a locally served answer is detectable.
func TestZPADNSEchoBypassReachesOSResolverUncached(t *testing.T) {
	oldDNS64 := dns64NetworkClassFn
	dns64NetworkClassFn = func() (bool, bool, error) { return true, false, nil }
	t.Cleanup(func() { dns64NetworkClassFn = oldDNS64 })

	listeners := []struct {
		name  string
		rules []ctrld.Rule
	}{
		{"no-rule", nil},
		{"exact-bypass-rule", []ctrld.Rule{{zpaDNSEchoDomain: []string{}}}},
		{"wildcard-bypass-rule", []ctrld.Rule{{"*.zscaler.com": []string{}}}},
	}
	qtypes := []uint16{dns.TypeA, dns.TypeAAAA, dns.TypeTXT, dns.TypeHTTPS, dns.TypeMX}
	for _, lcCase := range listeners {
		for _, qtype := range qtypes {
			for _, bypass := range []bool{true, false} {
				name := lcCase.name + "/" + dns.TypeToString[qtype]
				if bypass {
					name += "/bypass"
				} else {
					name += "/no-bypass"
				}
				t.Run(name, func(t *testing.T) {
					withZPADNSEchoBypass(t, bypass, true, false)
					stub := startOSResolverStub(t)

					cfg := testhelper.SampleConfig(t)
					cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false)
					p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load())}
					p.logger.Store(mainLog.Load())
					for _, nc := range p.cfg.Network {
						for _, cidr := range nc.Cidrs {
							_, ipNet, err := net.ParseCIDR(cidr)
							require.NoError(t, err)
							nc.IPNets = append(nc.IPNets, ipNet)
						}
					}
					cache, err := dnscache.NewLRUCache(64)
					require.NoError(t, err)
					p.cache = cache

					lc := p.cfg.Listener["0"]
					if lcCase.rules != nil {
						lc = listenerWithRules(lcCase.rules...)
					}

					msg := newDnsMsgWithHostname(zpaDNSEchoDomain+".", qtype)
					// Every candidate upstream has a fresh REFUSED answer
					// waiting. The stub answers NOERROR, so the rcode says
					// whether we went to the wire or were served from cache.
					seedCache := func() {
						for _, upstream := range []string{upstreamOS, "upstream.0", "upstream.1", "upstream.2"} {
							cached := new(dns.Msg)
							cached.SetRcode(msg, dns.RcodeRefused)
							p.cache.Add(dnscache.NewKey(msg, upstream), dnscache.NewValue(cached, time.Now().Add(time.Hour)))
						}
					}

					addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
					require.NoError(t, err)
					ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())

					// Query twice. The second pass is what a Client Connector
					// poll after a ZPA reconnect looks like: if the first answer
					// had been cached, this one would never reach the resolver.
					for pass := 1; pass <= 2; pass++ {
						seedCache()
						ufr := p.upstreamFor(ctx, "0", lc, addr, "", zpaDNSEchoDomain)
						assert.Equal(t, bypass, ufr.zpaDNSEcho)
						res := p.proxy(ctx, &proxyRequest{msg: msg, ufr: ufr})

						require.NotNil(t, res)
						require.NotNil(t, res.answer)
						if bypass {
							assert.False(t, res.cached, "pass %d: the echo query must not be answered from cache", pass)
							assert.Equal(t, dns.RcodeSuccess, res.answer.Rcode,
								"pass %d: %s echo query must be answered by the OS resolver", pass, dns.TypeToString[qtype])
						} else {
							assert.True(t, res.cached, "pass %d: an ordinary query still uses the cache", pass)
							assert.Equal(t, dns.RcodeRefused, res.answer.Rcode,
								"pass %d: %s echo query must not reach the OS resolver when the bypass is off", pass, dns.TypeToString[qtype])
						}
					}

					got := stub.received()
					want := dns.TypeToString[qtype] + " " + zpaDNSEchoDomain + "."
					if bypass {
						assert.Equal(t, []string{want, want}, got,
							"the OS resolver must see the echo query on every poll")
						// Write side: nothing may be stored under the OS
						// upstream either, or the next poll would be local.
						cached := p.cache.Get(dnscache.NewKey(msg, upstreamOS))
						require.NotNil(t, cached, "seeded entry disappeared")
						assert.Equal(t, dns.RcodeRefused, cached.Msg.Rcode,
							"the echo answer must not be written to the cache")
					} else {
						assert.Empty(t, got, "the OS resolver must not be queried when the bypass is off")
					}
				})
			}
		}
	}
}