Merge pull request #155 from Control-D-Inc/release-branch-v1.3.7

Release branch v1.3.7

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ jobs:
       fail-fast: false
       matrix:
         os: ["windows-latest", "ubuntu-latest", "macOS-latest"]
-        go: ["1.20.x"]
+        go: ["1.21.x"]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3

.gitignore

@@ -3,3 +3,12 @@ gon.hcl
 
 /Build
 .DS_Store
+
+# Release folder
+dist/
+
+# Binaries
+ctrld-*
+
+# generated file
+cmd/cli/rsrc_*.syso

README.md

@@ -4,13 +4,16 @@
 [![Go Reference](https://pkg.go.dev/badge/github.com/Control-D-Inc/ctrld.svg)](https://pkg.go.dev/github.com/Control-D-Inc/ctrld)
 [![Go Report Card](https://goreportcard.com/badge/github.com/Control-D-Inc/ctrld)](https://goreportcard.com/report/github.com/Control-D-Inc/ctrld)
 
+![ctrld spash image](/docs/ctrldsplash.png)
+
 A highly configurable DNS forwarding proxy with support for:
 - Multiple listeners for incoming queries
 - Multiple upstreams with fallbacks
 - Multiple network policy driven DNS query steering
 - Policy driven domain based "split horizon" DNS with wildcard support
 - Integrations with common router vendors and firmware
-- LAN client discovery via DHCP, mDNS, and ARP
+- LAN client discovery via DHCP, mDNS, ARP, NDP, hosts file parsing
+- Prometheus metrics exporter 
 
 ## TLDR
 Proxy legacy DNS traffic to secure DNS upstreams in highly configurable ways. 
@@ -61,7 +64,7 @@ $ docker pull controldns/ctrld
 Alternatively, if you know what you're doing you can download pre-compiled binaries from the [Releases](https://github.com/Control-D-Inc/ctrld/releases) section for the appropriate platform. 
 
 ## Build
-Lastly, you can build `ctrld` from source which requires `go1.19+`:
+Lastly, you can build `ctrld` from source which requires `go1.21+`:
 
 ```shell
 $ go build ./cmd/ctrld
@@ -232,7 +235,6 @@ See [Contribution Guideline](./docs/contributing.md)
 
 ## Roadmap
 The following functionality is on the roadmap and will be available in future releases. 
-- Prometheus metrics exporter 
 - DNS intercept mode
 - Direct listener mode
 - Support for more routers (let us know which ones)

cmd/cli/cli_test.go

@@ -21,3 +21,26 @@ func Test_writeConfigFile(t *testing.T) {
 	_, err = os.Stat(configPath)
 	require.NoError(t, err)
 }
+
+func Test_isStableVersion(t *testing.T) {
+	tests := []struct {
+		name     string
+		ver      string
+		isStable bool
+	}{
+		{"stable", "v1.3.5", true},
+		{"pre", "v1.3.5-next", false},
+		{"pre with commit hash", "v1.3.5-next-asdf", false},
+		{"dev", "dev", false},
+		{"empty", "dev", false},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := isStableVersion(tc.ver); got != tc.isStable {
+				t.Errorf("unexpected result for %s, want: %v, got: %v", tc.ver, tc.isStable, got)
+			}
+		})
+	}
+}

cmd/cli/control_client.go

@@ -27,3 +27,8 @@ func newControlClient(addr string) *controlClient {
 func (c *controlClient) post(path string, data io.Reader) (*http.Response, error) {
 	return c.c.Post("http://unix"+path, contentTypeJson, data)
 }
+
+// deactivationRequest represents request for validating deactivation pin.
+type deactivationRequest struct {
+	Pin int64 `json:"pin"`
+}

cmd/cli/control_server.go

@@ -10,16 +10,21 @@ import (
 	"sort"
 	"time"
 
+	"github.com/kardianos/service"
+
 	dto "github.com/prometheus/client_model/go"
 
 	"github.com/Control-D-Inc/ctrld"
 )
 
 const (
-	contentTypeJson = "application/json"
-	listClientsPath = "/clients"
-	startedPath     = "/started"
-	reloadPath      = "/reload"
+	contentTypeJson  = "application/json"
+	listClientsPath  = "/clients"
+	startedPath      = "/started"
+	reloadPath       = "/reload"
+	deactivationPath = "/deactivation"
+	cdPath           = "/cd"
+	ifacePath        = "/iface"
 )
 
 type controlServer struct {
@@ -146,6 +151,48 @@ func (p *prog) registerControlServerHandler() {
 		// Otherwise, reload is done.
 		w.WriteHeader(http.StatusOK)
 	}))
+	p.cs.register(deactivationPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		// Non-cd mode or pin code not set, always allowing deactivation.
+		if cdUID == "" || deactivationPinNotSet() {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+
+		var req deactivationRequest
+		if err := json.NewDecoder(request.Body).Decode(&req); err != nil {
+			w.WriteHeader(http.StatusPreconditionFailed)
+			mainLog.Load().Err(err).Msg("invalid deactivation request")
+			return
+		}
+
+		code := http.StatusForbidden
+		switch req.Pin {
+		case cdDeactivationPin:
+			code = http.StatusOK
+		case defaultDeactivationPin:
+			// If the pin code was set, but users do not provide --pin, return proper code to client.
+			code = http.StatusBadRequest
+		}
+		w.WriteHeader(code)
+	}))
+	p.cs.register(cdPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		if cdUID != "" {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		w.WriteHeader(http.StatusBadRequest)
+	}))
+	p.cs.register(ifacePath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		// p.setDNS is only called when running as a service
+		if !service.Interactive() {
+			<-p.csSetDnsDone
+			if p.csSetDnsOk {
+				w.Write([]byte(iface))
+				return
+			}
+		}
+		w.WriteHeader(http.StatusBadRequest)
+	}))
 }
 
 func jsonResponse(next http.Handler) http.Handler {

cmd/cli/dns_proxy.go

@@ -101,6 +101,15 @@ func (p *prog) serveDNS(listenerNum string) error {
 		go p.detectLoop(m)
 		q := m.Question[0]
 		domain := canonicalName(q.Name)
+		if domain == selfCheckInternalTestDomain {
+			answer := resolveInternalDomainTestQuery(ctx, domain, m)
+			_ = w.WriteMsg(answer)
+			return
+		}
+		if _, ok := p.cacheFlushDomainsMap[domain]; ok && p.cache != nil {
+			p.cache.Purge()
+			ctrld.Log(ctx, mainLog.Load().Debug(), "received query %q, local cache is purged", domain)
+		}
 		remoteIP, _, _ := net.SplitHostPort(w.RemoteAddr().String())
 		ci := p.getClientInfo(remoteIP, m)
 		ci.ClientIDPref = p.cfg.Service.ClientIDPref
@@ -201,12 +210,9 @@ func (p *prog) serveDNS(listenerNum string) error {
 			addr := net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port))
 			s, errCh := runDNSServer(addr, proto, handler)
 			defer s.Shutdown()
-			select {
-			case err := <-errCh:
-				return err
-			case <-time.After(5 * time.Second):
-				p.started <- struct{}{}
-			}
+
+			p.started <- struct{}{}
+
 			select {
 			case <-p.stopCh:
 			case <-ctx.Done():
@@ -282,7 +288,7 @@ networkRules:
 macRules:
 	for _, rule := range lc.Policy.Macs {
 		for source, targets := range rule {
-			if source != "" && strings.EqualFold(source, srcMac) {
+			if source != "" && (strings.EqualFold(source, srcMac) || wildcardMatches(strings.ToLower(source), strings.ToLower(srcMac))) {
 				matchedPolicy = lc.Policy.Name
 				matchedNetwork = source
 				networkTargets = targets
@@ -590,7 +596,8 @@ func canonicalName(fqdn string) string {
 	return q
 }
 
-func wildcardMatches(wildcard, domain string) bool {
+// wildcardMatches reports whether string str matches the wildcard pattern.
+func wildcardMatches(wildcard, str string) bool {
 	// Wildcard match.
 	wildCardParts := strings.Split(wildcard, "*")
 	if len(wildCardParts) != 2 {
@@ -600,15 +607,15 @@ func wildcardMatches(wildcard, domain string) bool {
 	switch {
 	case len(wildCardParts[0]) > 0 && len(wildCardParts[1]) > 0:
 		// Domain must match both prefix and suffix.
-		return strings.HasPrefix(domain, wildCardParts[0]) && strings.HasSuffix(domain, wildCardParts[1])
+		return strings.HasPrefix(str, wildCardParts[0]) && strings.HasSuffix(str, wildCardParts[1])
 
 	case len(wildCardParts[1]) > 0:
 		// Only suffix must match.
-		return strings.HasSuffix(domain, wildCardParts[1])
+		return strings.HasSuffix(str, wildCardParts[1])
 
 	case len(wildCardParts[0]) > 0:
 		// Only prefix must match.
-		return strings.HasPrefix(domain, wildCardParts[0])
+		return strings.HasPrefix(str, wildCardParts[0])
 	}
 
 	return false
@@ -742,20 +749,19 @@ func runDNSServer(addr, network string, handler dns.Handler) (*dns.Server, <-cha
 		Handler: handler,
 	}
 
-	waitLock := sync.Mutex{}
-	waitLock.Lock()
-	s.NotifyStartedFunc = waitLock.Unlock
+	startedCh := make(chan struct{})
+	s.NotifyStartedFunc = func() { sync.OnceFunc(func() { close(startedCh) })() }
 
 	errCh := make(chan error)
 	go func() {
 		defer close(errCh)
 		if err := s.ListenAndServe(); err != nil {
-			waitLock.Unlock()
+			s.NotifyStartedFunc()
 			mainLog.Load().Error().Err(err).Msgf("could not listen and serve on: %s", s.Addr)
 			errCh <- err
 		}
 	}()
-	waitLock.Lock()
+	<-startedCh
 	return s, errCh
 }
 
@@ -806,6 +812,13 @@ func (p *prog) getClientInfo(remoteIP string, msg *dns.Msg) *ctrld.ClientInfo {
 		ci.Hostname = p.ciTable.LookupHostname(ci.IP, ci.Mac)
 	}
 	ci.Self = queryFromSelf(ci.IP)
+	// If this is a query from self, but ci.IP is not loopback IP,
+	// try using hostname mapping for lookback IP if presents.
+	if ci.Self {
+		if name := p.ciTable.LocalHostname(); name != "" {
+			ci.Hostname = name
+		}
+	}
 	p.spoofLoopbackIpInClientInfo(ci)
 	return ci
 }
@@ -936,3 +949,21 @@ func isWanClient(na net.Addr) bool {
 		!ip.IsLinkLocalMulticast() &&
 		!tsaddr.CGNATRange().Contains(ip)
 }
+
+// resolveInternalDomainTestQuery resolves internal test domain query, returning the answer to the caller.
+func resolveInternalDomainTestQuery(ctx context.Context, domain string, m *dns.Msg) *dns.Msg {
+	ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain test query")
+
+	q := m.Question[0]
+	answer := new(dns.Msg)
+	rrStr := fmt.Sprintf("%s A %s", domain, net.IPv4zero)
+	if q.Qtype == dns.TypeAAAA {
+		rrStr = fmt.Sprintf("%s AAAA %s", domain, net.IPv6zero)
+	}
+	rr, err := dns.NewRR(rrStr)
+	if err == nil {
+		answer.Answer = append(answer.Answer, rr)
+	}
+	answer.SetReply(m)
+	return answer
+}

cmd/cli/dns_proxy_test.go

@@ -22,14 +22,21 @@ func Test_wildcardMatches(t *testing.T) {
 		domain   string
 		match    bool
 	}{
-		{"prefix parent should not match", "*.windscribe.com", "windscribe.com", false},
-		{"prefix", "*.windscribe.com", "anything.windscribe.com", true},
-		{"prefix not match other domain", "*.windscribe.com", "example.com", false},
-		{"prefix not match domain in name", "*.windscribe.com", "wwindscribe.com", false},
-		{"suffix", "suffix.*", "suffix.windscribe.com", true},
-		{"suffix not match other", "suffix.*", "suffix1.windscribe.com", false},
-		{"both", "suffix.*.windscribe.com", "suffix.anything.windscribe.com", true},
-		{"both not match", "suffix.*.windscribe.com", "suffix1.suffix.windscribe.com", false},
+		{"domain - prefix parent should not match", "*.windscribe.com", "windscribe.com", false},
+		{"domain - prefix", "*.windscribe.com", "anything.windscribe.com", true},
+		{"domain - prefix not match other s", "*.windscribe.com", "example.com", false},
+		{"domain - prefix not match s in name", "*.windscribe.com", "wwindscribe.com", false},
+		{"domain - suffix", "suffix.*", "suffix.windscribe.com", true},
+		{"domain - suffix not match other", "suffix.*", "suffix1.windscribe.com", false},
+		{"domain - both", "suffix.*.windscribe.com", "suffix.anything.windscribe.com", true},
+		{"domain - both not match", "suffix.*.windscribe.com", "suffix1.suffix.windscribe.com", false},
+		{"mac - prefix", "*:98:05:b4:2b", "d4:67:98:05:b4:2b", true},
+		{"mac - prefix not match other s", "*:98:05:b4:2b", "0d:ba:54:09:94:2c", false},
+		{"mac - prefix not match s in name", "*:98:05:b4:2b", "e4:67:97:05:b4:2b", false},
+		{"mac - suffix", "d4:67:98:*", "d4:67:98:05:b4:2b", true},
+		{"mac - suffix not match other", "d4:67:98:*", "d4:67:97:15:b4:2b", false},
+		{"mac - both", "d4:67:98:*:b4:2b", "d4:67:98:05:b4:2b", true},
+		{"mac - both not match", "d4:67:98:*:b4:2b", "d4:67:97:05:c4:2b", false},
 	}
 
 	for _, tc := range tests {

cmd/cli/loop.go

@@ -105,6 +105,10 @@ func (p *prog) checkDnsLoop() {
 	for uid := range p.loop {
 		msg := loopTestMsg(uid)
 		uc := upstream[uid]
+		// Skipping upstream which is being marked as down.
+		if uc == nil {
+			continue
+		}
 		resolver, err := ctrld.NewResolver(uc)
 		if err != nil {
 			mainLog.Load().Warn().Err(err).Msgf("could not perform loop check for upstream: %q, endpoint: %q", uc.Name, uc.Endpoint)

cmd/cli/main.go

@@ -34,6 +34,8 @@ var (
 	ifaceStartStop    string
 	nextdns           string
 	cdUpstreamProto   string
+	deactivationPin   int64
+	skipSelfChecks    bool
 
 	mainLog       atomic.Pointer[zerolog.Logger]
 	consoleWriter zerolog.ConsoleWriter

cmd/cli/net.go

@@ -0,0 +1,34 @@
+package cli
+
+import "strings"
+
+// Copied from https://gist.github.com/Ultraporing/fe52981f678be6831f747c206a4861cb
+
+// Mac Address parts to look for, and identify non-physical devices. There may be more, update me!
+var macAddrPartsToFilter = []string{
+	"00:03:FF",       // Microsoft Hyper-V, Virtual Server, Virtual PC
+	"0A:00:27",       // VirtualBox
+	"00:00:00:00:00", // Teredo Tunneling Pseudo-Interface
+	"00:50:56",       // VMware ESX 3, Server, Workstation, Player
+	"00:1C:14",       // VMware ESX 3, Server, Workstation, Player
+	"00:0C:29",       // VMware ESX 3, Server, Workstation, Player
+	"00:05:69",       // VMware ESX 3, Server, Workstation, Player
+	"00:1C:42",       // Microsoft Hyper-V, Virtual Server, Virtual PC
+	"00:0F:4B",       // Virtual Iron 4
+	"00:16:3E",       // Red Hat Xen, Oracle VM, XenSource, Novell Xen
+	"08:00:27",       // Sun xVM VirtualBox
+	"7A:79",          // Hamachi
+}
+
+// Filters the possible physical interface address by comparing it to known popular VM Software addresses
+// and Teredo Tunneling Pseudo-Interface.
+//
+//lint:ignore U1000 use in net_windows.go
+func isPhysicalInterface(addr string) bool {
+	for _, macPart := range macAddrPartsToFilter {
+		if strings.HasPrefix(strings.ToLower(addr), strings.ToLower(macPart)) {
+			return false
+		}
+	}
+	return true
+}

cmd/cli/net_darwin.go

@@ -42,3 +42,21 @@ func networkServiceName(ifaceName string, r io.Reader) string {
 	}
 	return ""
 }
+
+// validInterface reports whether the *net.Interface is a valid one, which includes:
+//
+// - en0: physical wireless
+// - en1: Thunderbolt 1
+// - en2: Thunderbolt 2
+// - en3: Thunderbolt 3
+// - en4: Thunderbolt 4
+//
+// For full list, see: https://unix.stackexchange.com/questions/603506/what-are-these-ifconfig-interfaces-on-macos
+func validInterface(iface *net.Interface) bool {
+	switch iface.Name {
+	case "en0", "en1", "en2", "en3", "en4":
+		return true
+	default:
+		return false
+	}
+}

cmd/cli/net_others.go

@@ -1,7 +1,9 @@
-//go:build !darwin
+//go:build !darwin && !windows
 
 package cli
 
 import "net"
 
 func patchNetIfaceName(iface *net.Interface) error { return nil }
+
+func validInterface(iface *net.Interface) bool { return true }

cmd/cli/net_windows.go

@@ -0,0 +1,21 @@
+package cli
+
+import (
+	"net"
+)
+
+func patchNetIfaceName(iface *net.Interface) error {
+	return nil
+}
+
+// validInterface reports whether the *net.Interface is a valid one.
+// On Windows, only physical interfaces are considered valid.
+func validInterface(iface *net.Interface) bool {
+	if iface == nil {
+		return false
+	}
+	if isPhysicalInterface(iface.HardwareAddr.String()) {
+		return true
+	}
+	return false
+}

cmd/cli/os_darwin.go

@@ -1,8 +1,12 @@
 package cli
 
 import (
+	"bufio"
+	"bytes"
+	"fmt"
 	"net"
 	"os/exec"
+	"strings"
 
 	"github.com/Control-D-Inc/ctrld/internal/resolvconffile"
 )
@@ -27,6 +31,18 @@ func deAllocateIP(ip string) error {
 	return nil
 }
 
+// setDnsIgnoreUnusableInterface likes setDNS, but return a nil error if the interface is not usable.
+func setDnsIgnoreUnusableInterface(iface *net.Interface, nameservers []string) error {
+	if err := setDNS(iface, nameservers); err != nil {
+		// TODO: investiate whether we can detect this without relying on error message.
+		if strings.Contains(err.Error(), " is not a recognized network service") {
+			return nil
+		}
+		return err
+	}
+	return nil
+}
+
 // set the dns server for the provided network interface
 // networksetup -setdnsservers Wi-Fi 8.8.8.8 1.1.1.1
 // TODO(cuonglm): use system API
@@ -34,9 +50,19 @@ func setDNS(iface *net.Interface, nameservers []string) error {
 	cmd := "networksetup"
 	args := []string{"-setdnsservers", iface.Name}
 	args = append(args, nameservers...)
+	if out, err := exec.Command(cmd, args...).CombinedOutput(); err != nil {
+		return fmt.Errorf("%v: %w", string(out), err)
+	}
+	return nil
+}
 
-	if err := exec.Command(cmd, args...).Run(); err != nil {
-		mainLog.Load().Error().Err(err).Msgf("setDNS failed, ips = %q", nameservers)
+// resetDnsIgnoreUnusableInterface likes resetDNS, but return a nil error if the interface is not usable.
+func resetDnsIgnoreUnusableInterface(iface *net.Interface) error {
+	if err := resetDNS(iface); err != nil {
+		// TODO: investiate whether we can detect this without relying on error message.
+		if strings.Contains(err.Error(), " is not a recognized network service") {
+			return nil
+		}
 		return err
 	}
 	return nil
@@ -44,12 +70,15 @@ func setDNS(iface *net.Interface, nameservers []string) error {
 
 // TODO(cuonglm): use system API
 func resetDNS(iface *net.Interface) error {
+	if ns := savedStaticNameservers(iface); len(ns) > 0 {
+		if err := setDNS(iface, ns); err == nil {
+			return nil
+		}
+	}
 	cmd := "networksetup"
 	args := []string{"-setdnsservers", iface.Name, "empty"}
-
-	if err := exec.Command(cmd, args...).Run(); err != nil {
-		mainLog.Load().Error().Err(err).Msgf("resetDNS failed")
-		return err
+	if out, err := exec.Command(cmd, args...).CombinedOutput(); err != nil {
+		return fmt.Errorf("%v: %w", string(out), err)
 	}
 	return nil
 }
@@ -57,3 +86,22 @@ func resetDNS(iface *net.Interface) error {
 func currentDNS(_ *net.Interface) []string {
 	return resolvconffile.NameServers("")
 }
+
+// currentStaticDNS returns the current static DNS settings of given interface.
+func currentStaticDNS(iface *net.Interface) ([]string, error) {
+	cmd := "networksetup"
+	args := []string{"-getdnsservers", iface.Name}
+	out, err := exec.Command(cmd, args...).Output()
+	if err != nil {
+		return nil, err
+	}
+	scanner := bufio.NewScanner(bytes.NewReader(out))
+	var ns []string
+	for scanner.Scan() {
+		line := scanner.Text()
+		if ip := net.ParseIP(line); ip != nil {
+			ns = append(ns, ip.String())
+		}
+	}
+	return ns, nil
+}

cmd/cli/os_freebsd.go

@@ -29,6 +29,11 @@ func deAllocateIP(ip string) error {
 	return nil
 }
 
+// setDnsIgnoreUnusableInterface likes setDNS, but return a nil error if the interface is not usable.
+func setDnsIgnoreUnusableInterface(iface *net.Interface, nameservers []string) error {
+	return setDNS(iface, nameservers)
+}
+
 // set the dns server for the provided network interface
 func setDNS(iface *net.Interface, nameservers []string) error {
 	r, err := dns.NewOSConfigurator(logf, iface.Name)
@@ -49,6 +54,11 @@ func setDNS(iface *net.Interface, nameservers []string) error {
 	return nil
 }
 
+// resetDnsIgnoreUnusableInterface likes resetDNS, but return a nil error if the interface is not usable.
+func resetDnsIgnoreUnusableInterface(iface *net.Interface) error {
+	return resetDNS(iface)
+}
+
 func resetDNS(iface *net.Interface) error {
 	r, err := dns.NewOSConfigurator(logf, iface.Name)
 	if err != nil {
@@ -66,3 +76,8 @@ func resetDNS(iface *net.Interface) error {
 func currentDNS(_ *net.Interface) []string {
 	return resolvconffile.NameServers("")
 }
+
+// currentStaticDNS returns the current static DNS settings of given interface.
+func currentStaticDNS(iface *net.Interface) ([]string, error) {
+	return currentDNS(iface), nil
+}

cmd/cli/os_linux.go

@@ -9,12 +9,11 @@ import (
 	"net"
 	"net/netip"
 	"os/exec"
-	"path/filepath"
+	"slices"
 	"strings"
 	"syscall"
 	"time"
 
-	"github.com/fsnotify/fsnotify"
 	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
 	"github.com/insomniacslk/dhcp/dhcpv6"
 	"github.com/insomniacslk/dhcp/dhcpv6/client6"
@@ -25,11 +24,6 @@ import (
 	"github.com/Control-D-Inc/ctrld/internal/resolvconffile"
 )
 
-const (
-	resolvConfPath            = "/etc/resolv.conf"
-	resolvConfBackupFailedMsg = "open /etc/resolv.pre-ctrld-backup.conf: read-only file system"
-)
-
 // allocate loopback ip
 // sudo ip a add 127.0.0.2/24 dev lo
 func allocateIP(ip string) error {
@@ -52,7 +46,11 @@ func deAllocateIP(ip string) error {
 
 const maxSetDNSAttempts = 5
 
-// set the dns server for the provided network interface
+// setDnsIgnoreUnusableInterface likes setDNS, but return a nil error if the interface is not usable.
+func setDnsIgnoreUnusableInterface(iface *net.Interface, nameservers []string) error {
+	return setDNS(iface, nameservers)
+}
+
 func setDNS(iface *net.Interface, nameservers []string) error {
 	r, err := dns.NewOSConfigurator(logf, iface.Name)
 	if err != nil {
@@ -69,12 +67,6 @@ func setDNS(iface *net.Interface, nameservers []string) error {
 		Nameservers:   ns,
 		SearchDomains: []dnsname.FQDN{},
 	}
-	defer func() {
-		if r.Mode() == "direct" {
-			go watchResolveConf(osConfig)
-		}
-	}()
-
 	trySystemdResolve := false
 	for i := 0; i < maxSetDNSAttempts; i++ {
 		if err := r.SetDNS(osConfig); err != nil {
@@ -128,6 +120,11 @@ func setDNS(iface *net.Interface, nameservers []string) error {
 	return nil
 }
 
+// resetDnsIgnoreUnusableInterface likes resetDNS, but return a nil error if the interface is not usable.
+func resetDnsIgnoreUnusableInterface(iface *net.Interface) error {
+	return resetDNS(iface)
+}
+
 func resetDNS(iface *net.Interface) (err error) {
 	defer func() {
 		if err == nil {
@@ -203,6 +200,11 @@ func currentDNS(iface *net.Interface) []string {
 	return nil
 }
 
+// currentStaticDNS returns the current static DNS settings of given interface.
+func currentStaticDNS(iface *net.Interface) ([]string, error) {
+	return currentDNS(iface), nil
+}
+
 func getDNSByResolvectl(iface string) []string {
 	b, err := exec.Command("resolvectl", "dns", "-i", iface).Output()
 	if err != nil {
@@ -284,8 +286,7 @@ func ignoringEINTR(fn func() error) error {
 func isSubSet(s1, s2 []string) bool {
 	ok := true
 	for _, ns := range s1 {
-		// TODO(cuonglm): use slices.Contains once upgrading to go1.21
-		if sliceContains(s2, ns) {
+		if slices.Contains(s2, ns) {
 			continue
 		}
 		ok = false
@@ -293,75 +294,3 @@ func isSubSet(s1, s2 []string) bool {
 	}
 	return ok
 }
-
-// sliceContains reports whether v is present in s.
-func sliceContains[S ~[]E, E comparable](s S, v E) bool {
-	return sliceIndex(s, v) >= 0
-}
-
-// sliceIndex returns the index of the first occurrence of v in s,
-// or -1 if not present.
-func sliceIndex[S ~[]E, E comparable](s S, v E) int {
-	for i := range s {
-		if v == s[i] {
-			return i
-		}
-	}
-	return -1
-}
-
-// watchResolveConf watches any changes to /etc/resolv.conf file,
-// and reverting to the original config set by ctrld.
-func watchResolveConf(oc dns.OSConfig) {
-	mainLog.Load().Debug().Msg("start watching /etc/resolv.conf file")
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		mainLog.Load().Warn().Err(err).Msg("could not create watcher for /etc/resolv.conf")
-		return
-	}
-
-	// We watch /etc instead of /etc/resolv.conf directly,
-	// see: https://github.com/fsnotify/fsnotify#watching-a-file-doesnt-work-well
-	watchDir := filepath.Dir(resolvConfPath)
-	if err := watcher.Add(watchDir); err != nil {
-		mainLog.Load().Warn().Err(err).Msg("could not add /etc/resolv.conf to watcher list")
-		return
-	}
-
-	r, err := dns.NewOSConfigurator(func(format string, args ...any) {}, "lo") // interface name does not matter.
-	if err != nil {
-		mainLog.Load().Error().Err(err).Msg("failed to create DNS OS configurator")
-		return
-	}
-
-	for {
-		select {
-		case event, ok := <-watcher.Events:
-			if !ok {
-				return
-			}
-			if event.Name != resolvConfPath { // skip if not /etc/resolv.conf changes.
-				continue
-			}
-			if event.Has(fsnotify.Write) || event.Has(fsnotify.Create) {
-				mainLog.Load().Debug().Msg("/etc/resolv.conf changes detected, reverting to ctrld setting")
-				if err := watcher.Remove(watchDir); err != nil {
-					mainLog.Load().Error().Err(err).Msg("failed to pause watcher")
-					continue
-				}
-				if err := r.SetDNS(oc); err != nil {
-					mainLog.Load().Error().Err(err).Msg("failed to revert /etc/resolv.conf changes")
-				}
-				if err := watcher.Add(watchDir); err != nil {
-					mainLog.Load().Error().Err(err).Msg("failed to continue running watcher")
-					return
-				}
-			}
-		case err, ok := <-watcher.Errors:
-			if !ok {
-				return
-			}
-			mainLog.Load().Err(err).Msg("could not get event for /etc/resolv.conf")
-		}
-	}
-}

cmd/cli/os_windows.go

@@ -2,21 +2,57 @@ package cli
 
 import (
 	"errors"
+	"fmt"
 	"net"
+	"os"
 	"os/exec"
 	"strconv"
+	"strings"
+	"sync"
 
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 
 	ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
 )
 
+const (
+	forwardersFilename       = ".forwarders.txt"
+	v4InterfaceKeyPathFormat = `HKLM:\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces\`
+	v6InterfaceKeyPathFormat = `HKLM:\SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces\`
+)
+
+var (
+	setDNSOnce   sync.Once
+	resetDNSOnce sync.Once
+)
+
+// setDnsIgnoreUnusableInterface likes setDNS, but return a nil error if the interface is not usable.
+func setDnsIgnoreUnusableInterface(iface *net.Interface, nameservers []string) error {
+	return setDNS(iface, nameservers)
+}
+
+// setDNS sets the dns server for the provided network interface
 func setDNS(iface *net.Interface, nameservers []string) error {
 	if len(nameservers) == 0 {
 		return errors.New("empty DNS nameservers")
 	}
+	setDNSOnce.Do(func() {
+		// If there's a Dns server running, that means we are on AD with Dns feature enabled.
+		// Configuring the Dns server to forward queries to ctrld instead.
+		if windowsHasLocalDnsServerRunning() {
+			file := absHomeDir(forwardersFilename)
+			oldForwardersContent, _ := os.ReadFile(file)
+			if err := os.WriteFile(file, []byte(strings.Join(nameservers, ",")), 0600); err != nil {
+				mainLog.Load().Warn().Err(err).Msg("could not save forwarders settings")
+			}
+			oldForwarders := strings.Split(string(oldForwardersContent), ",")
+			if err := addDnsServerForwarders(nameservers, oldForwarders); err != nil {
+				mainLog.Load().Warn().Err(err).Msg("could not set forwarders settings")
+			}
+		}
+	})
 	primaryDNS := nameservers[0]
-	if err := setPrimaryDNS(iface, primaryDNS); err != nil {
+	if err := setPrimaryDNS(iface, primaryDNS, true); err != nil {
 		return err
 	}
 	if len(nameservers) > 1 {
@@ -26,22 +62,71 @@ func setDNS(iface *net.Interface, nameservers []string) error {
 	return nil
 }
 
+// resetDnsIgnoreUnusableInterface likes resetDNS, but return a nil error if the interface is not usable.
+func resetDnsIgnoreUnusableInterface(iface *net.Interface) error {
+	return resetDNS(iface)
+}
+
 // TODO(cuonglm): should we use system API?
 func resetDNS(iface *net.Interface) error {
+	resetDNSOnce.Do(func() {
+		// See corresponding comment in setDNS.
+		if windowsHasLocalDnsServerRunning() {
+			file := absHomeDir(forwardersFilename)
+			content, err := os.ReadFile(file)
+			if err != nil {
+				mainLog.Load().Error().Err(err).Msg("could not read forwarders settings")
+				return
+			}
+			nameservers := strings.Split(string(content), ",")
+			if err := removeDnsServerForwarders(nameservers); err != nil {
+				mainLog.Load().Error().Err(err).Msg("could not remove forwarders settings")
+				return
+			}
+		}
+	})
+
+	// Restoring ipv6 first.
 	if ctrldnet.SupportsIPv6ListenLocal() {
 		if output, err := netsh("interface", "ipv6", "set", "dnsserver", strconv.Itoa(iface.Index), "dhcp"); err != nil {
 			mainLog.Load().Warn().Err(err).Msgf("failed to reset ipv6 DNS: %s", string(output))
 		}
 	}
+	// Restoring ipv4 DHCP.
 	output, err := netsh("interface", "ipv4", "set", "dnsserver", strconv.Itoa(iface.Index), "dhcp")
 	if err != nil {
-		mainLog.Load().Error().Err(err).Msgf("failed to reset ipv4 DNS: %s", string(output))
-		return err
+		return fmt.Errorf("%s: %w", string(output), err)
+	}
+	// If there's static DNS saved, restoring it.
+	if nss := savedStaticNameservers(iface); len(nss) > 0 {
+		v4ns := make([]string, 0, 2)
+		v6ns := make([]string, 0, 2)
+		for _, ns := range nss {
+			if ctrldnet.IsIPv6(ns) {
+				v6ns = append(v6ns, ns)
+			} else {
+				v4ns = append(v4ns, ns)
+			}
+		}
+
+		for _, ns := range [][]string{v4ns, v6ns} {
+			if len(ns) == 0 {
+				continue
+			}
+			primaryDNS := ns[0]
+			if err := setPrimaryDNS(iface, primaryDNS, false); err != nil {
+				return err
+			}
+			if len(ns) > 1 {
+				secondaryDNS := ns[1]
+				_ = addSecondaryDNS(iface, secondaryDNS)
+			}
+		}
 	}
 	return nil
 }
 
-func setPrimaryDNS(iface *net.Interface, dns string) error {
+func setPrimaryDNS(iface *net.Interface, dns string, disablev6 bool) error {
 	ipVer := "ipv4"
 	if ctrldnet.IsIPv6(dns) {
 		ipVer = "ipv6"
@@ -52,7 +137,7 @@ func setPrimaryDNS(iface *net.Interface, dns string) error {
 		mainLog.Load().Error().Err(err).Msgf("failed to set primary DNS: %s", string(output))
 		return err
 	}
-	if ipVer == "ipv4" && ctrldnet.SupportsIPv6ListenLocal() {
+	if disablev6 && ipVer == "ipv4" && ctrldnet.SupportsIPv6ListenLocal() {
 		// Disable IPv6 DNS, so the query will be fallback to IPv4.
 		_, _ = netsh("interface", "ipv6", "set", "dnsserver", idx, "static", "::1", "primary")
 	}
@@ -93,3 +178,72 @@ func currentDNS(iface *net.Interface) []string {
 	}
 	return ns
 }
+
+// currentStaticDNS returns the current static DNS settings of given interface.
+func currentStaticDNS(iface *net.Interface) ([]string, error) {
+	luid, err := winipcfg.LUIDFromIndex(uint32(iface.Index))
+	if err != nil {
+		return nil, err
+	}
+	guid, err := luid.GUID()
+	if err != nil {
+		return nil, err
+	}
+	var ns []string
+	for _, path := range []string{v4InterfaceKeyPathFormat, v6InterfaceKeyPathFormat} {
+		interfaceKeyPath := path + guid.String()
+		found := false
+		for _, key := range []string{"NameServer", "ProfileNameServer"} {
+			if found {
+				continue
+			}
+			cmd := fmt.Sprintf(`Get-ItemPropertyValue -Path "%s" -Name "%s"`, interfaceKeyPath, key)
+			out, err := powershell(cmd)
+			if err == nil && len(out) > 0 {
+				found = true
+				ns = append(ns, strings.Split(string(out), ",")...)
+			}
+		}
+	}
+	return ns, nil
+}
+
+// addDnsServerForwarders adds given nameservers to DNS server forwarders list,
+// and also removing old forwarders if provided.
+func addDnsServerForwarders(nameservers, old []string) error {
+	newForwardersMap := make(map[string]struct{})
+	newForwarders := make([]string, len(nameservers))
+	for i := range nameservers {
+		newForwardersMap[nameservers[i]] = struct{}{}
+		newForwarders[i] = fmt.Sprintf("%q", nameservers[i])
+	}
+	oldForwarders := old[:0]
+	for _, fwd := range old {
+		if _, ok := newForwardersMap[fwd]; !ok {
+			oldForwarders = append(oldForwarders, fwd)
+		}
+	}
+	// NOTE: It is important to add new forwarder before removing old one.
+	//       Testing on Windows Server 2022 shows that removing forwarder1
+	//       then adding forwarder2 sometimes ends up adding both of them
+	//       to the forwarders list.
+	cmd := fmt.Sprintf("Add-DnsServerForwarder -IPAddress %s", strings.Join(newForwarders, ","))
+	if len(oldForwarders) > 0 {
+		cmd = fmt.Sprintf("%s ; Remove-DnsServerForwarder -IPAddress %s -Force", cmd, strings.Join(oldForwarders, ","))
+	}
+	if out, err := powershell(cmd); err != nil {
+		return fmt.Errorf("%w: %s", err, string(out))
+	}
+	return nil
+}
+
+// removeDnsServerForwarders removes given nameservers from DNS server forwarders list.
+func removeDnsServerForwarders(nameservers []string) error {
+	for _, ns := range nameservers {
+		cmd := fmt.Sprintf("Remove-DnsServerForwarder -IPAddress %s -Force", ns)
+		if out, err := powershell(cmd); err != nil {
+			return fmt.Errorf("%w: %s", err, string(out))
+		}
+	}
+	return nil
+}

cmd/cli/prog.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/fs"
 	"math/rand"
 	"net"
 	"net/netip"
@@ -13,6 +14,7 @@ import (
 	"runtime"
 	"sort"
 	"strconv"
+	"strings"
 	"sync"
 	"syscall"
 
@@ -31,11 +33,22 @@ const (
 	defaultSemaphoreCap  = 256
 	ctrldLogUnixSock     = "ctrld_start.sock"
 	ctrldControlUnixSock = "ctrld_control.sock"
-	upstreamPrefix       = "upstream."
-	upstreamOS           = upstreamPrefix + "os"
-	upstreamPrivate      = upstreamPrefix + "private"
+	// iOS unix socket name max length is 11.
+	ctrldControlUnixSockMobile = "cd.sock"
+	upstreamPrefix             = "upstream."
+	upstreamOS                 = upstreamPrefix + "os"
+	upstreamPrivate            = upstreamPrefix + "private"
 )
 
+// ControlSocketName returns name for control unix socket.
+func ControlSocketName() string {
+	if isMobile() {
+		return ctrldControlUnixSockMobile
+	} else {
+		return ctrldControlUnixSock
+	}
+}
+
 var logf = func(format string, args ...any) {
 	mainLog.Load().Debug().Msgf(format, args...)
 }
@@ -56,18 +69,21 @@ type prog struct {
 	reloadDoneCh chan struct{}
 	logConn      net.Conn
 	cs           *controlServer
+	csSetDnsDone chan struct{}
+	csSetDnsOk   bool
 
-	cfg            *ctrld.Config
-	localUpstreams []string
-	ptrNameservers []string
-	appCallback    *AppCallback
-	cache          dnscache.Cacher
-	sema           semaphore
-	ciTable        *clientinfo.Table
-	um             *upstreamMonitor
-	router         router.Router
-	ptrLoopGuard   *loopGuard
-	lanLoopGuard   *loopGuard
+	cfg                  *ctrld.Config
+	localUpstreams       []string
+	ptrNameservers       []string
+	appCallback          *AppCallback
+	cache                dnscache.Cacher
+	cacheFlushDomainsMap map[string]struct{}
+	sema                 semaphore
+	ciTable              *clientinfo.Table
+	um                   *upstreamMonitor
+	router               router.Router
+	ptrLoopGuard         *loopGuard
+	lanLoopGuard         *loopGuard
 
 	loopMu sync.Mutex
 	loop   map[string]bool
@@ -180,9 +196,6 @@ func (p *prog) runWait() {
 }
 
 func (p *prog) preRun() {
-	if !service.Interactive() {
-		p.setDNS()
-	}
 	if runtime.GOOS == "darwin" {
 		p.onStopped = append(p.onStopped, func() {
 			if !service.Interactive() {
@@ -192,6 +205,15 @@ func (p *prog) preRun() {
 	}
 }
 
+func (p *prog) postRun() {
+	if !service.Interactive() {
+		p.resetDNS()
+		ns := ctrld.InitializeOsResolver()
+		mainLog.Load().Debug().Msgf("initialized OS resolver with nameservers: %v", ns)
+		p.setDNS()
+	}
+}
+
 func (p *prog) setupUpstream(cfg *ctrld.Config) {
 	localUpstreams := make([]string, 0, len(cfg.Upstream))
 	ptrNameservers := make([]string, 0, len(cfg.Upstream))
@@ -235,17 +257,30 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) {
 	numListeners := len(p.cfg.Listener)
 	if !reload {
 		p.started = make(chan struct{}, numListeners)
+		if p.cs != nil {
+			p.csSetDnsDone = make(chan struct{}, 1)
+			p.registerControlServerHandler()
+			if err := p.cs.start(); err != nil {
+				mainLog.Load().Warn().Err(err).Msg("could not start control server")
+			}
+			mainLog.Load().Debug().Msgf("control server started: %s", p.cs.addr)
+		}
 	}
 	p.onStartedDone = make(chan struct{})
 	p.loop = make(map[string]bool)
 	p.lanLoopGuard = newLoopGuard()
 	p.ptrLoopGuard = newLoopGuard()
+	p.cacheFlushDomainsMap = nil
 	if p.cfg.Service.CacheEnable {
 		cacher, err := dnscache.NewLRUCache(p.cfg.Service.CacheSize)
 		if err != nil {
 			mainLog.Load().Error().Err(err).Msg("failed to create cacher, caching is disabled")
 		} else {
 			p.cache = cacher
+			p.cacheFlushDomainsMap = make(map[string]struct{}, 256)
+			for _, domain := range p.cfg.Service.CacheFlushDomains {
+				p.cacheFlushDomainsMap[canonicalName(domain)] = struct{}{}
+			}
 		}
 	}
 
@@ -362,12 +397,7 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) {
 		if p.logConn != nil {
 			_ = p.logConn.Close()
 		}
-		if p.cs != nil {
-			p.registerControlServerHandler()
-			if err := p.cs.start(); err != nil {
-				mainLog.Load().Warn().Err(err).Msg("could not start control server")
-			}
-		}
+		p.postRun()
 	}
 	wg.Wait()
 }
@@ -411,21 +441,35 @@ func (p *prog) deAllocateIP() error {
 }
 
 func (p *prog) setDNS() {
+	setDnsOK := false
+	defer func() {
+		p.csSetDnsOk = setDnsOK
+		p.csSetDnsDone <- struct{}{}
+		close(p.csSetDnsDone)
+	}()
+
 	if cfg.Listener == nil {
 		return
 	}
 	if iface == "" {
 		return
 	}
-	if iface == "auto" {
-		iface = defaultIfaceName()
+	runningIface := iface
+	// allIfaces tracks whether we should set DNS for all physical interfaces.
+	allIfaces := false
+	if runningIface == "auto" {
+		runningIface = defaultIfaceName()
+		// If runningIface is "auto", it means user does not specify "--iface" flag.
+		// In this case, ctrld has to set DNS for all physical interfaces, so
+		// thing will still work when user switch from one to the other.
+		allIfaces = requiredMultiNICsConfig()
 	}
 	lc := cfg.FirstListener()
 	if lc == nil {
 		return
 	}
-	logger := mainLog.Load().With().Str("iface", iface).Logger()
-	netIface, err := netInterface(iface)
+	logger := mainLog.Load().With().Str("iface", runningIface).Logger()
+	netIface, err := netInterface(runningIface)
 	if err != nil {
 		logger.Error().Err(err).Msg("could not get interface")
 		return
@@ -459,18 +503,35 @@ func (p *prog) setDNS() {
 		logger.Error().Err(err).Msgf("could not set DNS for interface")
 		return
 	}
+	setDnsOK = true
 	logger.Debug().Msg("setting DNS successfully")
+	if shouldWatchResolvconf() {
+		servers := make([]netip.Addr, len(nameservers))
+		for i := range nameservers {
+			servers[i] = netip.MustParseAddr(nameservers[i])
+		}
+		go watchResolvConf(netIface, servers, setResolvConf)
+	}
+	if allIfaces {
+		withEachPhysicalInterfaces(netIface.Name, "set DNS", func(i *net.Interface) error {
+			return setDnsIgnoreUnusableInterface(i, nameservers)
+		})
+	}
 }
 
 func (p *prog) resetDNS() {
 	if iface == "" {
 		return
 	}
-	if iface == "auto" {
-		iface = defaultIfaceName()
+	runningIface := iface
+	allIfaces := false
+	if runningIface == "auto" {
+		runningIface = defaultIfaceName()
+		// See corresponding comments in (*prog).setDNS function.
+		allIfaces = requiredMultiNICsConfig()
 	}
-	logger := mainLog.Load().With().Str("iface", iface).Logger()
-	netIface, err := netInterface(iface)
+	logger := mainLog.Load().With().Str("iface", runningIface).Logger()
+	netIface, err := netInterface(runningIface)
 	if err != nil {
 		logger.Error().Err(err).Msg("could not get interface")
 		return
@@ -485,6 +546,9 @@ func (p *prog) resetDNS() {
 		return
 	}
 	logger.Debug().Msg("Restoring DNS successfully")
+	if allIfaces {
+		withEachPhysicalInterfaces(netIface.Name, "reset DNS", resetDnsIgnoreUnusableInterface)
+	}
 }
 
 func randomLocalIP() string {
@@ -568,6 +632,15 @@ func errNetworkError(err error) bool {
 	return false
 }
 
+// errConnectionRefused reports whether err is connection refused.
+func errConnectionRefused(err error) bool {
+	var opErr *net.OpError
+	if !errors.As(err, &opErr) {
+		return false
+	}
+	return errors.Is(opErr.Err, syscall.ECONNREFUSED) || errors.Is(opErr.Err, windowsECONNREFUSED)
+}
+
 func ifaceFirstPrivateIP(iface *net.Interface) string {
 	if iface == nil {
 		return ""
@@ -649,3 +722,87 @@ func canBeLocalUpstream(addr string) bool {
 	}
 	return false
 }
+
+// withEachPhysicalInterfaces runs the function f with each physical interfaces, excluding
+// the interface that matches excludeIfaceName. The context is used to clarify the
+// log message when error happens.
+func withEachPhysicalInterfaces(excludeIfaceName, context string, f func(i *net.Interface) error) {
+	interfaces.ForeachInterface(func(i interfaces.Interface, prefixes []netip.Prefix) {
+		// Skip loopback/virtual interface.
+		if i.IsLoopback() || len(i.HardwareAddr) == 0 {
+			return
+		}
+		// Skip invalid interface.
+		if !validInterface(i.Interface) {
+			return
+		}
+		netIface := i.Interface
+		if err := patchNetIfaceName(netIface); err != nil {
+			mainLog.Load().Debug().Err(err).Msg("failed to patch net interface name")
+			return
+		}
+		// Skip excluded interface.
+		if netIface.Name == excludeIfaceName {
+			return
+		}
+		// TODO: investigate whether we should report this error?
+		if err := f(netIface); err == nil {
+			mainLog.Load().Debug().Msgf("%s for interface %q successfully", context, i.Name)
+		} else if !errors.Is(err, errSaveCurrentStaticDNSNotSupported) {
+			mainLog.Load().Err(err).Msgf("%s for interface %q failed", context, i.Name)
+		}
+	})
+}
+
+// requiredMultiNicConfig reports whether ctrld needs to set/reset DNS for multiple NICs.
+func requiredMultiNICsConfig() bool {
+	switch runtime.GOOS {
+	case "windows", "darwin":
+		return true
+	default:
+		return false
+	}
+}
+
+var errSaveCurrentStaticDNSNotSupported = errors.New("saving current DNS is not supported on this platform")
+
+// saveCurrentStaticDNS saves the current static DNS settings for restoring later.
+// Only works on Windows and Mac.
+func saveCurrentStaticDNS(iface *net.Interface) error {
+	switch runtime.GOOS {
+	case "windows", "darwin":
+	default:
+		return errSaveCurrentStaticDNSNotSupported
+	}
+	file := savedStaticDnsSettingsFilePath(iface)
+	ns, _ := currentStaticDNS(iface)
+	if len(ns) == 0 {
+		_ = os.Remove(file) // removing old static DNS settings
+		return nil
+	}
+	if err := os.Remove(file); err != nil && !errors.Is(err, fs.ErrNotExist) {
+		mainLog.Load().Warn().Err(err).Msg("could not remove old static DNS settings file")
+	}
+	mainLog.Load().Debug().Msgf("DNS settings for %s is static, saving ...", iface.Name)
+	if err := os.WriteFile(file, []byte(strings.Join(ns, ",")), 0600); err != nil {
+		mainLog.Load().Err(err).Msgf("could not save DNS settings for iface: %s", iface.Name)
+		return err
+	}
+	return nil
+}
+
+// savedStaticDnsSettingsFilePath returns the path to saved DNS settings of the given interface.
+func savedStaticDnsSettingsFilePath(iface *net.Interface) string {
+	return absHomeDir(".dns_" + iface.Name)
+}
+
+// savedStaticNameservers returns the static DNS nameservers of the given interface.
+//
+//lint:ignore U1000 use in os_windows.go and os_darwin.go
+func savedStaticNameservers(iface *net.Interface) []string {
+	file := savedStaticDnsSettingsFilePath(iface)
+	if data, _ := os.ReadFile(file); len(data) > 0 {
+		return strings.Split(string(data), ",")
+	}
+	return nil
+}

cmd/cli/prog_linux.go

@@ -1,6 +1,13 @@
 package cli
 
 import (
+	"bufio"
+	"bytes"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+
 	"github.com/kardianos/service"
 
 	"github.com/Control-D-Inc/ctrld/internal/dns"
@@ -10,6 +17,10 @@ func init() {
 	if r, err := dns.NewOSConfigurator(func(format string, args ...any) {}, "lo"); err == nil {
 		useSystemdResolved = r.Mode() == "systemd-resolved"
 	}
+	// Disable quic-go's ECN support by default, see https://github.com/quic-go/quic-go/issues/3911
+	if os.Getenv("QUIC_GO_DISABLE_ECN") == "" {
+		os.Setenv("QUIC_GO_DISABLE_ECN", "true")
+	}
 }
 
 func setDependencies(svc *service.Config) {
@@ -18,12 +29,34 @@ func setDependencies(svc *service.Config) {
 		"After=network-online.target",
 		"Wants=NetworkManager-wait-online.service",
 		"After=NetworkManager-wait-online.service",
-		"Wants=systemd-networkd-wait-online.service",
 		"Wants=nss-lookup.target",
 		"After=nss-lookup.target",
 	}
+	if out, _ := exec.Command("networkctl", "--no-pager").CombinedOutput(); len(out) > 0 {
+		if wantsSystemDNetworkdWaitOnline(bytes.NewReader(out)) {
+			svc.Dependencies = append(svc.Dependencies, "Wants=systemd-networkd-wait-online.service")
+		}
+	}
 }
 
 func setWorkingDirectory(svc *service.Config, dir string) {
 	svc.WorkingDirectory = dir
 }
+
+// wantsSystemDNetworkdWaitOnline reports whether "systemd-networkd-wait-online" service
+// is required to be added to ctrld dependencies services.
+// The input reader r is the output of "networkctl --no-pager" command.
+func wantsSystemDNetworkdWaitOnline(r io.Reader) bool {
+	scanner := bufio.NewScanner(r)
+	// Skip header
+	scanner.Scan()
+	configured := false
+	for scanner.Scan() {
+		fields := strings.Fields(scanner.Text())
+		if len(fields) > 0 && fields[len(fields)-1] == "configured" {
+			configured = true
+			break
+		}
+	}
+	return configured
+}

cmd/cli/prog_linux_test.go

@@ -0,0 +1,48 @@
+package cli
+
+import (
+	"io"
+	"strings"
+	"testing"
+)
+
+const (
+	networkctlUnmanagedOutput = `IDX LINK            TYPE     OPERATIONAL SETUP    
+  1 lo              loopback carrier     unmanaged
+  2 wlp0s20f3       wlan     routable    unmanaged
+  3 tailscale0      none     routable    unmanaged
+  4 br-9ac33145e060 bridge   no-carrier  unmanaged
+  5 docker0         bridge   no-carrier  unmanaged
+
+5 links listed.
+`
+	networkctlManagedOutput = `IDX LINK            TYPE     OPERATIONAL SETUP    
+  1 lo              loopback carrier     unmanaged
+  2 wlp0s20f3       wlan     routable    configured
+  3 tailscale0      none     routable    unmanaged
+  4 br-9ac33145e060 bridge   no-carrier  unmanaged
+  5 docker0         bridge   no-carrier  unmanaged
+
+5 links listed.
+`
+)
+
+func Test_wantsSystemDNetworkdWaitOnline(t *testing.T) {
+	tests := []struct {
+		name     string
+		r        io.Reader
+		required bool
+	}{
+		{"unmanaged", strings.NewReader(networkctlUnmanagedOutput), false},
+		{"managed", strings.NewReader(networkctlManagedOutput), true},
+		{"empty", strings.NewReader(""), false},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			if required := wantsSystemDNetworkdWaitOnline(tc.r); required != tc.required {
+				t.Errorf("wants %v got %v", tc.required, required)
+			}
+		})
+	}
+}

cmd/cli/resolvconf.go

@@ -0,0 +1,65 @@
+package cli
+
+import (
+	"net"
+	"net/netip"
+	"path/filepath"
+
+	"github.com/fsnotify/fsnotify"
+)
+
+const (
+	resolvConfPath            = "/etc/resolv.conf"
+	resolvConfBackupFailedMsg = "open /etc/resolv.pre-ctrld-backup.conf: read-only file system"
+)
+
+// watchResolvConf watches any changes to /etc/resolv.conf file,
+// and reverting to the original config set by ctrld.
+func watchResolvConf(iface *net.Interface, ns []netip.Addr, setDnsFn func(iface *net.Interface, ns []netip.Addr) error) {
+	mainLog.Load().Debug().Msg("start watching /etc/resolv.conf file")
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		mainLog.Load().Warn().Err(err).Msg("could not create watcher for /etc/resolv.conf")
+		return
+	}
+	defer watcher.Close()
+
+	// We watch /etc instead of /etc/resolv.conf directly,
+	// see: https://github.com/fsnotify/fsnotify#watching-a-file-doesnt-work-well
+	watchDir := filepath.Dir(resolvConfPath)
+	if err := watcher.Add(watchDir); err != nil {
+		mainLog.Load().Warn().Err(err).Msg("could not add /etc/resolv.conf to watcher list")
+		return
+	}
+
+	for {
+		select {
+		case event, ok := <-watcher.Events:
+			if !ok {
+				return
+			}
+			if event.Name != resolvConfPath { // skip if not /etc/resolv.conf changes.
+				continue
+			}
+			if event.Has(fsnotify.Write) || event.Has(fsnotify.Create) {
+				mainLog.Load().Debug().Msg("/etc/resolv.conf changes detected, reverting to ctrld setting")
+				if err := watcher.Remove(watchDir); err != nil {
+					mainLog.Load().Error().Err(err).Msg("failed to pause watcher")
+					continue
+				}
+				if err := setDnsFn(iface, ns); err != nil {
+					mainLog.Load().Error().Err(err).Msg("failed to revert /etc/resolv.conf changes")
+				}
+				if err := watcher.Add(watchDir); err != nil {
+					mainLog.Load().Error().Err(err).Msg("failed to continue running watcher")
+					return
+				}
+			}
+		case err, ok := <-watcher.Errors:
+			if !ok {
+				return
+			}
+			mainLog.Load().Err(err).Msg("could not get event for /etc/resolv.conf")
+		}
+	}
+}

cmd/cli/resolvconf_darwin.go

@@ -0,0 +1,20 @@
+package cli
+
+import (
+	"net"
+	"net/netip"
+)
+
+// setResolvConf sets the content of resolv.conf file using the given nameservers list.
+func setResolvConf(iface *net.Interface, ns []netip.Addr) error {
+	servers := make([]string, len(ns))
+	for i := range ns {
+		servers[i] = ns[i].String()
+	}
+	return setDNS(iface, servers)
+}
+
+// shouldWatchResolvconf reports whether ctrld should watch changes to resolv.conf file with given OS configurator.
+func shouldWatchResolvconf() bool {
+	return true
+}

cmd/cli/resolvconf_not_darwin_unix.go

@@ -0,0 +1,40 @@
+//go:build unix && !darwin
+
+package cli
+
+import (
+	"net"
+	"net/netip"
+
+	"tailscale.com/util/dnsname"
+
+	"github.com/Control-D-Inc/ctrld/internal/dns"
+)
+
+// setResolvConf sets the content of resolv.conf file using the given nameservers list.
+func setResolvConf(iface *net.Interface, ns []netip.Addr) error {
+	r, err := dns.NewOSConfigurator(func(format string, args ...any) {}, "lo") // interface name does not matter.
+	if err != nil {
+		return err
+	}
+
+	oc := dns.OSConfig{
+		Nameservers:   ns,
+		SearchDomains: []dnsname.FQDN{},
+	}
+	return r.SetDNS(oc)
+}
+
+// shouldWatchResolvconf reports whether ctrld should watch changes to resolv.conf file with given OS configurator.
+func shouldWatchResolvconf() bool {
+	r, err := dns.NewOSConfigurator(func(format string, args ...any) {}, "lo") // interface name does not matter.
+	if err != nil {
+		return false
+	}
+	switch r.Mode() {
+	case "direct", "resolvconf":
+		return true
+	default:
+		return false
+	}
+}

cmd/cli/resolvconf_windows.go

@@ -0,0 +1,16 @@
+package cli
+
+import (
+	"net"
+	"net/netip"
+)
+
+// setResolvConf sets the content of resolv.conf file using the given nameservers list.
+func setResolvConf(_ *net.Interface, _ []netip.Addr) error {
+	return nil
+}
+
+// shouldWatchResolvconf reports whether ctrld should watch changes to resolv.conf file with given OS configurator.
+func shouldWatchResolvconf() bool {
+	return false
+}

cmd/cli/service.go

@@ -20,8 +20,8 @@ func newService(i service.Interface, c *service.Config) (service.Service, error)
 		return nil, err
 	}
 	switch {
-	case router.IsOldOpenwrt():
-		return &procd{&sysV{s}}, nil
+	case router.IsOldOpenwrt(), router.IsNetGearOrbi():
+		return &procd{sysV: &sysV{s}, svcConfig: c}, nil
 	case router.IsGLiNet():
 		return &sysV{s}, nil
 	case s.Platform() == "unix-systemv":
@@ -89,18 +89,24 @@ func (s *sysV) Status() (service.Status, error) {
 // like old GL.iNET Opal router.
 type procd struct {
 	*sysV
+	svcConfig *service.Config
 }
 
 func (s *procd) Status() (service.Status, error) {
 	if !s.installed() {
 		return service.StatusUnknown, service.ErrNotInstalled
 	}
-	exe, err := os.Executable()
-	if err != nil {
-		return service.StatusUnknown, nil
+	bin := s.svcConfig.Executable
+	if bin == "" {
+		exe, err := os.Executable()
+		if err != nil {
+			return service.StatusUnknown, nil
+		}
+		bin = exe
 	}
+
 	// Looking for something like "/sbin/ctrld run ".
-	shellCmd := fmt.Sprintf("ps | grep -q %q", exe+" [r]un ")
+	shellCmd := fmt.Sprintf("ps | grep -q %q", bin+" [r]un ")
 	if err := exec.Command("sh", "-c", shellCmd).Run(); err != nil {
 		return service.StatusStopped, nil
 	}

cmd/cli/winres/winres.json

@@ -0,0 +1,20 @@
+{
+  "RT_VERSION": {
+    "#1": {
+      "0000": {
+        "fixed": {
+          "file_version": "0.0.0.1"
+        },
+        "info": {
+          "0409": {
+            "CompanyName": "ControlD Inc",
+            "FileDescription": "Control D DNS daemon",
+            "ProductName": "ctrld",
+            "InternalName": "ctrld",
+            "LegalCopyright": "ControlD Inc 2024"
+          }
+        }
+      }
+    }
+  }
+}

cmd/cli/winres_windows.go

@@ -0,0 +1,4 @@
+//go:generate go-winres make --product-version=git-tag --file-version=git-tag
+package cli
+
+// Placeholder file for windows builds.

cmd/ctrld_library/main.go

@@ -61,13 +61,18 @@ func mapCallback(callback AppCallback) cli.AppCallback {
 	}
 }
 
-func (c *Controller) Stop() bool {
-	if c.stopCh != nil {
+func (c *Controller) Stop(restart bool, pin int64) int {
+	var errorCode = 0
+	// Force disconnect without checking pin.
+	// In iOS restart is required if vpn detects no connectivity after network change.
+	if !restart {
+		errorCode = cli.CheckDeactivationPin(pin, c.stopCh)
+	}
+	if errorCode == 0 && c.stopCh != nil {
 		close(c.stopCh)
 		c.stopCh = nil
-		return true
 	}
-	return false
+	return errorCode
 }
 
 func (c *Controller) IsRunning() bool {

config.go

@@ -46,6 +46,15 @@ const (
 	// depending on the record type of the DNS query.
 	IpStackSplit = "split"
 
+	// FreeDnsDomain is the domain name of free ControlD service.
+	FreeDnsDomain = "freedns.controld.com"
+	// FreeDNSBoostrapIP is the IP address of freedns.controld.com.
+	FreeDNSBoostrapIP = "76.76.2.11"
+	// PremiumDnsDomain is the domain name of premium ControlD service.
+	PremiumDnsDomain = "dns.controld.com"
+	// PremiumDNSBoostrapIP is the IP address of dns.controld.com.
+	PremiumDNSBoostrapIP = "76.76.2.22"
+
 	controlDComDomain = "controld.com"
 	controlDNetDomain = "controld.net"
 	controlDDevDomain = "controld.dev"
@@ -104,14 +113,14 @@ func InitConfig(v *viper.Viper, name string) {
 	})
 	v.SetDefault("upstream", map[string]*UpstreamConfig{
 		"0": {
-			BootstrapIP: "76.76.2.11",
+			BootstrapIP: FreeDNSBoostrapIP,
 			Name:        "Control D - Anti-Malware",
 			Type:        ResolverTypeDOH,
 			Endpoint:    "https://freedns.controld.com/p1",
 			Timeout:     5000,
 		},
 		"1": {
-			BootstrapIP: "76.76.2.11",
+			BootstrapIP: FreeDNSBoostrapIP,
 			Name:        "Control D - No Ads",
 			Type:        ResolverTypeDOQ,
 			Endpoint:    "p2.freedns.controld.com",
@@ -179,26 +188,27 @@ func (c *Config) FirstUpstream() *UpstreamConfig {
 
 // ServiceConfig specifies the general ctrld config.
 type ServiceConfig struct {
-	LogLevel                string `mapstructure:"log_level" toml:"log_level,omitempty"`
-	LogPath                 string `mapstructure:"log_path" toml:"log_path,omitempty"`
-	CacheEnable             bool   `mapstructure:"cache_enable" toml:"cache_enable,omitempty"`
-	CacheSize               int    `mapstructure:"cache_size" toml:"cache_size,omitempty"`
-	CacheTTLOverride        int    `mapstructure:"cache_ttl_override" toml:"cache_ttl_override,omitempty"`
-	CacheServeStale         bool   `mapstructure:"cache_serve_stale" toml:"cache_serve_stale,omitempty"`
-	MaxConcurrentRequests   *int   `mapstructure:"max_concurrent_requests" toml:"max_concurrent_requests,omitempty" validate:"omitempty,gte=0"`
-	DHCPLeaseFile           string `mapstructure:"dhcp_lease_file_path" toml:"dhcp_lease_file_path" validate:"omitempty,file"`
-	DHCPLeaseFileFormat     string `mapstructure:"dhcp_lease_file_format" toml:"dhcp_lease_file_format" validate:"required_unless=DHCPLeaseFile '',omitempty,oneof=dnsmasq isc-dhcp"`
-	DiscoverMDNS            *bool  `mapstructure:"discover_mdns" toml:"discover_mdns,omitempty"`
-	DiscoverARP             *bool  `mapstructure:"discover_arp" toml:"discover_arp,omitempty"`
-	DiscoverDHCP            *bool  `mapstructure:"discover_dhcp" toml:"discover_dhcp,omitempty"`
-	DiscoverPtr             *bool  `mapstructure:"discover_ptr" toml:"discover_ptr,omitempty"`
-	DiscoverHosts           *bool  `mapstructure:"discover_hosts" toml:"discover_hosts,omitempty"`
-	DiscoverRefreshInterval int    `mapstructure:"discover_refresh_interval" toml:"discover_refresh_interval,omitempty"`
-	ClientIDPref            string `mapstructure:"client_id_preference" toml:"client_id_preference,omitempty" validate:"omitempty,oneof=host mac"`
-	MetricsQueryStats       bool   `mapstructure:"metrics_query_stats" toml:"metrics_query_stats,omitempty"`
-	MetricsListener         string `mapstructure:"metrics_listener" toml:"metrics_listener,omitempty"`
-	Daemon                  bool   `mapstructure:"-" toml:"-"`
-	AllocateIP              bool   `mapstructure:"-" toml:"-"`
+	LogLevel                string   `mapstructure:"log_level" toml:"log_level,omitempty"`
+	LogPath                 string   `mapstructure:"log_path" toml:"log_path,omitempty"`
+	CacheEnable             bool     `mapstructure:"cache_enable" toml:"cache_enable,omitempty"`
+	CacheSize               int      `mapstructure:"cache_size" toml:"cache_size,omitempty"`
+	CacheTTLOverride        int      `mapstructure:"cache_ttl_override" toml:"cache_ttl_override,omitempty"`
+	CacheServeStale         bool     `mapstructure:"cache_serve_stale" toml:"cache_serve_stale,omitempty"`
+	CacheFlushDomains       []string `mapstructure:"cache_flush_domains" toml:"cache_flush_domains" validate:"max=256"`
+	MaxConcurrentRequests   *int     `mapstructure:"max_concurrent_requests" toml:"max_concurrent_requests,omitempty" validate:"omitempty,gte=0"`
+	DHCPLeaseFile           string   `mapstructure:"dhcp_lease_file_path" toml:"dhcp_lease_file_path" validate:"omitempty,file"`
+	DHCPLeaseFileFormat     string   `mapstructure:"dhcp_lease_file_format" toml:"dhcp_lease_file_format" validate:"required_unless=DHCPLeaseFile '',omitempty,oneof=dnsmasq isc-dhcp"`
+	DiscoverMDNS            *bool    `mapstructure:"discover_mdns" toml:"discover_mdns,omitempty"`
+	DiscoverARP             *bool    `mapstructure:"discover_arp" toml:"discover_arp,omitempty"`
+	DiscoverDHCP            *bool    `mapstructure:"discover_dhcp" toml:"discover_dhcp,omitempty"`
+	DiscoverPtr             *bool    `mapstructure:"discover_ptr" toml:"discover_ptr,omitempty"`
+	DiscoverHosts           *bool    `mapstructure:"discover_hosts" toml:"discover_hosts,omitempty"`
+	DiscoverRefreshInterval int      `mapstructure:"discover_refresh_interval" toml:"discover_refresh_interval,omitempty"`
+	ClientIDPref            string   `mapstructure:"client_id_preference" toml:"client_id_preference,omitempty" validate:"omitempty,oneof=host mac"`
+	MetricsQueryStats       bool     `mapstructure:"metrics_query_stats" toml:"metrics_query_stats,omitempty"`
+	MetricsListener         string   `mapstructure:"metrics_listener" toml:"metrics_listener,omitempty"`
+	Daemon                  bool     `mapstructure:"-" toml:"-"`
+	AllocateIP              bool     `mapstructure:"-" toml:"-"`
 }
 
 // NetworkConfig specifies configuration for networks where ctrld will handle requests.
@@ -285,6 +295,7 @@ type Rule map[string][]string
 
 // Init initialized necessary values for an UpstreamConfig.
 func (uc *UpstreamConfig) Init() {
+	uc.initDoHScheme()
 	uc.uid = upstreamUID()
 	if u, err := url.Parse(uc.Endpoint); err == nil {
 		uc.Domain = u.Host
@@ -510,35 +521,55 @@ func (uc *UpstreamConfig) newDOHTransport(addrs []string) *http.Transport {
 
 // Ping warms up the connection to DoH/DoH3 upstream.
 func (uc *UpstreamConfig) Ping() {
+	_ = uc.ping()
+}
+
+// ErrorPing is like Ping, but return an error if any.
+func (uc *UpstreamConfig) ErrorPing() error {
+	return uc.ping()
+}
+
+func (uc *UpstreamConfig) ping() error {
 	switch uc.Type {
 	case ResolverTypeDOH, ResolverTypeDOH3:
 	default:
-		return
+		return nil
 	}
 
-	ping := func(t http.RoundTripper) {
+	ping := func(t http.RoundTripper) error {
 		if t == nil {
-			return
+			return nil
 		}
 		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
 		defer cancel()
-		req, _ := http.NewRequestWithContext(ctx, "HEAD", uc.Endpoint, nil)
-		resp, _ := t.RoundTrip(req)
-		if resp == nil {
-			return
+		req, err := http.NewRequestWithContext(ctx, "HEAD", uc.Endpoint, nil)
+		if err != nil {
+			return err
+		}
+		resp, err := t.RoundTrip(req)
+		if err != nil {
+			return err
 		}
 		defer resp.Body.Close()
 		_, _ = io.Copy(io.Discard, resp.Body)
+		return nil
 	}
 
 	for _, typ := range []uint16{dns.TypeA, dns.TypeAAAA} {
 		switch uc.Type {
 		case ResolverTypeDOH:
-			ping(uc.dohTransport(typ))
+
+			if err := ping(uc.dohTransport(typ)); err != nil {
+				return err
+			}
 		case ResolverTypeDOH3:
-			ping(uc.doh3Transport(typ))
+			if err := ping(uc.doh3Transport(typ)); err != nil {
+				return err
+			}
 		}
 	}
+
+	return nil
 }
 
 func (uc *UpstreamConfig) isControlD() bool {
@@ -631,6 +662,18 @@ func (uc *UpstreamConfig) netForDNSType(dnsType uint16) (string, string) {
 	return "tcp-tls", "udp"
 }
 
+// initDoHScheme initializes the endpoint scheme for DoH/DoH3 upstream if not present.
+func (uc *UpstreamConfig) initDoHScheme() {
+	switch uc.Type {
+	case ResolverTypeDOH, ResolverTypeDOH3:
+	default:
+		return
+	}
+	if !strings.HasPrefix(uc.Endpoint, "https://") {
+		uc.Endpoint = "https://" + uc.Endpoint
+	}
+}
+
 // Init initialized necessary values for an ListenerConfig.
 func (lc *ListenerConfig) Init() {
 	if lc.Policy != nil {
@@ -683,6 +726,7 @@ func upstreamConfigStructLevelValidation(sl validator.StructLevel) {
 		return
 	}
 
+	uc.initDoHScheme()
 	// DoH/DoH3 requires endpoint is an HTTP url.
 	if uc.Type == ResolverTypeDOH || uc.Type == ResolverTypeDOH3 {
 		u, err := url.Parse(uc.Endpoint)
@@ -690,10 +734,6 @@ func upstreamConfigStructLevelValidation(sl validator.StructLevel) {
 			sl.ReportError(uc.Endpoint, "endpoint", "Endpoint", "http_url", "")
 			return
 		}
-		if u.Scheme != "http" && u.Scheme != "https" {
-			sl.ReportError(uc.Endpoint, "endpoint", "Endpoint", "http_url", "")
-			return
-		}
 	}
 }
 

config_quic.go

@@ -1,5 +1,3 @@
-//go:build !qf
-
 package ctrld
 
 import (

config_quic_free.go

@@ -1,9 +0,0 @@
-//go:build qf
-
-package ctrld
-
-import "net/http"
-
-func (uc *UpstreamConfig) setupDOH3Transport() {}
-
-func (uc *UpstreamConfig) doh3Transport(dnsType uint16) http.RoundTripper { return nil }

config_test.go

@@ -1,6 +1,7 @@
 package ctrld_test
 
 import (
+	"fmt"
 	"os"
 	"strings"
 	"testing"
@@ -102,6 +103,8 @@ func TestConfigValidation(t *testing.T) {
 		{"invalid lease file format", configWithInvalidLeaseFileFormat(t), true},
 		{"invalid doh/doh3 endpoint", configWithInvalidDoHEndpoint(t), true},
 		{"invalid client id pref", configWithInvalidClientIDPref(t), true},
+		{"doh endpoint without scheme", dohUpstreamEndpointWithoutScheme(t), false},
+		{"maximum number of flush cache domains", configWithInvalidFlushCacheDomain(t), true},
 	}
 
 	for _, tc := range tests {
@@ -167,6 +170,12 @@ func invalidUpstreamType(t *testing.T) *ctrld.Config {
 	return cfg
 }
 
+func dohUpstreamEndpointWithoutScheme(t *testing.T) *ctrld.Config {
+	cfg := defaultConfig(t)
+	cfg.Upstream["0"].Endpoint = "freedns.controld.com/p1"
+	return cfg
+}
+
 func invalidUpstreamTimeout(t *testing.T) *ctrld.Config {
 	cfg := defaultConfig(t)
 	cfg.Upstream["0"].Timeout = -1
@@ -258,7 +267,7 @@ func configWithInvalidLeaseFileFormat(t *testing.T) *ctrld.Config {
 
 func configWithInvalidDoHEndpoint(t *testing.T) *ctrld.Config {
 	cfg := defaultConfig(t)
-	cfg.Upstream["0"].Endpoint = "1.1.1.1"
+	cfg.Upstream["0"].Endpoint = "/1.1.1.1"
 	cfg.Upstream["0"].Type = ctrld.ResolverTypeDOH
 	return cfg
 }
@@ -268,3 +277,12 @@ func configWithInvalidClientIDPref(t *testing.T) *ctrld.Config {
 	cfg.Service.ClientIDPref = "foo"
 	return cfg
 }
+
+func configWithInvalidFlushCacheDomain(t *testing.T) *ctrld.Config {
+	cfg := defaultConfig(t)
+	cfg.Service.CacheFlushDomains = make([]string, 257)
+	for i := range cfg.Service.CacheFlushDomains {
+		cfg.Service.CacheFlushDomains[i] = fmt.Sprintf("%d.com", i)
+	}
+	return cfg
+}

docs/config.md

@@ -14,7 +14,7 @@ The config file allows for advanced configuration of the `ctrld` utility to cove
 
 
 ## Config Location
-`ctrld` uses [TOML](toml_link) format for its configuration file. Default configuration file is `ctrld.toml` found in following order:
+`ctrld` uses [TOML][toml_link] format for its configuration file. Default configuration file is `ctrld.toml` found in following order:
 
  - `/etc/controld` on *nix.
  - User's home directory on Windows.
@@ -157,6 +157,13 @@ stale cached records (regardless of their TTLs) until upstream comes online.
 - Required: no
 - Default: false
 
+### cache_flush_domains
+When `ctrld` receives query with domain name in `cache_flush_domains`, the local cache will be discarded
+before serving the query.
+
+- Type: array of strings
+- Required: no
+
 ### max_concurrent_requests
 The number of concurrent requests that will be handled, must be a non-negative integer. 
 Tweaking this value depends on the capacity of your system.
@@ -220,15 +227,12 @@ DHCP leases file format.
 
 - Type: string
 - Required: no
-- Valid values: `dnsmasq`, `isc-dhcp`
+- Valid values: `dnsmasq`, `isc-dhcp`, `kea-dhcp4`
 - Default: ""
 
 ### client_id_preference
-Decide how the client ID is generated
+Decide how the client ID is generated. By default client ID will use both MAC address and Hostname i.e. `hash(mac + host)`. To override this behavior, select one of the 2 allowed values to scope client ID to just MAC address OR Hostname.  
 
-If `host` -> client id will only use the hostname i.e.`hash(hostname)`.
-If `mac` -> client id will only use the MAC address `hash(mac)`.
-Else -> client ID will use both Mac and Hostname i.e. `hash(mac + host)
 - Type: string
 - Required: no
 - Valid values: `mac`, `host`
@@ -242,7 +246,7 @@ If set to `true`, collect and export the query counters, and show them in `clien
 - Default: false
 
 ### metrics_listener
-Specifying the `ip` and `port` of the metrics server.
+Specifying the `ip` and `port` of the Prometheus metrics server. The Prometheus metrics will be available on: `http://ip:port/metrics`. You can also append `/metrics/json` to get the same data in json format. 
 
 - Type: string
 - Required: no
@@ -332,7 +336,7 @@ The protocol that `ctrld` will use to send DNS requests to upstream.
 
  - Type: string
  - Required: yes
- - Valid values: `doh`, `doh3`, `dot`, `doq`, `legacy`, `os`
+ - Valid values: `doh`, `doh3`, `dot`, `doq`, `legacy`
 
 ### ip_stack
 Specifying what kind of ip stack that `ctrld` will use to connect to upstream.
@@ -534,7 +538,7 @@ And within each policy, the rules are processed from top to bottom.
 ### failover_rcodes
 For non success response, `failover_rcodes` allows the request to be forwarded to next upstream, if the response `RCODE` matches any value defined in `failover_rcodes`.
 
-- Type: array of string
+- Type: array of strings
 - Required: no
 - Default: []
 - 
@@ -551,7 +555,7 @@ networks = [
 
 If `upstream.0` returns a NXDOMAIN response, the request will be forwarded to `upstream.1` instead of returning immediately to the client.
 
-See all available DNS Rcodes value [here](rcode_link).
+See all available DNS Rcodes value [here][rcode_link].
 
 [toml_link]: https://toml.io/en
 [rcode_link]: https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-6

doh.go

@@ -60,17 +60,10 @@ func init() {
 	}
 }
 
-// TODO: use sync.OnceValue when upgrading to go1.21
-var xCdOsValueOnce sync.Once
-var xCdOsValue string
-
-func dohOsHeaderValue() string {
-	xCdOsValueOnce.Do(func() {
-		oi := osinfo.New()
-		xCdOsValue = strings.Join([]string{EncodeOsNameMap[runtime.GOOS], EncodeArchNameMap[runtime.GOARCH], oi.Dist}, "-")
-	})
-	return xCdOsValue
-}
+var dohOsHeaderValue = sync.OnceValue(func() string {
+	oi := osinfo.New()
+	return strings.Join([]string{EncodeOsNameMap[runtime.GOOS], EncodeArchNameMap[runtime.GOARCH], oi.Dist}, "-")
+})()
 
 func newDohResolver(uc *UpstreamConfig) *dohResolver {
 	r := &dohResolver{
@@ -172,7 +165,6 @@ func addHeader(ctx context.Context, req *http.Request, uc *UpstreamConfig) {
 // newControlDHeaders returns DoH/Doh3 HTTP request headers for ControlD upstream.
 func newControlDHeaders(ci *ClientInfo) http.Header {
 	header := make(http.Header)
-	header.Set(dohOsHeader, dohOsHeaderValue())
 	if ci.Mac != "" {
 		header.Set(dohMacHeader, ci.Mac)
 	}
@@ -183,7 +175,7 @@ func newControlDHeaders(ci *ClientInfo) http.Header {
 		header.Set(dohHostHeader, ci.Hostname)
 	}
 	if ci.Self {
-		header.Set(dohOsHeader, dohOsHeaderValue())
+		header.Set(dohOsHeader, dohOsHeaderValue)
 	}
 	switch ci.ClientIDPref {
 	case "mac":

doh_test.go

@@ -6,7 +6,7 @@ import (
 )
 
 func Test_dohOsHeaderValue(t *testing.T) {
-	val := dohOsHeaderValue()
+	val := dohOsHeaderValue
 	if val == "" {
 		t.Fatalf("empty %s", dohOsHeader)
 	}

go.mod

@@ -1,8 +1,9 @@
 module github.com/Control-D-Inc/ctrld
 
-go 1.20
+go 1.21
 
 require (
+	github.com/Masterminds/semver v1.5.0
 	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/cuonglm/osinfo v0.0.0-20230921071424-e0e1b1e0bbbf
 	github.com/frankban/quicktest v1.14.5
@@ -17,25 +18,28 @@ require (
 	github.com/kardianos/service v1.2.1
 	github.com/mdlayher/ndp v1.0.1
 	github.com/miekg/dns v1.1.55
+	github.com/minio/selfupdate v0.6.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pelletier/go-toml/v2 v2.0.8
 	github.com/prometheus/client_golang v1.15.1
+	github.com/prometheus/client_model v0.4.0
 	github.com/prometheus/prom2json v1.3.3
-	github.com/quic-go/quic-go v0.38.0
+	github.com/quic-go/quic-go v0.42.0
 	github.com/rs/zerolog v1.28.0
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.3
 	github.com/vishvananda/netlink v1.2.1-beta.2
-	golang.org/x/net v0.17.0
+	golang.org/x/net v0.23.0
 	golang.org/x/sync v0.2.0
-	golang.org/x/sys v0.13.0
+	golang.org/x/sys v0.18.0
 	golang.zx2c4.com/wireguard/windows v0.5.3
 	tailscale.com v1.44.0
 )
 
 require (
+	aead.dev/minisign v0.2.0 // indirect
 	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -43,7 +47,6 @@ require (
 	github.com/go-playground/locales v0.14.0 // indirect
 	github.com/go-playground/universal-translator v0.18.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
-	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
@@ -66,11 +69,9 @@ require (
 	github.com/onsi/ginkgo/v2 v2.9.5 // indirect
 	github.com/pierrec/lz4/v4 v4.1.17 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
-	github.com/quic-go/qtls-go1-20 v0.3.2 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
@@ -79,13 +80,14 @@ require (
 	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/u-root/uio v0.0.0-20230305220412-3e8cd9d6bf63 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
+	go.uber.org/mock v0.4.0 // indirect
 	go4.org/mem v0.0.0-20220726221520-4f986261bf13 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect
-	golang.org/x/mod v0.10.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/mod v0.11.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.9.1 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,3 +1,5 @@
+aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk=
+aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -38,6 +40,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Windscribe/zerolog v0.0.0-20230503170159-e6aa153233be h1:qBKVRi7Mom5heOkyZ+NCIu9HZBiNCsRqrRe5t9pooik=
 github.com/Windscribe/zerolog v0.0.0-20230503170159-e6aa153233be/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
 github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
@@ -51,6 +55,7 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cilium/ebpf v0.10.0 h1:nk5HPMeoBXtOzbkZBWym+ZWq1GIiHUsBFXxwewXAHLQ=
+github.com/cilium/ebpf v0.10.0/go.mod h1:DPiVdY/kT534dgc9ERmvP8mWA+9gvwgKfRvk4nNWnoE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -78,6 +83,7 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
@@ -102,8 +108,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
-github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -162,6 +166,7 @@ github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyf
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8=
+github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/illarion/gonotify v1.0.1 h1:F1d+0Fgbq/sDWjj/r66ekjDG+IDeecQKUFH4wNwsoio=
@@ -221,6 +226,8 @@ github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U
 github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=
 github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo=
 github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
+github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU=
+github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
@@ -228,6 +235,7 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6
 github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
 github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
 github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
+github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
 github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
 github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
 github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@@ -251,10 +259,8 @@ github.com/prometheus/prom2json v1.3.3 h1:IYfSMiZ7sSOfliBoo89PcufjWO4eAR0gznGcET
 github.com/prometheus/prom2json v1.3.3/go.mod h1:Pv4yIPktEkK7btWsrUTWDDDrnpUrAELaOCj+oFwlgmc=
 github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
 github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
-github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI=
-github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
-github.com/quic-go/quic-go v0.38.0 h1:T45lASr5q/TrVwt+jrVccmqHhPL2XuSyoCLVCpfOSLc=
-github.com/quic-go/quic-go v0.38.0/go.mod h1:MPCuRq7KBK2hNcfKj/1iD1BGuN3eAYMeNxp3T42LRUg=
+github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=
+github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -304,13 +310,14 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
+go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 go4.org/mem v0.0.0-20220726221520-4f986261bf13 h1:CbZeCBZ0aZj8EfVgnqQcYZgf0lpZ3H9rmp5nkDTAst8=
 go4.org/mem v0.0.0-20220726221520-4f986261bf13/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -318,11 +325,13 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -358,9 +367,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
-golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
+golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -393,10 +401,9 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -416,7 +423,6 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
 golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -431,6 +437,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -456,10 +463,9 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -469,8 +475,9 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -480,11 +487,13 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -532,7 +541,6 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
-golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
 golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -631,8 +639,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

internal/clientinfo/client_info.go

@@ -14,6 +14,11 @@ import (
 	"github.com/Control-D-Inc/ctrld/internal/controld"
 )
 
+const (
+	ipV4Loopback = "127.0.0.1"
+	ipv6Loopback = "::1"
+)
+
 // IpResolver is the interface for retrieving IP from Mac.
 type IpResolver interface {
 	fmt.Stringer
@@ -224,6 +229,7 @@ func (t *Table) init() {
 			cancel()
 		}()
 		go t.ndp.listen(ctx)
+		go t.ndp.subscribe(ctx)
 	}
 	// PTR lookup.
 	if t.discoverPTR() {
@@ -321,6 +327,16 @@ func (t *Table) LookupRFC1918IPv4(mac string) string {
 	return ""
 }
 
+// LocalHostname returns the localhost hostname associated with loopback IP.
+func (t *Table) LocalHostname() string {
+	for _, ip := range []string{ipV4Loopback, ipv6Loopback} {
+		if name := t.LookupHostname(ip, ""); name != "" {
+			return name
+		}
+	}
+	return ""
+}
+
 type macEntry struct {
 	mac string
 	src string
@@ -384,6 +400,7 @@ func (t *Table) ListClients() []*Client {
 			}
 		}
 	}
+	clientsByMAC := make(map[string]*Client)
 	for ip := range ipMap {
 		c := ipMap[ip]
 		for _, e := range t.lookupMacAll(ip) {
@@ -397,6 +414,7 @@ func (t *Table) ListClients() []*Client {
 		for _, e := range t.lookupHostnameAll(ip, c.Mac) {
 			if c.Hostname == "" && e.name != "" {
 				c.Hostname = e.name
+				clientsByMAC[c.Mac] = c
 			}
 			if e.name != "" {
 				c.Source[e.src] = struct{}{}
@@ -405,6 +423,11 @@ func (t *Table) ListClients() []*Client {
 	}
 	clients := make([]*Client, 0, len(ipMap))
 	for _, c := range ipMap {
+		// If we found a client with empty hostname, use hostname from
+		// an existed client which has the same MAC address.
+		if cFromMac := clientsByMAC[c.Mac]; cFromMac != nil && c.Hostname == "" {
+			c.Hostname = cFromMac.Hostname
+		}
 		clients = append(clients, c)
 	}
 	return clients

internal/clientinfo/client_info_test.go

@@ -44,3 +44,31 @@ func TestTable_LookupRFC1918IPv4(t *testing.T) {
 		t.Fatalf("unexpected result, want: %s, got: %s", rfc1918IPv4, got)
 	}
 }
+
+func TestTable_ListClients(t *testing.T) {
+	mac := "74:56:3c:44:eb:5e"
+	ipv6_1 := "2405:4803:a04b:4190:fbe9:cd14:d522:bbae"
+	ipv6_2 := "2405:4803:a04b:4190:fbe9:cd14:d522:bbab"
+	table := &Table{}
+
+	// NDP init.
+	table.ndp = &ndpDiscover{}
+	table.ndp.mac.Store(ipv6_1, mac)
+	table.ndp.mac.Store(ipv6_2, mac)
+	table.ndp.ip.Store(mac, ipv6_1)
+	table.ndp.ip.Store(mac, ipv6_2)
+	table.ipResolvers = append(table.ipResolvers, table.ndp)
+	table.macResolvers = append(table.macResolvers, table.ndp)
+
+	hostname := "foo"
+	// mdns init.
+	table.mdns = &mdns{}
+	table.mdns.name.Store(ipv6_2, hostname)
+	table.hostnameResolvers = append(table.hostnameResolvers, table.mdns)
+
+	for _, c := range table.ListClients() {
+		if c.Hostname != hostname {
+			t.Fatalf("missing hostname for client: %v", c)
+		}
+	}
+}

internal/clientinfo/dhcp.go

@@ -353,8 +353,8 @@ func (d *dhcp) addSelf() {
 		return
 	}
 	hostname = normalizeHostname(hostname)
-	d.ip2name.Store("127.0.0.1", hostname)
-	d.ip2name.Store("::1", hostname)
+	d.ip2name.Store(ipV4Loopback, hostname)
+	d.ip2name.Store(ipv6Loopback, hostname)
 	found := false
 	interfaces.ForeachInterface(func(i interfaces.Interface, prefixes []netip.Prefix) {
 		mac := i.HardwareAddr.String()
@@ -375,15 +375,17 @@ func (d *dhcp) addSelf() {
 			d.mac.Store(ip.String(), mac)
 			d.ip.Store(mac, ip.String())
 			if ip.To4() != nil {
-				d.mac.Store("127.0.0.1", mac)
+				d.mac.Store(ipV4Loopback, mac)
 			} else {
-				d.mac.Store("::1", mac)
+				d.mac.Store(ipv6Loopback, mac)
 			}
 			d.mac2name.Store(mac, hostname)
 			d.ip2name.Store(ip.String(), hostname)
 			// If we have self IP set, and this IP is it, use this IP only.
 			if ip.String() == d.selfIP {
 				found = true
+				d.mac.Store(ipV4Loopback, mac)
+				d.mac.Store(ipv6Loopback, mac)
 			}
 		}
 	})

internal/clientinfo/hostsfile.go

@@ -95,7 +95,7 @@ func (hf *hostsFile) LookupHostnameByIP(ip string) string {
 	hf.mu.Lock()
 	defer hf.mu.Unlock()
 	if names := hf.m[ip]; len(names) > 0 {
-		isLoopback := ip == "127.0.0.1" || ip == "::1"
+		isLoopback := ip == ipV4Loopback || ip == ipv6Loopback
 		for _, hostname := range names {
 			name := normalizeHostname(hostname)
 			// Ignoring ipv4/ipv6 loopback entry.

internal/clientinfo/ndp.go

@@ -15,6 +15,7 @@ import (
 	"github.com/mdlayher/ndp"
 
 	"github.com/Control-D-Inc/ctrld"
+	ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
 )
 
 // ndpDiscover provides client discovery functionality using NDP protocol.
@@ -69,15 +70,45 @@ func (nd *ndpDiscover) List() []string {
 	return ips
 }
 
+// saveInfo saves ip and mac info to mapping table.
+func (nd *ndpDiscover) saveInfo(ip, mac string) {
+	ip = normalizeIP(ip)
+	// Store ip => map mapping,
+	nd.mac.Store(ip, mac)
+
+	// Do not store mac => ip mapping if new ip is a link local unicast.
+	if ctrldnet.IsLinkLocalUnicastIPv6(ip) {
+		return
+	}
+
+	// If there is old ip => mac mapping, delete it.
+	if old, existed := nd.ip.Load(mac); existed {
+		oldIP := old.(string)
+		if oldIP != ip {
+			nd.mac.Delete(oldIP)
+		}
+	}
+	// Store mac => ip mapping.
+	nd.ip.Store(mac, ip)
+}
+
 // listen listens on ipv6 link local for Neighbor Solicitation message
 // to update new neighbors information to ndp table.
 func (nd *ndpDiscover) listen(ctx context.Context) {
-	ifi, err := firstInterfaceWithV6LinkLocal()
+	ifis, err := allInterfacesWithV6LinkLocal()
 	if err != nil {
-		ctrld.ProxyLogger.Load().Debug().Err(err).Msg("failed to find valid ipv6")
+		ctrld.ProxyLogger.Load().Debug().Err(err).Msg("failed to find valid ipv6 interfaces")
 		return
 	}
-	c, ip, err := ndp.Listen(ifi, ndp.LinkLocal)
+	for _, ifi := range ifis {
+		go func(ifi *net.Interface) {
+			nd.listenOnInterface(ctx, ifi)
+		}(ifi)
+	}
+}
+
+func (nd *ndpDiscover) listenOnInterface(ctx context.Context, ifi *net.Interface) {
+	c, ip, err := ndp.Listen(ifi, ndp.Unspecified)
 	if err != nil {
 		ctrld.ProxyLogger.Load().Debug().Err(err).Msg("ndp listen failed")
 		return
@@ -111,8 +142,7 @@ func (nd *ndpDiscover) listen(ctx context.Context) {
 		for _, opt := range am.Options {
 			if lla, ok := opt.(*ndp.LinkLayerAddress); ok {
 				mac := lla.Addr.String()
-				nd.mac.Store(fromIP, mac)
-				nd.ip.Store(mac, fromIP)
+				nd.saveInfo(fromIP, mac)
 			}
 		}
 	}
@@ -127,8 +157,7 @@ func (nd *ndpDiscover) scanWindows(r io.Reader) {
 			continue
 		}
 		if mac := parseMAC(fields[1]); mac != "" {
-			nd.mac.Store(fields[0], mac)
-			nd.ip.Store(mac, fields[0])
+			nd.saveInfo(fields[0], mac)
 		}
 	}
 }
@@ -147,8 +176,7 @@ func (nd *ndpDiscover) scanUnix(r io.Reader) {
 			if idx := strings.IndexByte(ip, '%'); idx != -1 {
 				ip = ip[:idx]
 			}
-			nd.mac.Store(ip, mac)
-			nd.ip.Store(mac, ip)
+			nd.saveInfo(ip, mac)
 		}
 	}
 }
@@ -183,14 +211,15 @@ func parseMAC(mac string) string {
 	return hw.String()
 }
 
-// firstInterfaceWithV6LinkLocal returns the first interface which is capable of using NDP.
-func firstInterfaceWithV6LinkLocal() (*net.Interface, error) {
+// allInterfacesWithV6LinkLocal returns all interfaces which is capable of using NDP.
+func allInterfacesWithV6LinkLocal() ([]*net.Interface, error) {
 	ifis, err := net.Interfaces()
 	if err != nil {
 		return nil, err
 	}
-
+	res := make([]*net.Interface, 0, len(ifis))
 	for _, ifi := range ifis {
+		ifi := ifi
 		// Skip if iface is down/loopback/non-multicast.
 		if ifi.Flags&net.FlagUp == 0 || ifi.Flags&net.FlagLoopback != 0 || ifi.Flags&net.FlagMulticast == 0 {
 			continue
@@ -211,9 +240,10 @@ func firstInterfaceWithV6LinkLocal() (*net.Interface, error) {
 				return nil, fmt.Errorf("invalid ip address: %s", ipNet.String())
 			}
 			if ip.Is6() && !ip.Is4In6() {
-				return &ifi, nil
+				res = append(res, &ifi)
+				break
 			}
 		}
 	}
-	return nil, errors.New("no interface can be used")
+	return res, nil
 }

internal/clientinfo/ndp_linux.go

@@ -1,7 +1,10 @@
 package clientinfo
 
 import (
+	"context"
+
 	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 
 	"github.com/Control-D-Inc/ctrld"
 )
@@ -15,10 +18,47 @@ func (nd *ndpDiscover) scan() {
 	}
 
 	for _, n := range neighs {
+		// Skipping non-reachable neighbors.
+		if n.State&netlink.NUD_REACHABLE == 0 {
+			continue
+		}
 		ip := n.IP.String()
 		mac := n.HardwareAddr.String()
-		nd.mac.Store(ip, mac)
-		nd.ip.Store(mac, ip)
+		nd.saveInfo(ip, mac)
+	}
+}
+
+// subscribe watches NDP table changes and update new information to local table.
+func (nd *ndpDiscover) subscribe(ctx context.Context) {
+	ch := make(chan netlink.NeighUpdate)
+	done := make(chan struct{})
+	defer close(done)
+	if err := netlink.NeighSubscribe(ch, done); err != nil {
+		ctrld.ProxyLogger.Load().Err(err).Msg("could not perform neighbor subscribing")
+		return
+	}
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case nu := <-ch:
+			if nu.Family != netlink.FAMILY_V6 {
+				continue
+			}
+			ip := normalizeIP(nu.IP.String())
+			if nu.Type == unix.RTM_DELNEIGH {
+				ctrld.ProxyLogger.Load().Debug().Msgf("removing NDP neighbor: %s", ip)
+				nd.mac.Delete(ip)
+				continue
+			}
+			mac := nu.HardwareAddr.String()
+			switch nu.State {
+			case netlink.NUD_REACHABLE:
+				nd.saveInfo(ip, mac)
+			case netlink.NUD_FAILED:
+				ctrld.ProxyLogger.Load().Debug().Msgf("removing NDP neighbor with failed state: %s", ip)
+				nd.mac.Delete(ip)
+			}
+		}
 	}
-
 }

internal/clientinfo/ndp_others.go

@@ -4,6 +4,7 @@ package clientinfo
 
 import (
 	"bytes"
+	"context"
 	"os/exec"
 	"runtime"
 
@@ -29,3 +30,7 @@ func (nd *ndpDiscover) scan() {
 		nd.scanUnix(bytes.NewReader(data))
 	}
 }
+
+// subscribe watches NDP table changes and update new information to local table.
+// This is a stub method, and only works on Linux at this moment.
+func (nd *ndpDiscover) subscribe(ctx context.Context) {}

internal/clientinfo/ndp_test.go

@@ -45,20 +45,22 @@ ff02::c                                       33-33-00-00-00-0c  Permanent
 	nd.scanWindows(r)
 
 	count := 0
+	expectedCount := 6
 	nd.mac.Range(func(key, value any) bool {
 		count++
 		return true
 	})
-	if count != 6 {
-		t.Errorf("unexpected count, want 6, got: %d", count)
+	if count != expectedCount {
+		t.Errorf("unexpected count, want %d, got: %d", expectedCount, count)
 	}
 
 	count = 0
+	expectedCount = 4
 	nd.ip.Range(func(key, value any) bool {
 		count++
 		return true
 	})
-	if count != 5 {
-		t.Errorf("unexpected count, want 5, got: %d", count)
+	if count != expectedCount {
+		t.Errorf("unexpected count, want %d, got: %d", expectedCount, count)
 	}
 }

internal/controld/config.go

@@ -35,8 +35,9 @@ type ResolverConfig struct {
 	Ctrld struct {
 		CustomConfig string `json:"custom_config"`
 	} `json:"ctrld"`
-	Exclude []string `json:"exclude"`
-	UID     string   `json:"uid"`
+	Exclude         []string `json:"exclude"`
+	UID             string   `json:"uid"`
+	DeactivationPin *int64   `json:"deactivation_pin,omitempty"`
 }
 
 type utilityResponse struct {

internal/dnscache/cache.go

@@ -12,6 +12,7 @@ import (
 type Cacher interface {
 	Get(Key) *Value
 	Add(Key, *Value)
+	Purge()
 }
 
 // Key is the caching key for DNS message.
@@ -34,15 +35,22 @@ type LRUCache struct {
 	cacher *lru.ARCCache[Key, *Value]
 }
 
+// Get looks up key's value from cache.
 func (l *LRUCache) Get(key Key) *Value {
 	v, _ := l.cacher.Get(key)
 	return v
 }
 
+// Add adds a value to cache.
 func (l *LRUCache) Add(key Key, value *Value) {
 	l.cacher.Add(key, value)
 }
 
+// Purge clears the cache.
+func (l *LRUCache) Purge() {
+	l.cacher.Purge()
+}
+
 // NewLRUCache creates a new LRUCache instance with given size.
 func NewLRUCache(size int) (*LRUCache, error) {
 	cacher, err := lru.NewARC[Key, *Value](size)

internal/net/net.go

@@ -115,6 +115,15 @@ func IsIPv6(ip string) bool {
 	return parsedIP != nil && parsedIP.To4() == nil && parsedIP.To16() != nil
 }
 
+// IsLinkLocalUnicastIPv6 checks if the provided IP is a link local unicast v6 address.
+func IsLinkLocalUnicastIPv6(ip string) bool {
+	parsedIP := net.ParseIP(ip)
+	if parsedIP == nil || parsedIP.To4() != nil || parsedIP.To16() == nil {
+		return false
+	}
+	return parsedIP.To16().IsLinkLocalUnicast()
+}
+
 type parallelDialerResult struct {
 	conn net.Conn
 	err  error

internal/router/dnsmasq/dnsmasq.go

@@ -10,6 +10,8 @@ import (
 	"github.com/Control-D-Inc/ctrld"
 )
 
+const CtrldMarker = `# GENERATED BY ctrld - DO NOT MODIFY`
+
 const ConfigContentTmpl = `# GENERATED BY ctrld - DO NOT MODIFY
 no-resolv
 {{- range .Upstreams}}

internal/router/netgear_orbi_voxel/procd.go

@@ -0,0 +1,22 @@
+package netgear
+
+const openWrtScript = `#!/bin/sh /etc/rc.common
+USE_PROCD=1
+# After dnsmasq starts
+START=61
+# Before network stops
+STOP=89
+cmd="{{.Path}}{{range .Arguments}} {{.|cmd}}{{end}}"
+name="{{.Name}}"
+pid_file="/var/run/${name}.pid"
+
+start_service() {
+    echo "Starting ${name}"
+    procd_open_instance
+    procd_set_param command ${cmd}
+    procd_set_param respawn             # respawn automatically if something died
+    procd_set_param pidfile ${pid_file} # write a pid file on instance start and remove it on stop
+    procd_close_instance
+    echo "${name} has been started"
+}
+`

internal/router/netgear_orbi_voxel/voxel.go

@@ -0,0 +1,220 @@
+package netgear
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/kardianos/service"
+
+	"github.com/Control-D-Inc/ctrld"
+	"github.com/Control-D-Inc/ctrld/internal/router/dnsmasq"
+	"github.com/Control-D-Inc/ctrld/internal/router/nvram"
+)
+
+const (
+	Name                                = "netgear_orbi_voxel"
+	netgearOrbiVoxelDNSMasqConfigPath   = "/etc/dnsmasq.conf"
+	netgearOrbiVoxelHomedir             = "/mnt/bitdefender"
+	netgearOrbiVoxelStartupScript       = "/mnt/bitdefender/rc.user"
+	netgearOrbiVoxelStartupScriptBackup = "/mnt/bitdefender/rc.user.bak"
+	netgearOrbiVoxelStartupScriptMarker = "\n# GENERATED BY ctrld"
+)
+
+var nvramKvMap = map[string]string{
+	"dns_hijack": "0", // Disable dns hijacking
+}
+
+type NetgearOrbiVoxel struct {
+	cfg *ctrld.Config
+}
+
+// New returns a router.Router for configuring/setup/run ctrld on ddwrt routers.
+func New(cfg *ctrld.Config) *NetgearOrbiVoxel {
+	return &NetgearOrbiVoxel{cfg: cfg}
+}
+
+func (d *NetgearOrbiVoxel) ConfigureService(svc *service.Config) error {
+	if err := d.checkInstalledDir(); err != nil {
+		return err
+	}
+	svc.Option["SysvScript"] = openWrtScript
+	return nil
+}
+
+func (d *NetgearOrbiVoxel) Install(_ *service.Config) error {
+	// Ignoring error here at this moment is ok, since everything will be wiped out on reboot.
+	_ = exec.Command("/etc/init.d/ctrld", "enable").Run()
+	if err := d.checkInstalledDir(); err != nil {
+		return err
+	}
+	if err := backupVoxelStartupScript(); err != nil {
+		return fmt.Errorf("backup startup script: %w", err)
+	}
+	if err := writeVoxelStartupScript(); err != nil {
+		return fmt.Errorf("writing startup script: %w", err)
+	}
+	return nil
+}
+
+func (d *NetgearOrbiVoxel) Uninstall(_ *service.Config) error {
+	if err := os.Remove(netgearOrbiVoxelStartupScript); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	err := os.Rename(netgearOrbiVoxelStartupScriptBackup, netgearOrbiVoxelStartupScript)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+func (d *NetgearOrbiVoxel) PreRun() error {
+	return nil
+}
+
+func (d *NetgearOrbiVoxel) Setup() error {
+	if d.cfg.FirstListener().IsDirectDnsListener() {
+		return nil
+	}
+	// Already setup.
+	if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val == "1" {
+		return nil
+	}
+
+	data, err := dnsmasq.ConfTmplWithCacheDisabled(dnsmasq.ConfigContentTmpl, d.cfg, false)
+	if err != nil {
+		return err
+	}
+	currentConfig, _ := os.ReadFile(netgearOrbiVoxelDNSMasqConfigPath)
+	configContent := append(currentConfig, data...)
+	if err := os.WriteFile(netgearOrbiVoxelDNSMasqConfigPath, configContent, 0600); err != nil {
+		return err
+	}
+	// Restart dnsmasq service.
+	if err := restartDNSMasq(); err != nil {
+		return err
+	}
+
+	if err := nvram.SetKV(nvramKvMap, nvram.CtrldSetupKey); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *NetgearOrbiVoxel) Cleanup() error {
+	if d.cfg.FirstListener().IsDirectDnsListener() {
+		return nil
+	}
+	if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val != "1" {
+		return nil // was restored, nothing to do.
+	}
+
+	// Restore old configs.
+	if err := nvram.Restore(nvramKvMap, nvram.CtrldSetupKey); err != nil {
+		return err
+	}
+
+	// Restore dnsmasq config.
+	if err := restoreDnsmasqConf(); err != nil {
+		return err
+	}
+
+	// Restart dnsmasq service.
+	if err := restartDNSMasq(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// checkInstalledDir checks that ctrld binary was installed in the correct directory.
+func (d *NetgearOrbiVoxel) checkInstalledDir() error {
+	exePath, err := os.Executable()
+	if err != nil {
+		return fmt.Errorf("checkHomeDir: failed to get binary path %w", err)
+	}
+	if !strings.HasSuffix(filepath.Dir(exePath), netgearOrbiVoxelHomedir) {
+		return fmt.Errorf("checkHomeDir: could not install service outside %s", netgearOrbiVoxelHomedir)
+	}
+	return nil
+}
+
+// backupVoxelStartupScript creates a backup of original startup script if existed.
+func backupVoxelStartupScript() error {
+	// Do nothing if the startup script was modified by ctrld.
+	script, _ := os.ReadFile(netgearOrbiVoxelStartupScript)
+	if bytes.Contains(script, []byte(netgearOrbiVoxelStartupScriptMarker)) {
+		return nil
+	}
+	err := os.Rename(netgearOrbiVoxelStartupScript, netgearOrbiVoxelStartupScriptBackup)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("backupVoxelStartupScript: %w", err)
+	}
+	return nil
+}
+
+// writeVoxelStartupScript writes startup script to re-install ctrld upon reboot.
+// See: https://github.com/SVoxel/ORBI-RBK50/pull/7
+func writeVoxelStartupScript() error {
+	exe, err := os.Executable()
+	if err != nil {
+		return fmt.Errorf("configure service: failed to get binary path %w", err)
+	}
+	// This is called when "ctrld start ..." runs, so recording
+	// the same command line arguments to use in startup script.
+	argStr := strings.Join(os.Args[1:], " ")
+	script, _ := os.ReadFile(netgearOrbiVoxelStartupScriptBackup)
+	script = append(script, fmt.Sprintf("%s\n%q %s\n", netgearOrbiVoxelStartupScriptMarker, exe, argStr)...)
+	f, err := os.Create(netgearOrbiVoxelStartupScript)
+	if err != nil {
+		return fmt.Errorf("failed to create startup script: %w", err)
+	}
+	defer f.Close()
+
+	if _, err := f.Write(script); err != nil {
+		return fmt.Errorf("failed to write startup script: %w", err)
+	}
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("failed to save startup script: %w", err)
+	}
+	return nil
+}
+
+// restoreDnsmasqConf restores original dnsmasq configuration.
+func restoreDnsmasqConf() error {
+	f, err := os.Open(netgearOrbiVoxelDNSMasqConfigPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	var bs []byte
+	buf := bytes.NewBuffer(bs)
+
+	removed := false
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == dnsmasq.CtrldMarker {
+			removed = true
+		}
+		if !removed {
+			_, err := buf.WriteString(line + "\n")
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return os.WriteFile(netgearOrbiVoxelDNSMasqConfigPath, buf.Bytes(), 0644)
+}
+
+func restartDNSMasq() error {
+	if out, err := exec.Command("/etc/init.d/dnsmasq", "restart").CombinedOutput(); err != nil {
+		return fmt.Errorf("restartDNSMasq: %s, %w", string(out), err)
+	}
+	return nil
+}

internal/router/openwrt/procd.go

@@ -18,6 +18,7 @@ start_service() {
     procd_set_param stdout 1            # forward stdout of the command to logd
     procd_set_param stderr 1            # same for stderr
     procd_set_param pidfile ${pid_file} # write a pid file on instance start and remove it on stop
+    procd_set_param term_timeout 10
     procd_close_instance
     echo "${name} has been started"
 }

internal/router/os_config_freebsd.go

@@ -0,0 +1,40 @@
+package router
+
+import (
+	"encoding/xml"
+	"os"
+)
+
+// Config represents /conf/config.xml file found on pfsense/opnsense.
+type Config struct {
+	PfsenseUnbound  *string `xml:"unbound>enable,omitempty"`
+	OPNsenseUnbound *string `xml:"OPNsense>unboundplus>general>enabled,omitempty"`
+	Dnsmasq         *string `xml:"dnsmasq>enable,omitempty"`
+}
+
+// DnsmasqEnabled reports whether dnsmasq is enabled.
+func (c *Config) DnsmasqEnabled() bool {
+	if isPfsense() { // pfsense only set the attribute if dnsmasq is enabled.
+		return c.Dnsmasq != nil
+	}
+	return c.Dnsmasq != nil && *c.Dnsmasq == "1"
+}
+
+// UnboundEnabled reports whether unbound is enabled.
+func (c *Config) UnboundEnabled() bool {
+	if isPfsense() { // pfsense only set the attribute if unbound is enabled.
+		return c.PfsenseUnbound != nil
+	}
+	return c.OPNsenseUnbound != nil && *c.OPNsenseUnbound == "1"
+}
+
+// currentConfig does unmarshalling /conf/config.xml file,
+// return the corresponding *Config represent it.
+func currentConfig() (*Config, error) {
+	buf, _ := os.ReadFile("/conf/config.xml")
+	c := Config{}
+	if err := xml.Unmarshal(buf, &c); err != nil {
+		return nil, err
+	}
+	return &c, nil
+}

internal/router/os_freebsd.go

@@ -111,8 +111,16 @@ func (or *osRouter) Setup() error {
 
 func (or *osRouter) Cleanup() error {
 	if or.cdMode {
-		_ = exec.Command(unboundRcPath, "onerestart").Run()
-		_ = exec.Command(dnsmasqRcPath, "onerestart").Run()
+		c, err := currentConfig()
+		if err != nil {
+			return err
+		}
+		if c.UnboundEnabled() {
+			_ = exec.Command(unboundRcPath, "onerestart").Run()
+		}
+		if c.DnsmasqEnabled() {
+			_ = exec.Command(dnsmasqRcPath, "onerestart").Run()
+		}
 	}
 	return nil
 }

internal/router/router.go

@@ -18,6 +18,7 @@ import (
 	"github.com/Control-D-Inc/ctrld/internal/router/edgeos"
 	"github.com/Control-D-Inc/ctrld/internal/router/firewalla"
 	"github.com/Control-D-Inc/ctrld/internal/router/merlin"
+	netgear "github.com/Control-D-Inc/ctrld/internal/router/netgear_orbi_voxel"
 	"github.com/Control-D-Inc/ctrld/internal/router/openwrt"
 	"github.com/Control-D-Inc/ctrld/internal/router/synology"
 	"github.com/Control-D-Inc/ctrld/internal/router/tomato"
@@ -66,10 +67,17 @@ func New(cfg *ctrld.Config, cdMode bool) Router {
 		return tomato.New(cfg)
 	case firewalla.Name:
 		return firewalla.New(cfg)
+	case netgear.Name:
+		return netgear.New(cfg)
 	}
 	return newOsRouter(cfg, cdMode)
 }
 
+// IsNetGearOrbi reports whether the router is a Netgear Orbi router.
+func IsNetGearOrbi() bool {
+	return Name() == netgear.Name
+}
+
 // IsGLiNet reports whether the router is an GL.iNet router.
 func IsGLiNet() bool {
 	if Name() != openwrt.Name {
@@ -90,6 +98,11 @@ func IsOldOpenwrt() bool {
 	return cmd == ""
 }
 
+// WaitProcessExited reports whether the "ctrld stop" command have to wait until ctrld process exited.
+func WaitProcessExited() bool {
+	return Name() == openwrt.Name
+}
+
 var routerPlatform atomic.Pointer[router]
 
 type router struct {
@@ -145,7 +158,7 @@ func LocalResolverIP() string {
 // HomeDir returns the home directory of ctrld on current router.
 func HomeDir() (string, error) {
 	switch Name() {
-	case ddwrt.Name, merlin.Name, tomato.Name:
+	case ddwrt.Name, firewalla.Name, merlin.Name, netgear.Name, tomato.Name:
 		exe, err := os.Executable()
 		if err != nil {
 			return "", err
@@ -198,8 +211,11 @@ func distroName() string {
 	case bytes.HasPrefix(unameO(), []byte("ASUSWRT-Merlin")):
 		return merlin.Name
 	case haveFile("/etc/openwrt_version"):
+		if haveFile("/bin/config") { // TODO: is there any more reliable way?
+			return netgear.Name
+		}
 		return openwrt.Name
-	case haveDir("/data/unifi"):
+	case isUbios():
 		return ubios.Name
 	case bytes.HasPrefix(unameU(), []byte("synology")):
 		return synology.Name
@@ -234,3 +250,14 @@ func unameU() []byte {
 	out, _ := exec.Command("uname", "-u").Output()
 	return out
 }
+
+// isUbios reports whether the current machine is running on Ubios.
+func isUbios() bool {
+	if haveDir("/data/unifi") {
+		return true
+	}
+	if err := exec.Command("ubnt-device-info", "firmware").Run(); err == nil {
+		return true
+	}
+	return false
+}

internal/router/service_merlin.go

@@ -49,11 +49,15 @@ func (s *merlinSvc) Platform() string {
 }
 
 func (s *merlinSvc) configPath() string {
-	path, err := os.Executable()
-	if err != nil {
-		return ""
+	bin := s.Config.Executable
+	if bin == "" {
+		path, err := os.Executable()
+		if err != nil {
+			return ""
+		}
+		bin = path
 	}
-	return path + ".startup"
+	return bin + ".startup"
 }
 
 func (s *merlinSvc) template() *template.Template {

nameservers_windows.go

@@ -1,12 +1,9 @@
 package ctrld
 
 import (
-	"net"
 	"syscall"
 
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
-
-	"golang.org/x/sys/windows"
 )
 
 func dnsFns() []dnsFn {
@@ -20,40 +17,23 @@ func dnsFromAdapter() []string {
 	}
 	ns := make([]string, 0, len(aas)*2)
 	seen := make(map[string]bool)
-	do := func(addr windows.SocketAddress) {
-		sa, err := addr.Sockaddr.Sockaddr()
-		if err != nil {
-			return
+	addressMap := make(map[string]struct{})
+	for _, aa := range aas {
+		for a := aa.FirstUnicastAddress; a != nil; a = a.Next {
+			addressMap[a.Address.IP().String()] = struct{}{}
 		}
-		var ip net.IP
-		switch sa := sa.(type) {
-		case *syscall.SockaddrInet4:
-			ip = net.IPv4(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3])
-		case *syscall.SockaddrInet6:
-			ip = make(net.IP, net.IPv6len)
-			copy(ip, sa.Addr[:])
-			if ip[0] == 0xfe && ip[1] == 0xc0 {
-				// Ignore these fec0/10 ones. Windows seems to
-				// populate them as defaults on its misc rando
-				// interfaces.
-				return
-			}
-		default:
-			return
-
-		}
-		if ip.IsLoopback() || seen[ip.String()] {
-			return
-		}
-		seen[ip.String()] = true
-		ns = append(ns, ip.String())
 	}
 	for _, aa := range aas {
 		for dns := aa.FirstDNSServerAddress; dns != nil; dns = dns.Next {
-			do(dns.Address)
-		}
-		for gw := aa.FirstGatewayAddress; gw != nil; gw = gw.Next {
-			do(gw.Address)
+			ip := dns.Address.IP()
+			if ip == nil || ip.IsLoopback() || seen[ip.String()] {
+				continue
+			}
+			if _, ok := addressMap[ip.String()]; ok {
+				continue
+			}
+			seen[ip.String()] = true
+			ns = append(ns, ip.String())
 		}
 	}
 	return ns

resolver.go

@@ -35,13 +35,26 @@ const bootstrapDNS = "76.76.2.22"
 // or is the Resolver used for ResolverTypeOS.
 var or = &osResolver{nameservers: defaultNameservers()}
 
-// defaultNameservers returns OS nameservers plus ctrld bootstrap nameserver.
+// defaultNameservers returns nameservers used by the OS.
+// If no nameservers can be found, ctrld bootstrap nameserver will be used.
 func defaultNameservers() []string {
 	ns := nameservers()
-	ns = append(ns, net.JoinHostPort(bootstrapDNS, "53"))
+	if len(ns) == 0 {
+		ns = append(ns, net.JoinHostPort(bootstrapDNS, "53"))
+	}
 	return ns
 }
 
+// InitializeOsResolver initializes OS resolver using the current system DNS settings.
+// It returns the nameservers that is going to be used by the OS resolver.
+//
+// It's the caller's responsibility to ensure the system DNS is in a clean state before
+// calling this function.
+func InitializeOsResolver() []string {
+	or.nameservers = defaultNameservers()
+	return or.nameservers
+}
+
 // Resolver is the interface that wraps the basic DNS operations.
 //
 // Resolve resolves the DNS query, return the result and the corresponding error.

scripts/build.sh

@@ -72,17 +72,9 @@ build() {
       if [ "$CGO_ENABLED" = "0" ]; then
         binary=${binary}-nocgo
       fi
+      GOOS=${goos} GOARCH=${goarch} GOARM=${3} "$go" generate ./...
       GOOS=${goos} GOARCH=${goarch} GOARM=${3} "$go" build -ldflags="$ldflags" -o "$binary" ./cmd/ctrld
       compress "$binary"
-
-      if [ -z "${CTRLD_NO_QF}" ]; then
-        binary_qf=${executable_name}-qf-${goos}-${goarch}v${3}
-        if [ "$CGO_ENABLED" = "0" ]; then
-          binary_qf=${binary_qf}-nocgo
-        fi
-        GOOS=${goos} GOARCH=${goarch} GOARM=${3} "$go" build -ldflags="$ldflags" -tags=qf -o "$binary_qf" ./cmd/ctrld
-        compress "$binary_qf"
-      fi
       ;;
     *)
       # GOMIPS is required for linux/mips: https://nileshgr.com/2020/02/16/golang-on-openwrt-mips/
@@ -90,17 +82,9 @@ build() {
       if [ "$CGO_ENABLED" = "0" ]; then
         binary=${binary}-nocgo
       fi
+      GOOS=${goos} GOARCH=${goarch} GOMIPS=softfloat "$go" generate ./...
       GOOS=${goos} GOARCH=${goarch} GOMIPS=softfloat "$go" build -ldflags="$ldflags" -o "$binary" ./cmd/ctrld
       compress "$binary"
-
-      if [ -z "${CTRLD_NO_QF}" ]; then
-        binary_qf=${executable_name}-qf-${goos}-${goarch}
-        if [ "$CGO_ENABLED" = "0" ]; then
-          binary_qf=${binary_qf}-nocgo
-        fi
-        GOOS=${goos} GOARCH=${goarch} GOMIPS=softfloat "$go" build -ldflags="$ldflags" -tags=qf -o "$binary_qf" ./cmd/ctrld
-        compress "$binary_qf"
-      fi
       ;;
   esac
 }

scripts/build_lib.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# This script is used to locally build Android .aar library and iOS .xcframework from ctrld source using go mobile tool.
+
+# Requirements:
+#   - Android NDK (version 23+)
+#   - Android SDK (version 33+)
+#   - Xcode 15 + Build tools
+#   - Go 1.21
+#   - Git
+# usage: $ ./build_lib.sh v1.3.4
+
+TAG="$1"
+# Hacky way to replace version info.
+update_versionInfo() {
+    local file="$1/ctrld/cmd/cli/cli.go"
+    local tag="$2"
+    local commit="$3"
+    awk -v tag="$tag" -v commit="$commit" '
+        BEGIN { version_updated = 0; commit_updated = 0 }
+        /^\tversion/ {
+            sub(/= ".+"/, "= \"" tag "\"");
+            version_updated = 1;
+        }
+        /^\tcommit/ {
+            sub(/= ".+"/, "= \"" commit "\"");
+            commit_updated = 1;
+        }
+        { print }
+        END {
+            if (version_updated == 0) {
+                print "\tversion = \"" tag "\"";
+            }
+            if (commit_updated == 0) {
+                print "\tcommit = \"" commit "\"";
+            }
+        }
+    ' "$file" > "$file.tmp" && mv "$file.tmp" "$file"
+}
+export ANDROID_NDK_HOME=~/Library/Android/sdk/ndk/23.0.7599858
+mkdir bin
+cd bin || exit
+root=$(pwd)
+# Get source from github and switch to tag
+git clone --depth 1 --branch "$TAG" https://github.com/Control-D-Inc/ctrld.git
+# Prepare gomobile tool
+sourcePath=./ctrld/cmd/ctrld_library
+cd $sourcePath || exit
+go mod tidy
+go install golang.org/x/mobile/cmd/gomobile@latest
+go get golang.org/x/mobile/bind
+gomobile init
+# Prepare build info
+buildDir=$root/../build
+mkdir -p "$buildDir"
+COMMIT=$(git rev-parse HEAD)
+update_versionInfo "$root" "$TAG" "$COMMIT"
+ldflags="-s -w"
+# Build
+gomobile bind -target ios/arm64 -ldflags="$ldflags" -o "$buildDir"/ctrld-"$TAG".xcframework || exit
+gomobile bind -ldflags="$ldflags" -o "$buildDir"/ctrld-"$TAG".aar || exit
+# Clean up
+rm -r "$root"
+echo "Successfully built Ctrld library $TAG($COMMIT)."