cmd: allow import/running ctrld as library

cmd/cli/cli_test.go

@@ -0,0 +1,23 @@
+package cli
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_writeConfigFile(t *testing.T) {
+	tmpdir := t.TempDir()
+	// simulate --config CLI flag by setting configPath manually.
+	configPath = filepath.Join(tmpdir, "ctrld.toml")
+	_, err := os.Stat(configPath)
+	assert.True(t, os.IsNotExist(err))
+
+	assert.NoError(t, writeConfigFile())
+
+	_, err = os.Stat(configPath)
+	require.NoError(t, err)
+}

cmd/cli/conn.go

@@ -0,0 +1,51 @@
+package cli
+
+import (
+	"net"
+	"time"
+)
+
+// logConn wraps a net.Conn, override the Write behavior.
+// runCmd uses this wrapper, so as long as startCmd finished,
+// ctrld log won't be flushed with un-necessary write errors.
+type logConn struct {
+	conn net.Conn
+}
+
+func (lc *logConn) Read(b []byte) (n int, err error) {
+	return lc.conn.Read(b)
+}
+
+func (lc *logConn) Close() error {
+	return lc.conn.Close()
+}
+
+func (lc *logConn) LocalAddr() net.Addr {
+	return lc.conn.LocalAddr()
+}
+
+func (lc *logConn) RemoteAddr() net.Addr {
+	return lc.conn.RemoteAddr()
+}
+
+func (lc *logConn) SetDeadline(t time.Time) error {
+	return lc.conn.SetDeadline(t)
+}
+
+func (lc *logConn) SetReadDeadline(t time.Time) error {
+	return lc.conn.SetReadDeadline(t)
+}
+
+func (lc *logConn) SetWriteDeadline(t time.Time) error {
+	return lc.conn.SetWriteDeadline(t)
+}
+
+func (lc *logConn) Write(b []byte) (int, error) {
+	// Write performs writes with underlying net.Conn, ignore any errors happen.
+	// "ctrld run" command use this wrapper to report errors to "ctrld start".
+	// If no error occurred, "ctrld start" may finish before "ctrld run" attempt
+	// to close the connection, so ignore errors conservatively here, prevent
+	// un-necessary error "write to closed connection" flushed to ctrld log.
+	_, _ = lc.conn.Write(b)
+	return len(b), nil
+}

cmd/cli/control_client.go

@@ -0,0 +1,29 @@
+package cli
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"time"
+)
+
+type controlClient struct {
+	c *http.Client
+}
+
+func newControlClient(addr string) *controlClient {
+	return &controlClient{c: &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+				d := net.Dialer{}
+				return d.DialContext(ctx, "unix", addr)
+			},
+		},
+		Timeout: time.Second * 30,
+	}}
+}
+
+func (c *controlClient) post(path string, data io.Reader) (*http.Response, error) {
+	return c.c.Post("http://unix"+path, contentTypeJson, data)
+}

cmd/cli/control_server.go

@@ -0,0 +1,85 @@
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"net"
+	"net/http"
+	"os"
+	"sort"
+	"time"
+)
+
+const (
+	contentTypeJson = "application/json"
+	listClientsPath = "/clients"
+	startedPath     = "/started"
+)
+
+type controlServer struct {
+	server *http.Server
+	mux    *http.ServeMux
+	addr   string
+}
+
+func newControlServer(addr string) (*controlServer, error) {
+	mux := http.NewServeMux()
+	s := &controlServer{
+		server: &http.Server{Handler: mux},
+		mux:    mux,
+	}
+	s.addr = addr
+	return s, nil
+}
+
+func (s *controlServer) start() error {
+	_ = os.Remove(s.addr)
+	unixListener, err := net.Listen("unix", s.addr)
+	if l, ok := unixListener.(*net.UnixListener); ok {
+		l.SetUnlinkOnClose(true)
+	}
+	if err != nil {
+		return err
+	}
+	go s.server.Serve(unixListener)
+	return nil
+}
+
+func (s *controlServer) stop() error {
+	_ = os.Remove(s.addr)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*2)
+	defer cancel()
+	return s.server.Shutdown(ctx)
+}
+
+func (s *controlServer) register(pattern string, handler http.Handler) {
+	s.mux.Handle(pattern, jsonResponse(handler))
+}
+
+func (p *prog) registerControlServerHandler() {
+	p.cs.register(listClientsPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		clients := p.ciTable.ListClients()
+		sort.Slice(clients, func(i, j int) bool {
+			return clients[i].IP.Less(clients[j].IP)
+		})
+		if err := json.NewEncoder(w).Encode(&clients); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}))
+	p.cs.register(startedPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		select {
+		case <-p.onStartedDone:
+			w.WriteHeader(http.StatusOK)
+		case <-time.After(10 * time.Second):
+			w.WriteHeader(http.StatusRequestTimeout)
+		}
+	}))
+}
+
+func jsonResponse(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		next.ServeHTTP(w, r)
+	})
+}

cmd/cli/control_server_test.go

@@ -0,0 +1,54 @@
+package cli
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"os"
+	"testing"
+)
+
+func TestControlServer(t *testing.T) {
+	f, err := os.CreateTemp("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(f.Name())
+	f.Close()
+
+	s, err := newControlServer(f.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	pattern := "/ping"
+	respBody := []byte("pong")
+	s.register(pattern, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write(respBody)
+	}))
+	if err := s.start(); err != nil {
+		t.Fatal(err)
+	}
+
+	c := newControlClient(f.Name())
+	resp, err := c.post(pattern, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		t.Fatalf("unepxected response code: %d", resp.StatusCode)
+	}
+	if ct := resp.Header.Get("content-type"); ct != contentTypeJson {
+		t.Fatalf("unexpected content type: %s", ct)
+	}
+	buf, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(buf, respBody) {
+		t.Errorf("unexpected response body, want: %q, got: %q", string(respBody), string(buf))
+	}
+	if err := s.stop(); err != nil {
+		t.Fatal(err)
+	}
+}

cmd/cli/dns.go

@@ -0,0 +1,4 @@
+package cli
+
+//lint:ignore U1000 use in os_linux.go
+type getDNS func(iface string) []string

cmd/cli/dns_proxy.go

@@ -0,0 +1,569 @@
+package cli
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net"
+	"net/netip"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/miekg/dns"
+	"go4.org/mem"
+	"golang.org/x/sync/errgroup"
+	"tailscale.com/net/interfaces"
+	"tailscale.com/util/lineread"
+
+	"github.com/Control-D-Inc/ctrld"
+	"github.com/Control-D-Inc/ctrld/internal/dnscache"
+	ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
+)
+
+const (
+	staleTTL = 60 * time.Second
+	// EDNS0_OPTION_MAC is dnsmasq EDNS0 code for adding mac option.
+	// https://thekelleys.org.uk/gitweb/?p=dnsmasq.git;a=blob;f=src/dns-protocol.h;h=76ac66a8c28317e9c121a74ab5fd0e20f6237dc8;hb=HEAD#l81
+	// This is also dns.EDNS0LOCALSTART, but define our own constant here for clarification.
+	EDNS0_OPTION_MAC = 0xFDE9
+)
+
+var osUpstreamConfig = &ctrld.UpstreamConfig{
+	Name:    "OS resolver",
+	Type:    ctrld.ResolverTypeOS,
+	Timeout: 2000,
+}
+
+func (p *prog) serveDNS(listenerNum string) error {
+	listenerConfig := p.cfg.Listener[listenerNum]
+	// make sure ip is allocated
+	if allocErr := p.allocateIP(listenerConfig.IP); allocErr != nil {
+		mainLog.Load().Error().Err(allocErr).Str("ip", listenerConfig.IP).Msg("serveUDP: failed to allocate listen ip")
+		return allocErr
+	}
+	var failoverRcodes []int
+	if listenerConfig.Policy != nil {
+		failoverRcodes = listenerConfig.Policy.FailoverRcodeNumbers
+	}
+	handler := dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) {
+		p.sema.acquire()
+		defer p.sema.release()
+		q := m.Question[0]
+		domain := canonicalName(q.Name)
+		reqId := requestID()
+		remoteIP, _, _ := net.SplitHostPort(w.RemoteAddr().String())
+		mac := macFromMsg(m)
+		ci := p.getClientInfo(remoteIP, mac)
+		remoteAddr := spoofRemoteAddr(w.RemoteAddr(), ci)
+		fmtSrcToDest := fmtRemoteToLocal(listenerNum, remoteAddr.String(), w.LocalAddr().String())
+		t := time.Now()
+		ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, reqId)
+		ctrld.Log(ctx, mainLog.Load().Debug(), "%s received query: %s %s", fmtSrcToDest, dns.TypeToString[q.Qtype], domain)
+		upstreams, matched := p.upstreamFor(ctx, listenerNum, listenerConfig, remoteAddr, domain)
+		var answer *dns.Msg
+		if !matched && listenerConfig.Restricted {
+			answer = new(dns.Msg)
+			answer.SetRcode(m, dns.RcodeRefused)
+		} else {
+			answer = p.proxy(ctx, upstreams, failoverRcodes, m, ci)
+			rtt := time.Since(t)
+			ctrld.Log(ctx, mainLog.Load().Debug(), "received response of %d bytes in %s", answer.Len(), rtt)
+		}
+		if err := w.WriteMsg(answer); err != nil {
+			ctrld.Log(ctx, mainLog.Load().Error().Err(err), "serveUDP: failed to send DNS response to client")
+		}
+	})
+
+	g, ctx := errgroup.WithContext(context.Background())
+	for _, proto := range []string{"udp", "tcp"} {
+		proto := proto
+		if needLocalIPv6Listener() {
+			g.Go(func() error {
+				s, errCh := runDNSServer(net.JoinHostPort("::1", strconv.Itoa(listenerConfig.Port)), proto, handler)
+				defer s.Shutdown()
+				select {
+				case <-p.stopCh:
+				case <-ctx.Done():
+				case err := <-errCh:
+					// Local ipv6 listener should not terminate ctrld.
+					// It's a workaround for a quirk on Windows.
+					mainLog.Load().Warn().Err(err).Msg("local ipv6 listener failed")
+				}
+				return nil
+			})
+		}
+		// When we spawn a listener on 127.0.0.1, also spawn listeners on the RFC1918
+		// addresses of the machine. So ctrld could receive queries from LAN clients.
+		if needRFC1918Listeners(listenerConfig) {
+			g.Go(func() error {
+				for _, addr := range rfc1918Addresses() {
+					func() {
+						listenAddr := net.JoinHostPort(addr, strconv.Itoa(listenerConfig.Port))
+						s, errCh := runDNSServer(listenAddr, proto, handler)
+						defer s.Shutdown()
+						select {
+						case <-p.stopCh:
+						case <-ctx.Done():
+						case err := <-errCh:
+							// RFC1918 listener should not terminate ctrld.
+							// It's a workaround for a quirk on system with systemd-resolved.
+							mainLog.Load().Warn().Err(err).Msgf("could not listen on %s: %s", proto, listenAddr)
+						}
+					}()
+				}
+				return nil
+			})
+		}
+		g.Go(func() error {
+			s, errCh := runDNSServer(dnsListenAddress(listenerConfig), proto, handler)
+			defer s.Shutdown()
+			select {
+			case err := <-errCh:
+				return err
+			case <-time.After(5 * time.Second):
+				p.started <- struct{}{}
+			}
+			select {
+			case <-p.stopCh:
+			case <-ctx.Done():
+			case err := <-errCh:
+				return err
+			}
+			return nil
+		})
+	}
+	return g.Wait()
+}
+
+// upstreamFor returns the list of upstreams for resolving the given domain,
+// matching by policies defined in the listener config. The second return value
+// reports whether the domain matches the policy.
+//
+// Though domain policy has higher priority than network policy, it is still
+// processed later, because policy logging want to know whether a network rule
+// is disregarded in favor of the domain level rule.
+func (p *prog) upstreamFor(ctx context.Context, defaultUpstreamNum string, lc *ctrld.ListenerConfig, addr net.Addr, domain string) ([]string, bool) {
+	upstreams := []string{"upstream." + defaultUpstreamNum}
+	matchedPolicy := "no policy"
+	matchedNetwork := "no network"
+	matchedRule := "no rule"
+	matched := false
+
+	defer func() {
+		if !matched && lc.Restricted {
+			ctrld.Log(ctx, mainLog.Load().Info(), "query refused, %s does not match any network policy", addr.String())
+			return
+		}
+		if matched {
+			ctrld.Log(ctx, mainLog.Load().Info(), "%s, %s, %s -> %v", matchedPolicy, matchedNetwork, matchedRule, upstreams)
+		} else {
+			ctrld.Log(ctx, mainLog.Load().Info(), "no explicit policy matched, using default routing -> %v", upstreams)
+		}
+	}()
+
+	if lc.Policy == nil {
+		return upstreams, false
+	}
+
+	do := func(policyUpstreams []string) {
+		upstreams = append([]string(nil), policyUpstreams...)
+	}
+
+	var networkTargets []string
+	var sourceIP net.IP
+	switch addr := addr.(type) {
+	case *net.UDPAddr:
+		sourceIP = addr.IP
+	case *net.TCPAddr:
+		sourceIP = addr.IP
+	}
+
+networkRules:
+	for _, rule := range lc.Policy.Networks {
+		for source, targets := range rule {
+			networkNum := strings.TrimPrefix(source, "network.")
+			nc := p.cfg.Network[networkNum]
+			if nc == nil {
+				continue
+			}
+			for _, ipNet := range nc.IPNets {
+				if ipNet.Contains(sourceIP) {
+					matchedPolicy = lc.Policy.Name
+					matchedNetwork = source
+					networkTargets = targets
+					matched = true
+					break networkRules
+				}
+			}
+		}
+	}
+
+	for _, rule := range lc.Policy.Rules {
+		// There's only one entry per rule, config validation ensures this.
+		for source, targets := range rule {
+			if source == domain || wildcardMatches(source, domain) {
+				matchedPolicy = lc.Policy.Name
+				if len(networkTargets) > 0 {
+					matchedNetwork += " (unenforced)"
+				}
+				matchedRule = source
+				do(targets)
+				matched = true
+				return upstreams, matched
+			}
+		}
+	}
+
+	if matched {
+		do(networkTargets)
+	}
+
+	return upstreams, matched
+}
+
+func (p *prog) proxy(ctx context.Context, upstreams []string, failoverRcodes []int, msg *dns.Msg, ci *ctrld.ClientInfo) *dns.Msg {
+	var staleAnswer *dns.Msg
+	serveStaleCache := p.cache != nil && p.cfg.Service.CacheServeStale
+	upstreamConfigs := p.upstreamConfigsFromUpstreamNumbers(upstreams)
+	if len(upstreamConfigs) == 0 {
+		upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig}
+		upstreams = []string{"upstream.os"}
+	}
+	// Inverse query should not be cached: https://www.rfc-editor.org/rfc/rfc1035#section-7.4
+	if p.cache != nil && msg.Question[0].Qtype != dns.TypePTR {
+		for _, upstream := range upstreams {
+			cachedValue := p.cache.Get(dnscache.NewKey(msg, upstream))
+			if cachedValue == nil {
+				continue
+			}
+			answer := cachedValue.Msg.Copy()
+			answer.SetRcode(msg, answer.Rcode)
+			now := time.Now()
+			if cachedValue.Expire.After(now) {
+				ctrld.Log(ctx, mainLog.Load().Debug(), "hit cached response")
+				setCachedAnswerTTL(answer, now, cachedValue.Expire)
+				return answer
+			}
+			staleAnswer = answer
+		}
+	}
+	resolve1 := func(n int, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) (*dns.Msg, error) {
+		ctrld.Log(ctx, mainLog.Load().Debug(), "sending query to %s: %s", upstreams[n], upstreamConfig.Name)
+		dnsResolver, err := ctrld.NewResolver(upstreamConfig)
+		if err != nil {
+			ctrld.Log(ctx, mainLog.Load().Error().Err(err), "failed to create resolver")
+			return nil, err
+		}
+		resolveCtx, cancel := context.WithCancel(ctx)
+		defer cancel()
+		if upstreamConfig.Timeout > 0 {
+			timeoutCtx, cancel := context.WithTimeout(resolveCtx, time.Millisecond*time.Duration(upstreamConfig.Timeout))
+			defer cancel()
+			resolveCtx = timeoutCtx
+		}
+		return dnsResolver.Resolve(resolveCtx, msg)
+	}
+	resolve := func(n int, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) *dns.Msg {
+		if upstreamConfig.UpstreamSendClientInfo() && ci != nil {
+			ctrld.Log(ctx, mainLog.Load().Debug(), "including client info with the request")
+			ctx = context.WithValue(ctx, ctrld.ClientInfoCtxKey{}, ci)
+		}
+		answer, err := resolve1(n, upstreamConfig, msg)
+		if err != nil {
+			ctrld.Log(ctx, mainLog.Load().Error().Err(err), "failed to resolve query")
+			return nil
+		}
+		return answer
+	}
+	for n, upstreamConfig := range upstreamConfigs {
+		if upstreamConfig == nil {
+			continue
+		}
+		answer := resolve(n, upstreamConfig, msg)
+		if answer == nil {
+			if serveStaleCache && staleAnswer != nil {
+				ctrld.Log(ctx, mainLog.Load().Debug(), "serving stale cached response")
+				now := time.Now()
+				setCachedAnswerTTL(staleAnswer, now, now.Add(staleTTL))
+				return staleAnswer
+			}
+			continue
+		}
+		if answer.Rcode != dns.RcodeSuccess && len(upstreamConfigs) > 1 && containRcode(failoverRcodes, answer.Rcode) {
+			ctrld.Log(ctx, mainLog.Load().Debug(), "failover rcode matched, process to next upstream")
+			continue
+		}
+
+		// set compression, as it is not set by default when unpacking
+		answer.Compress = true
+
+		if p.cache != nil {
+			ttl := ttlFromMsg(answer)
+			now := time.Now()
+			expired := now.Add(time.Duration(ttl) * time.Second)
+			if cachedTTL := p.cfg.Service.CacheTTLOverride; cachedTTL > 0 {
+				expired = now.Add(time.Duration(cachedTTL) * time.Second)
+			}
+			setCachedAnswerTTL(answer, now, expired)
+			p.cache.Add(dnscache.NewKey(msg, upstreams[n]), dnscache.NewValue(answer, expired))
+			ctrld.Log(ctx, mainLog.Load().Debug(), "add cached response")
+		}
+		return answer
+	}
+	ctrld.Log(ctx, mainLog.Load().Error(), "all upstreams failed")
+	answer := new(dns.Msg)
+	answer.SetRcode(msg, dns.RcodeServerFailure)
+	return answer
+}
+
+func (p *prog) upstreamConfigsFromUpstreamNumbers(upstreams []string) []*ctrld.UpstreamConfig {
+	upstreamConfigs := make([]*ctrld.UpstreamConfig, 0, len(upstreams))
+	for _, upstream := range upstreams {
+		upstreamNum := strings.TrimPrefix(upstream, "upstream.")
+		upstreamConfigs = append(upstreamConfigs, p.cfg.Upstream[upstreamNum])
+	}
+	return upstreamConfigs
+}
+
+// canonicalName returns canonical name from FQDN with "." trimmed.
+func canonicalName(fqdn string) string {
+	q := strings.TrimSpace(fqdn)
+	q = strings.TrimSuffix(q, ".")
+	// https://datatracker.ietf.org/doc/html/rfc4343
+	q = strings.ToLower(q)
+
+	return q
+}
+
+func wildcardMatches(wildcard, domain string) bool {
+	// Wildcard match.
+	wildCardParts := strings.Split(wildcard, "*")
+	if len(wildCardParts) != 2 {
+		return false
+	}
+
+	switch {
+	case len(wildCardParts[0]) > 0 && len(wildCardParts[1]) > 0:
+		// Domain must match both prefix and suffix.
+		return strings.HasPrefix(domain, wildCardParts[0]) && strings.HasSuffix(domain, wildCardParts[1])
+
+	case len(wildCardParts[1]) > 0:
+		// Only suffix must match.
+		return strings.HasSuffix(domain, wildCardParts[1])
+
+	case len(wildCardParts[0]) > 0:
+		// Only prefix must match.
+		return strings.HasPrefix(domain, wildCardParts[0])
+	}
+
+	return false
+}
+
+func fmtRemoteToLocal(listenerNum, remote, local string) string {
+	return fmt.Sprintf("%s -> listener.%s: %s:", remote, listenerNum, local)
+}
+
+func requestID() string {
+	b := make([]byte, 3) // 6 chars
+	if _, err := rand.Read(b); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(b)
+}
+
+func containRcode(rcodes []int, rcode int) bool {
+	for i := range rcodes {
+		if rcodes[i] == rcode {
+			return true
+		}
+	}
+	return false
+}
+
+func setCachedAnswerTTL(answer *dns.Msg, now, expiredTime time.Time) {
+	ttlSecs := expiredTime.Sub(now).Seconds()
+	if ttlSecs < 0 {
+		return
+	}
+
+	ttl := uint32(ttlSecs)
+	for _, rr := range answer.Answer {
+		rr.Header().Ttl = ttl
+	}
+	for _, rr := range answer.Ns {
+		rr.Header().Ttl = ttl
+	}
+	for _, rr := range answer.Extra {
+		if rr.Header().Rrtype != dns.TypeOPT {
+			rr.Header().Ttl = ttl
+		}
+	}
+}
+
+func ttlFromMsg(msg *dns.Msg) uint32 {
+	for _, rr := range msg.Answer {
+		return rr.Header().Ttl
+	}
+	for _, rr := range msg.Ns {
+		return rr.Header().Ttl
+	}
+	return 0
+}
+
+func needLocalIPv6Listener() bool {
+	// On Windows, there's no easy way for disabling/removing IPv6 DNS resolver, so we check whether we can
+	// listen on ::1, then spawn a listener for receiving DNS requests.
+	return ctrldnet.SupportsIPv6ListenLocal() && runtime.GOOS == "windows"
+}
+
+func dnsListenAddress(lc *ctrld.ListenerConfig) string {
+	// If we are inside container and the listener loopback address, change
+	// the address to something like 0.0.0.0:53, so user can expose the port to outside.
+	if inContainer() {
+		if ip := net.ParseIP(lc.IP); ip != nil && ip.IsLoopback() {
+			return net.JoinHostPort("0.0.0.0", strconv.Itoa(lc.Port))
+		}
+	}
+	return net.JoinHostPort(lc.IP, strconv.Itoa(lc.Port))
+}
+
+func macFromMsg(msg *dns.Msg) string {
+	if opt := msg.IsEdns0(); opt != nil {
+		for _, s := range opt.Option {
+			switch e := s.(type) {
+			case *dns.EDNS0_LOCAL:
+				if e.Code == EDNS0_OPTION_MAC {
+					return net.HardwareAddr(e.Data).String()
+				}
+			}
+		}
+	}
+	return ""
+}
+
+func spoofRemoteAddr(addr net.Addr, ci *ctrld.ClientInfo) net.Addr {
+	if ci != nil && ci.IP != "" {
+		switch addr := addr.(type) {
+		case *net.UDPAddr:
+			udpAddr := &net.UDPAddr{
+				IP:   net.ParseIP(ci.IP),
+				Port: addr.Port,
+				Zone: addr.Zone,
+			}
+			return udpAddr
+		case *net.TCPAddr:
+			udpAddr := &net.TCPAddr{
+				IP:   net.ParseIP(ci.IP),
+				Port: addr.Port,
+				Zone: addr.Zone,
+			}
+			return udpAddr
+		}
+	}
+	return addr
+}
+
+// runDNSServer starts a DNS server for given address and network,
+// with the given handler. It ensures the server has started listening.
+// Any error will be reported to the caller via returned channel.
+//
+// It's the caller responsibility to call Shutdown to close the server.
+func runDNSServer(addr, network string, handler dns.Handler) (*dns.Server, <-chan error) {
+	s := &dns.Server{
+		Addr:    addr,
+		Net:     network,
+		Handler: handler,
+	}
+
+	waitLock := sync.Mutex{}
+	waitLock.Lock()
+	s.NotifyStartedFunc = waitLock.Unlock
+
+	errCh := make(chan error)
+	go func() {
+		defer close(errCh)
+		if err := s.ListenAndServe(); err != nil {
+			waitLock.Unlock()
+			mainLog.Load().Error().Err(err).Msgf("could not listen and serve on: %s", s.Addr)
+			errCh <- err
+		}
+	}()
+	waitLock.Lock()
+	return s, errCh
+}
+
+// inContainer reports whether we're running in a container.
+//
+// Copied from https://github.com/tailscale/tailscale/blob/v1.42.0/hostinfo/hostinfo.go#L260
+// with modification for ctrld usage.
+func inContainer() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+
+	var ret bool
+	if _, err := os.Stat("/.dockerenv"); err == nil {
+		return true
+	}
+	if _, err := os.Stat("/run/.containerenv"); err == nil {
+		// See https://github.com/cri-o/cri-o/issues/5461
+		return true
+	}
+	lineread.File("/proc/1/cgroup", func(line []byte) error {
+		if mem.Contains(mem.B(line), mem.S("/docker/")) ||
+			mem.Contains(mem.B(line), mem.S("/lxc/")) {
+			ret = true
+			return io.EOF // arbitrary non-nil error to stop loop
+		}
+		return nil
+	})
+	lineread.File("/proc/mounts", func(line []byte) error {
+		if mem.Contains(mem.B(line), mem.S("lxcfs /proc/cpuinfo fuse.lxcfs")) {
+			ret = true
+			return io.EOF
+		}
+		return nil
+	})
+	return ret
+}
+
+func (p *prog) getClientInfo(ip, mac string) *ctrld.ClientInfo {
+	ci := &ctrld.ClientInfo{}
+	if mac != "" {
+		ci.Mac = mac
+		ci.IP = p.ciTable.LookupIP(mac)
+	} else {
+		ci.IP = ip
+		ci.Mac = p.ciTable.LookupMac(ip)
+		if ip == "127.0.0.1" || ip == "::1" {
+			ci.IP = p.ciTable.LookupIP(ci.Mac)
+		}
+	}
+	ci.Hostname = p.ciTable.LookupHostname(ci.IP, ci.Mac)
+	return ci
+}
+
+func needRFC1918Listeners(lc *ctrld.ListenerConfig) bool {
+	return lc.IP == "127.0.0.1" && lc.Port == 53
+}
+
+func rfc1918Addresses() []string {
+	var res []string
+	interfaces.ForeachInterface(func(i interfaces.Interface, prefixes []netip.Prefix) {
+		addrs, _ := i.Addrs()
+		for _, addr := range addrs {
+			ipNet, ok := addr.(*net.IPNet)
+			if !ok || !ipNet.IP.IsPrivate() {
+				continue
+			}
+			res = append(res, ipNet.IP.String())
+		}
+	})
+	return res
+}

cmd/cli/dns_proxy_test.go

@@ -0,0 +1,218 @@
+package cli
+
+import (
+	"context"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/Control-D-Inc/ctrld"
+	"github.com/Control-D-Inc/ctrld/internal/dnscache"
+	"github.com/Control-D-Inc/ctrld/testhelper"
+)
+
+func Test_wildcardMatches(t *testing.T) {
+	tests := []struct {
+		name     string
+		wildcard string
+		domain   string
+		match    bool
+	}{
+		{"prefix parent should not match", "*.windscribe.com", "windscribe.com", false},
+		{"prefix", "*.windscribe.com", "anything.windscribe.com", true},
+		{"prefix not match other domain", "*.windscribe.com", "example.com", false},
+		{"prefix not match domain in name", "*.windscribe.com", "wwindscribe.com", false},
+		{"suffix", "suffix.*", "suffix.windscribe.com", true},
+		{"suffix not match other", "suffix.*", "suffix1.windscribe.com", false},
+		{"both", "suffix.*.windscribe.com", "suffix.anything.windscribe.com", true},
+		{"both not match", "suffix.*.windscribe.com", "suffix1.suffix.windscribe.com", false},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := wildcardMatches(tc.wildcard, tc.domain); got != tc.match {
+				t.Errorf("unexpected result, wildcard: %s, domain: %s, want: %v, got: %v", tc.wildcard, tc.domain, tc.match, got)
+			}
+		})
+	}
+}
+
+func Test_canonicalName(t *testing.T) {
+	tests := []struct {
+		name      string
+		domain    string
+		canonical string
+	}{
+		{"fqdn to canonical", "windscribe.com.", "windscribe.com"},
+		{"already canonical", "windscribe.com", "windscribe.com"},
+		{"case insensitive", "Windscribe.Com.", "windscribe.com"},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			if got := canonicalName(tc.domain); got != tc.canonical {
+				t.Errorf("unexpected result, want: %s, got: %s", tc.canonical, got)
+			}
+		})
+	}
+}
+
+func Test_prog_upstreamFor(t *testing.T) {
+	cfg := testhelper.SampleConfig(t)
+	prog := &prog{cfg: cfg}
+	for _, nc := range prog.cfg.Network {
+		for _, cidr := range nc.Cidrs {
+			_, ipNet, err := net.ParseCIDR(cidr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			nc.IPNets = append(nc.IPNets, ipNet)
+		}
+	}
+
+	tests := []struct {
+		name               string
+		ip                 string
+		defaultUpstreamNum string
+		lc                 *ctrld.ListenerConfig
+		domain             string
+		upstreams          []string
+		matched            bool
+		testLogMsg         string
+	}{
+		{"Policy map matches", "192.168.0.1:0", "0", prog.cfg.Listener["0"], "abc.xyz", []string{"upstream.1", "upstream.0"}, true, ""},
+		{"Policy split matches", "192.168.0.1:0", "0", prog.cfg.Listener["0"], "abc.ru", []string{"upstream.1"}, true, ""},
+		{"Policy map for other network matches", "192.168.1.2:0", "0", prog.cfg.Listener["0"], "abc.xyz", []string{"upstream.0"}, true, ""},
+		{"No policy map for listener", "192.168.1.2:0", "1", prog.cfg.Listener["1"], "abc.ru", []string{"upstream.1"}, false, ""},
+		{"unenforced loging", "192.168.1.2:0", "0", prog.cfg.Listener["0"], "abc.ru", []string{"upstream.1"}, true, "My Policy, network.1 (unenforced), *.ru -> [upstream.1]"},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			for _, network := range []string{"udp", "tcp"} {
+				var (
+					addr net.Addr
+					err  error
+				)
+				switch network {
+				case "udp":
+					addr, err = net.ResolveUDPAddr(network, tc.ip)
+				case "tcp":
+					addr, err = net.ResolveTCPAddr(network, tc.ip)
+				}
+				require.NoError(t, err)
+				require.NotNil(t, addr)
+				ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+				upstreams, matched := prog.upstreamFor(ctx, tc.defaultUpstreamNum, tc.lc, addr, tc.domain)
+				assert.Equal(t, tc.matched, matched)
+				assert.Equal(t, tc.upstreams, upstreams)
+				if tc.testLogMsg != "" {
+					assert.Contains(t, logOutput.String(), tc.testLogMsg)
+				}
+			}
+		})
+	}
+}
+
+func TestCache(t *testing.T) {
+	cfg := testhelper.SampleConfig(t)
+	prog := &prog{cfg: cfg}
+	for _, nc := range prog.cfg.Network {
+		for _, cidr := range nc.Cidrs {
+			_, ipNet, err := net.ParseCIDR(cidr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			nc.IPNets = append(nc.IPNets, ipNet)
+		}
+	}
+	cacher, err := dnscache.NewLRUCache(4096)
+	require.NoError(t, err)
+	prog.cache = cacher
+
+	msg := new(dns.Msg)
+	msg.SetQuestion("example.com", dns.TypeA)
+	msg.MsgHdr.RecursionDesired = true
+	answer1 := new(dns.Msg)
+	answer1.SetRcode(msg, dns.RcodeSuccess)
+
+	prog.cache.Add(dnscache.NewKey(msg, "upstream.1"), dnscache.NewValue(answer1, time.Now().Add(time.Minute)))
+	answer2 := new(dns.Msg)
+	answer2.SetRcode(msg, dns.RcodeRefused)
+	prog.cache.Add(dnscache.NewKey(msg, "upstream.0"), dnscache.NewValue(answer2, time.Now().Add(time.Minute)))
+
+	got1 := prog.proxy(context.Background(), []string{"upstream.1"}, nil, msg, nil)
+	got2 := prog.proxy(context.Background(), []string{"upstream.0"}, nil, msg, nil)
+	assert.NotSame(t, got1, got2)
+	assert.Equal(t, answer1.Rcode, got1.Rcode)
+	assert.Equal(t, answer2.Rcode, got2.Rcode)
+}
+
+func Test_macFromMsg(t *testing.T) {
+	tests := []struct {
+		name    string
+		mac     string
+		wantMac bool
+	}{
+		{"has mac", "4c:20:b8:ab:87:1b", true},
+		{"no mac", "4c:20:b8:ab:87:1b", false},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			hw, err := net.ParseMAC(tc.mac)
+			if err != nil {
+				t.Fatal(err)
+			}
+			m := new(dns.Msg)
+			m.SetQuestion("example.com.", dns.TypeA)
+			o := &dns.OPT{Hdr: dns.RR_Header{Name: ".", Rrtype: dns.TypeOPT}}
+			if tc.wantMac {
+				ec1 := &dns.EDNS0_LOCAL{Code: EDNS0_OPTION_MAC, Data: hw}
+				o.Option = append(o.Option, ec1)
+			}
+			m.Extra = append(m.Extra, o)
+			got := macFromMsg(m)
+			if tc.wantMac && got != tc.mac {
+				t.Errorf("mismatch, want: %q, got: %q", tc.mac, got)
+			}
+			if !tc.wantMac && got != "" {
+				t.Errorf("unexpected mac: %q", got)
+			}
+		})
+	}
+}
+
+func Test_remoteAddrFromMsg(t *testing.T) {
+	loopbackIP := net.ParseIP("127.0.0.1")
+	tests := []struct {
+		name string
+		addr net.Addr
+		ci   *ctrld.ClientInfo
+		want string
+	}{
+		{"tcp", &net.TCPAddr{IP: loopbackIP, Port: 12345}, &ctrld.ClientInfo{IP: "192.168.1.10"}, "192.168.1.10:12345"},
+		{"udp", &net.UDPAddr{IP: loopbackIP, Port: 12345}, &ctrld.ClientInfo{IP: "192.168.1.11"}, "192.168.1.11:12345"},
+		{"nil client info", &net.UDPAddr{IP: loopbackIP, Port: 12345}, nil, "127.0.0.1:12345"},
+		{"empty ip", &net.UDPAddr{IP: loopbackIP, Port: 12345}, &ctrld.ClientInfo{}, "127.0.0.1:12345"},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			addr := spoofRemoteAddr(tc.addr, tc.ci)
+			if addr.String() != tc.want {
+				t.Errorf("unexpected result, want: %q, got: %q", tc.want, addr.String())
+			}
+		})
+	}
+}

cmd/cli/main.go

@@ -0,0 +1,161 @@
+package cli
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"sync/atomic"
+	"time"
+
+	"github.com/kardianos/service"
+	"github.com/rs/zerolog"
+
+	"github.com/Control-D-Inc/ctrld"
+)
+
+var (
+	configPath        string
+	configBase64      string
+	daemon            bool
+	listenAddress     string
+	primaryUpstream   string
+	secondaryUpstream string
+	domains           []string
+	logPath           string
+	homedir           string
+	cacheSize         int
+	cfg               ctrld.Config
+	verbose           int
+	silent            bool
+	cdUID             string
+	cdOrg             string
+	cdDev             bool
+	iface             string
+	ifaceStartStop    string
+
+	mainLog       atomic.Pointer[zerolog.Logger]
+	consoleWriter zerolog.ConsoleWriter
+)
+
+func init() {
+	l := zerolog.New(io.Discard)
+	mainLog.Store(&l)
+}
+
+func Main() {
+	ctrld.InitConfig(v, "ctrld")
+	initCLI()
+	if err := rootCmd.Execute(); err != nil {
+		mainLog.Load().Error().Msg(err.Error())
+		os.Exit(1)
+	}
+}
+
+func normalizeLogFilePath(logFilePath string) string {
+	if logFilePath == "" || filepath.IsAbs(logFilePath) || service.Interactive() {
+		return logFilePath
+	}
+	if homedir != "" {
+		return filepath.Join(homedir, logFilePath)
+	}
+	dir, _ := userHomeDir()
+	if dir == "" {
+		return logFilePath
+	}
+	return filepath.Join(dir, logFilePath)
+}
+
+func initConsoleLogging() {
+	consoleWriter = zerolog.NewConsoleWriter(func(w *zerolog.ConsoleWriter) {
+		w.TimeFormat = time.StampMilli
+	})
+	multi := zerolog.MultiLevelWriter(consoleWriter)
+	l := mainLog.Load().Output(multi).With().Timestamp().Logger()
+	mainLog.Store(&l)
+	switch {
+	case silent:
+		zerolog.SetGlobalLevel(zerolog.NoLevel)
+	case verbose == 1:
+		zerolog.SetGlobalLevel(zerolog.InfoLevel)
+	case verbose > 1:
+		zerolog.SetGlobalLevel(zerolog.DebugLevel)
+	default:
+		zerolog.SetGlobalLevel(zerolog.NoticeLevel)
+	}
+}
+
+// initLogging initializes global logging setup.
+func initLogging() {
+	initLoggingWithBackup(true)
+}
+
+// initLoggingWithBackup initializes log setup base on current config.
+// If doBackup is true, backup old log file with ".1" suffix.
+//
+// This is only used in runCmd for special handling in case of logging config
+// change in cd mode. Without special reason, the caller should use initLogging
+// wrapper instead of calling this function directly.
+func initLoggingWithBackup(doBackup bool) {
+	writers := []io.Writer{io.Discard}
+	if logFilePath := normalizeLogFilePath(cfg.Service.LogPath); logFilePath != "" {
+		// Create parent directory if necessary.
+		if err := os.MkdirAll(filepath.Dir(logFilePath), 0750); err != nil {
+			mainLog.Load().Error().Msgf("failed to create log path: %v", err)
+			os.Exit(1)
+		}
+
+		// Default open log file in append mode.
+		flags := os.O_CREATE | os.O_RDWR | os.O_APPEND
+		if doBackup {
+			// Backup old log file with .1 suffix.
+			if err := os.Rename(logFilePath, logFilePath+".1"); err != nil && !os.IsNotExist(err) {
+				mainLog.Load().Error().Msgf("could not backup old log file: %v", err)
+			} else {
+				// Backup was created, set flags for truncating old log file.
+				flags = os.O_CREATE | os.O_RDWR
+			}
+		}
+		logFile, err := os.OpenFile(logFilePath, flags, os.FileMode(0o600))
+		if err != nil {
+			mainLog.Load().Error().Msgf("failed to create log file: %v", err)
+			os.Exit(1)
+		}
+		writers = append(writers, logFile)
+	}
+	writers = append(writers, consoleWriter)
+	multi := zerolog.MultiLevelWriter(writers...)
+	l := mainLog.Load().Output(multi).With().Timestamp().Logger()
+	mainLog.Store(&l)
+	// TODO: find a better way.
+	ctrld.ProxyLogger.Store(&l)
+
+	zerolog.SetGlobalLevel(zerolog.NoticeLevel)
+	logLevel := cfg.Service.LogLevel
+	switch {
+	case silent:
+		zerolog.SetGlobalLevel(zerolog.NoLevel)
+		return
+	case verbose == 1:
+		logLevel = "info"
+	case verbose > 1:
+		logLevel = "debug"
+	}
+	if logLevel == "" {
+		return
+	}
+	level, err := zerolog.ParseLevel(logLevel)
+	if err != nil {
+		mainLog.Load().Warn().Err(err).Msg("could not set log level")
+		return
+	}
+	zerolog.SetGlobalLevel(level)
+}
+
+func initCache() {
+	if !cfg.Service.CacheEnable {
+		return
+	}
+	if cfg.Service.CacheSize == 0 {
+		cfg.Service.CacheSize = 4096
+	}
+}

cmd/cli/main_test.go

@@ -0,0 +1,17 @@
+package cli
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/rs/zerolog"
+)
+
+var logOutput strings.Builder
+
+func TestMain(m *testing.M) {
+	l := zerolog.New(&logOutput)
+	mainLog.Store(&l)
+	os.Exit(m.Run())
+}

cmd/cli/net_darwin.go

@@ -0,0 +1,44 @@
+package cli
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"net"
+	"os/exec"
+	"strings"
+)
+
+func patchNetIfaceName(iface *net.Interface) error {
+	b, err := exec.Command("networksetup", "-listnetworkserviceorder").Output()
+	if err != nil {
+		return err
+	}
+
+	if name := networkServiceName(iface.Name, bytes.NewReader(b)); name != "" {
+		iface.Name = name
+		mainLog.Load().Debug().Str("network_service", name).Msg("found network service name for interface")
+	}
+	return nil
+}
+
+func networkServiceName(ifaceName string, r io.Reader) string {
+	scanner := bufio.NewScanner(r)
+	prevLine := ""
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.Contains(line, "*") {
+			// Network services is disabled.
+			continue
+		}
+		if !strings.Contains(line, "Device: "+ifaceName) {
+			prevLine = line
+			continue
+		}
+		parts := strings.SplitN(prevLine, " ", 2)
+		if len(parts) == 2 {
+			return strings.TrimSpace(parts[1])
+		}
+	}
+	return ""
+}

cmd/cli/net_darwin_test.go

@@ -0,0 +1,59 @@
+package cli
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const listnetworkserviceorderOutput = `
+(1) USB 10/100/1000 LAN 2
+(Hardware Port: USB 10/100/1000 LAN, Device: en7)
+
+(2) Ethernet
+(Hardware Port: Ethernet, Device: en0)
+
+(3) Wi-Fi
+(Hardware Port: Wi-Fi, Device: en1)
+
+(4) Bluetooth PAN
+(Hardware Port: Bluetooth PAN, Device: en4)
+
+(5) Thunderbolt Bridge
+(Hardware Port: Thunderbolt Bridge, Device: bridge0)
+
+(6) kernal
+(Hardware Port: com.wireguard.macos, Device: )
+
+(7) WS BT
+(Hardware Port: com.wireguard.macos, Device: )
+
+(8) ca-001-stg
+(Hardware Port: com.wireguard.macos, Device: )
+
+(9) ca-001-stg-2
+(Hardware Port: com.wireguard.macos, Device: )
+
+`
+
+func Test_networkServiceName(t *testing.T) {
+	tests := []struct {
+		ifaceName          string
+		networkServiceName string
+	}{
+		{"en7", "USB 10/100/1000 LAN 2"},
+		{"en0", "Ethernet"},
+		{"en1", "Wi-Fi"},
+		{"en4", "Bluetooth PAN"},
+		{"bridge0", "Thunderbolt Bridge"},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.ifaceName, func(t *testing.T) {
+			t.Parallel()
+			name := networkServiceName(tc.ifaceName, strings.NewReader(listnetworkserviceorderOutput))
+			assert.Equal(t, tc.networkServiceName, name)
+		})
+	}
+}

cmd/cli/net_others.go

@@ -0,0 +1,7 @@
+//go:build !darwin
+
+package cli
+
+import "net"
+
+func patchNetIfaceName(iface *net.Interface) error { return nil }

cmd/cli/netlink_linux.go

@@ -0,0 +1,27 @@
+package cli
+
+import (
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+)
+
+func (p *prog) watchLinkState() {
+	ch := make(chan netlink.LinkUpdate)
+	done := make(chan struct{})
+	defer close(done)
+	if err := netlink.LinkSubscribe(ch, done); err != nil {
+		mainLog.Load().Warn().Err(err).Msg("could not subscribe link")
+		return
+	}
+	for lu := range ch {
+		if lu.Change == 0xFFFFFFFF {
+			continue
+		}
+		if lu.Change&unix.IFF_UP != 0 {
+			mainLog.Load().Debug().Msgf("link state changed, re-bootstrapping")
+			for _, uc := range p.cfg.Upstream {
+				uc.ReBootstrap()
+			}
+		}
+	}
+}

cmd/cli/netlink_others.go

@@ -0,0 +1,5 @@
+//go:build !linux
+
+package cli
+
+func (p *prog) watchLinkState() {}

cmd/cli/network_manager_linux.go

@@ -0,0 +1,76 @@
+package cli
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/coreos/go-systemd/v22/dbus"
+)
+
+const (
+	nmConfDir           = "/etc/NetworkManager/conf.d"
+	nmCtrldConfFilename = "99-ctrld.conf"
+	nmCtrldConfContent  = `[main]
+dns=none
+systemd-resolved=false
+`
+	nmSystemdUnitName   = "NetworkManager.service"
+	systemdEnabledState = "enabled"
+)
+
+var networkManagerCtrldConfFile = filepath.Join(nmConfDir, nmCtrldConfFilename)
+
+func setupNetworkManager() error {
+	if content, _ := os.ReadFile(nmCtrldConfContent); string(content) == nmCtrldConfContent {
+		mainLog.Load().Debug().Msg("NetworkManager already setup, nothing to do")
+		return nil
+	}
+	err := os.WriteFile(networkManagerCtrldConfFile, []byte(nmCtrldConfContent), os.FileMode(0644))
+	if os.IsNotExist(err) {
+		mainLog.Load().Debug().Msg("NetworkManager is not available")
+		return nil
+	}
+	if err != nil {
+		mainLog.Load().Debug().Err(err).Msg("could not write NetworkManager ctrld config file")
+		return err
+	}
+
+	reloadNetworkManager()
+	mainLog.Load().Debug().Msg("setup NetworkManager done")
+	return nil
+}
+
+func restoreNetworkManager() error {
+	err := os.Remove(networkManagerCtrldConfFile)
+	if os.IsNotExist(err) {
+		mainLog.Load().Debug().Msg("NetworkManager is not available")
+		return nil
+	}
+	if err != nil {
+		mainLog.Load().Debug().Err(err).Msg("could not remove NetworkManager ctrld config file")
+		return err
+	}
+
+	reloadNetworkManager()
+	mainLog.Load().Debug().Msg("restore NetworkManager done")
+	return nil
+}
+
+func reloadNetworkManager() {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+	conn, err := dbus.NewSystemConnectionContext(ctx)
+	if err != nil {
+		mainLog.Load().Error().Err(err).Msg("could not create new system connection")
+		return
+	}
+	defer conn.Close()
+
+	waitCh := make(chan string)
+	if _, err := conn.ReloadUnitContext(ctx, nmSystemdUnitName, "ignore-dependencies", waitCh); err != nil {
+		mainLog.Load().Debug().Err(err).Msg("could not reload NetworkManager")
+	}
+	<-waitCh
+}

cmd/cli/network_manager_others.go

@@ -0,0 +1,15 @@
+//go:build !linux
+
+package cli
+
+func setupNetworkManager() error {
+	reloadNetworkManager()
+	return nil
+}
+
+func restoreNetworkManager() error {
+	reloadNetworkManager()
+	return nil
+}
+
+func reloadNetworkManager() {}

cmd/cli/os_darwin.go

@@ -0,0 +1,59 @@
+package cli
+
+import (
+	"net"
+	"os/exec"
+
+	"github.com/Control-D-Inc/ctrld/internal/resolvconffile"
+)
+
+// allocate loopback ip
+// sudo ifconfig lo0 alias 127.0.0.2 up
+func allocateIP(ip string) error {
+	cmd := exec.Command("ifconfig", "lo0", "alias", ip, "up")
+	if err := cmd.Run(); err != nil {
+		mainLog.Load().Error().Err(err).Msg("allocateIP failed")
+		return err
+	}
+	return nil
+}
+
+func deAllocateIP(ip string) error {
+	cmd := exec.Command("ifconfig", "lo0", "-alias", ip)
+	if err := cmd.Run(); err != nil {
+		mainLog.Load().Error().Err(err).Msg("deAllocateIP failed")
+		return err
+	}
+	return nil
+}
+
+// set the dns server for the provided network interface
+// networksetup -setdnsservers Wi-Fi 8.8.8.8 1.1.1.1
+// TODO(cuonglm): use system API
+func setDNS(iface *net.Interface, nameservers []string) error {
+	cmd := "networksetup"
+	args := []string{"-setdnsservers", iface.Name}
+	args = append(args, nameservers...)
+
+	if err := exec.Command(cmd, args...).Run(); err != nil {
+		mainLog.Load().Error().Err(err).Msgf("setDNS failed, ips = %q", nameservers)
+		return err
+	}
+	return nil
+}
+
+// TODO(cuonglm): use system API
+func resetDNS(iface *net.Interface) error {
+	cmd := "networksetup"
+	args := []string{"-setdnsservers", iface.Name, "empty"}
+
+	if err := exec.Command(cmd, args...).Run(); err != nil {
+		mainLog.Load().Error().Err(err).Msgf("resetDNS failed")
+		return err
+	}
+	return nil
+}
+
+func currentDNS(_ *net.Interface) []string {
+	return resolvconffile.NameServers("")
+}

cmd/cli/os_freebsd.go

@@ -0,0 +1,68 @@
+package cli
+
+import (
+	"net"
+	"net/netip"
+	"os/exec"
+
+	"github.com/Control-D-Inc/ctrld/internal/dns"
+	"github.com/Control-D-Inc/ctrld/internal/resolvconffile"
+)
+
+// allocate loopback ip
+// sudo ifconfig lo0 127.0.0.53 alias
+func allocateIP(ip string) error {
+	cmd := exec.Command("ifconfig", "lo0", ip, "alias")
+	if err := cmd.Run(); err != nil {
+		mainLog.Load().Error().Err(err).Msg("allocateIP failed")
+		return err
+	}
+	return nil
+}
+
+func deAllocateIP(ip string) error {
+	cmd := exec.Command("ifconfig", "lo0", ip, "-alias")
+	if err := cmd.Run(); err != nil {
+		mainLog.Load().Error().Err(err).Msg("deAllocateIP failed")
+		return err
+	}
+	return nil
+}
+
+// set the dns server for the provided network interface
+func setDNS(iface *net.Interface, nameservers []string) error {
+	r, err := dns.NewOSConfigurator(logf, iface.Name)
+	if err != nil {
+		mainLog.Load().Error().Err(err).Msg("failed to create DNS OS configurator")
+		return err
+	}
+
+	ns := make([]netip.Addr, 0, len(nameservers))
+	for _, nameserver := range nameservers {
+		ns = append(ns, netip.MustParseAddr(nameserver))
+	}
+
+	if err := r.SetDNS(dns.OSConfig{Nameservers: ns}); err != nil {
+		mainLog.Load().Error().Err(err).Msg("failed to set DNS")
+		return err
+	}
+	return nil
+}
+
+func resetDNS(iface *net.Interface) error {
+	r, err := dns.NewOSConfigurator(logf, iface.Name)
+	if err != nil {
+		mainLog.Load().Error().Err(err).Msg("failed to create DNS OS configurator")
+		return err
+	}
+
+	if err := r.Close(); err != nil {
+		mainLog.Load().Error().Err(err).Msg("failed to rollback DNS setting")
+		return err
+	}
+	return nil
+}
+
+func currentDNS(_ *net.Interface) []string {
+	return resolvconffile.NameServers("")
+}

cmd/cli/os_linux.go

@@ -0,0 +1,267 @@
+package cli
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/netip"
+	"os/exec"
+	"reflect"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
+	"github.com/insomniacslk/dhcp/dhcpv6"
+	"github.com/insomniacslk/dhcp/dhcpv6/client6"
+	"tailscale.com/util/dnsname"
+
+	"github.com/Control-D-Inc/ctrld/internal/dns"
+	ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
+	"github.com/Control-D-Inc/ctrld/internal/resolvconffile"
+)
+
+const resolvConfBackupFailedMsg = "open /etc/resolv.pre-ctrld-backup.conf: read-only file system"
+
+// allocate loopback ip
+// sudo ip a add 127.0.0.2/24 dev lo
+func allocateIP(ip string) error {
+	cmd := exec.Command("ip", "a", "add", ip+"/24", "dev", "lo")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		mainLog.Load().Error().Err(err).Msgf("allocateIP failed: %s", string(out))
+		return err
+	}
+	return nil
+}
+
+func deAllocateIP(ip string) error {
+	cmd := exec.Command("ip", "a", "del", ip+"/24", "dev", "lo")
+	if err := cmd.Run(); err != nil {
+		mainLog.Load().Error().Err(err).Msg("deAllocateIP failed")
+		return err
+	}
+	return nil
+}
+
+const maxSetDNSAttempts = 5
+
+// set the dns server for the provided network interface
+func setDNS(iface *net.Interface, nameservers []string) error {
+	r, err := dns.NewOSConfigurator(logf, iface.Name)
+	if err != nil {
+		mainLog.Load().Error().Err(err).Msg("failed to create DNS OS configurator")
+		return err
+	}
+
+	ns := make([]netip.Addr, 0, len(nameservers))
+	for _, nameserver := range nameservers {
+		ns = append(ns, netip.MustParseAddr(nameserver))
+	}
+
+	osConfig := dns.OSConfig{
+		Nameservers:   ns,
+		SearchDomains: []dnsname.FQDN{},
+	}
+
+	trySystemdResolve := false
+	for i := 0; i < maxSetDNSAttempts; i++ {
+		if err := r.SetDNS(osConfig); err != nil {
+			if strings.Contains(err.Error(), "Rejected send message") &&
+				strings.Contains(err.Error(), "org.freedesktop.network1.Manager") {
+				mainLog.Load().Warn().Msg("Interfaces are managed by systemd-networkd, switch to systemd-resolve for setting DNS")
+				trySystemdResolve = true
+				break
+			}
+			// This error happens on read-only file system, which causes ctrld failed to create backup
+			// for /etc/resolv.conf file. It is ok, because the DNS is still set anyway, and restore
+			// DNS will fallback to use DHCP if there's no backup /etc/resolv.conf file.
+			// The error format is controlled by us, so checking for error string is fine.
+			// See: ../../internal/dns/direct.go:L278
+			if r.Mode() == "direct" && strings.Contains(err.Error(), resolvConfBackupFailedMsg) {
+				return nil
+			}
+			return err
+		}
+		currentNS := currentDNS(iface)
+		if reflect.DeepEqual(currentNS, nameservers) {
+			return nil
+		}
+	}
+	if trySystemdResolve {
+		// Stop systemd-networkd and retry setting DNS.
+		if out, err := exec.Command("systemctl", "stop", "systemd-networkd").CombinedOutput(); err != nil {
+			return fmt.Errorf("%s: %w", string(out), err)
+		}
+		args := []string{"--interface=" + iface.Name, "--set-domain=~"}
+		for _, nameserver := range nameservers {
+			args = append(args, "--set-dns="+nameserver)
+		}
+		for i := 0; i < maxSetDNSAttempts; i++ {
+			if out, err := exec.Command("systemd-resolve", args...).CombinedOutput(); err != nil {
+				return fmt.Errorf("%s: %w", string(out), err)
+			}
+			currentNS := currentDNS(iface)
+			if reflect.DeepEqual(currentNS, nameservers) {
+				return nil
+			}
+			time.Sleep(time.Second)
+		}
+	}
+	mainLog.Load().Debug().Msg("DNS was not set for some reason")
+	return nil
+}
+
+func resetDNS(iface *net.Interface) (err error) {
+	defer func() {
+		if err == nil {
+			return
+		}
+		// Start systemd-networkd if present.
+		if exe, _ := exec.LookPath("/lib/systemd/systemd-networkd"); exe != "" {
+			_ = exec.Command("systemctl", "start", "systemd-networkd").Run()
+		}
+		if r, oerr := dns.NewOSConfigurator(logf, iface.Name); oerr == nil {
+			_ = r.SetDNS(dns.OSConfig{})
+			if err := r.Close(); err != nil {
+				mainLog.Load().Error().Err(err).Msg("failed to rollback DNS setting")
+				return
+			}
+			err = nil
+		}
+	}()
+
+	var ns []string
+	c, err := nclient4.New(iface.Name)
+	if err != nil {
+		return fmt.Errorf("nclient4.New: %w", err)
+	}
+	defer c.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	lease, err := c.Request(ctx)
+	if err != nil {
+		return fmt.Errorf("nclient4.Request: %w", err)
+	}
+	for _, nameserver := range lease.ACK.DNS() {
+		if nameserver.Equal(net.IPv4zero) {
+			continue
+		}
+		ns = append(ns, nameserver.String())
+	}
+
+	// TODO(cuonglm): handle DHCPv6 properly.
+	if ctrldnet.IPv6Available(ctx) {
+		c := client6.NewClient()
+		conversation, err := c.Exchange(iface.Name)
+		if err != nil && !errAddrInUse(err) {
+			mainLog.Load().Debug().Err(err).Msg("could not exchange DHCPv6")
+		}
+		for _, packet := range conversation {
+			if packet.Type() == dhcpv6.MessageTypeReply {
+				msg, err := packet.GetInnerMessage()
+				if err != nil {
+					mainLog.Load().Debug().Err(err).Msg("could not get inner DHCPv6 message")
+					return nil
+				}
+				nameservers := msg.Options.DNS()
+				for _, nameserver := range nameservers {
+					ns = append(ns, nameserver.String())
+				}
+			}
+		}
+	}
+
+	return ignoringEINTR(func() error {
+		return setDNS(iface, ns)
+	})
+}
+
+func currentDNS(iface *net.Interface) []string {
+	for _, fn := range []getDNS{getDNSByResolvectl, getDNSBySystemdResolved, getDNSByNmcli, resolvconffile.NameServers} {
+		if ns := fn(iface.Name); len(ns) > 0 {
+			return ns
+		}
+	}
+	return nil
+}
+
+func getDNSByResolvectl(iface string) []string {
+	b, err := exec.Command("resolvectl", "dns", "-i", iface).Output()
+	if err != nil {
+		return nil
+	}
+
+	parts := strings.Fields(strings.SplitN(string(b), "%", 2)[0])
+	if len(parts) > 2 {
+		return parts[3:]
+	}
+	return nil
+}
+
+func getDNSBySystemdResolved(iface string) []string {
+	b, err := exec.Command("systemd-resolve", "--status", iface).Output()
+	if err != nil {
+		return nil
+	}
+	return getDNSBySystemdResolvedFromReader(bytes.NewReader(b))
+}
+
+func getDNSBySystemdResolvedFromReader(r io.Reader) []string {
+	scanner := bufio.NewScanner(r)
+	var ret []string
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if len(ret) > 0 {
+			if net.ParseIP(line) != nil {
+				ret = append(ret, line)
+			}
+			continue
+		}
+		after, found := strings.CutPrefix(line, "DNS Servers: ")
+		if !found {
+			continue
+		}
+		if net.ParseIP(after) != nil {
+			ret = append(ret, after)
+		}
+	}
+	return ret
+}
+
+func getDNSByNmcli(iface string) []string {
+	b, err := exec.Command("nmcli", "dev", "show", iface).Output()
+	if err != nil {
+		return nil
+	}
+	s := bufio.NewScanner(bytes.NewReader(b))
+	var dns []string
+	do := func(line string) {
+		parts := strings.SplitN(line, ":", 2)
+		if len(parts) > 1 {
+			dns = append(dns, strings.TrimSpace(parts[1]))
+		}
+	}
+	for s.Scan() {
+		line := s.Text()
+		switch {
+		case strings.HasPrefix(line, "IP4.DNS"):
+			fallthrough
+		case strings.HasPrefix(line, "IP6.DNS"):
+			do(line)
+		}
+	}
+	return dns
+}
+
+func ignoringEINTR(fn func() error) error {
+	for {
+		err := fn()
+		if err != syscall.EINTR {
+			return err
+		}
+	}
+}

cmd/cli/os_linux_test.go

@@ -0,0 +1,23 @@
+package cli
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func Test_getDNSBySystemdResolvedFromReader(t *testing.T) {
+	r := strings.NewReader(`Link 2 (eth0)
+      Current Scopes: DNS
+       LLMNR setting: yes
+MulticastDNS setting: no
+      DNSSEC setting: no
+    DNSSEC supported: no
+         DNS Servers: 8.8.8.8
+                      8.8.4.4`)
+	want := []string{"8.8.8.8", "8.8.4.4"}
+	ns := getDNSBySystemdResolvedFromReader(r)
+	if !reflect.DeepEqual(ns, want) {
+		t.Logf("unexpected result, want: %v, got: %v", want, ns)
+	}
+}

cmd/cli/os_others.go

@@ -0,0 +1,13 @@
+//go:build !linux && !darwin && !freebsd
+
+package cli
+
+// TODO(cuonglm): implement.
+func allocateIP(ip string) error {
+	return nil
+}
+
+// TODO(cuonglm): implement.
+func deAllocateIP(ip string) error {
+	return nil
+}

cmd/cli/os_windows.go

@@ -0,0 +1,95 @@
+package cli
+
+import (
+	"errors"
+	"net"
+	"os/exec"
+	"strconv"
+
+	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
+
+	ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
+)
+
+func setDNS(iface *net.Interface, nameservers []string) error {
+	if len(nameservers) == 0 {
+		return errors.New("empty DNS nameservers")
+	}
+	primaryDNS := nameservers[0]
+	if err := setPrimaryDNS(iface, primaryDNS); err != nil {
+		return err
+	}
+	if len(nameservers) > 1 {
+		secondaryDNS := nameservers[1]
+		_ = addSecondaryDNS(iface, secondaryDNS)
+	}
+	return nil
+}
+
+// TODO(cuonglm): should we use system API?
+func resetDNS(iface *net.Interface) error {
+	if ctrldnet.SupportsIPv6ListenLocal() {
+		if output, err := netsh("interface", "ipv6", "set", "dnsserver", strconv.Itoa(iface.Index), "dhcp"); err != nil {
+			mainLog.Load().Warn().Err(err).Msgf("failed to reset ipv6 DNS: %s", string(output))
+		}
+	}
+	output, err := netsh("interface", "ipv4", "set", "dnsserver", strconv.Itoa(iface.Index), "dhcp")
+	if err != nil {
+		mainLog.Load().Error().Err(err).Msgf("failed to reset ipv4 DNS: %s", string(output))
+		return err
+	}
+	return nil
+}
+
+func setPrimaryDNS(iface *net.Interface, dns string) error {
+	ipVer := "ipv4"
+	if ctrldnet.IsIPv6(dns) {
+		ipVer = "ipv6"
+	}
+	idx := strconv.Itoa(iface.Index)
+	output, err := netsh("interface", ipVer, "set", "dnsserver", idx, "static", dns)
+	if err != nil {
+		mainLog.Load().Error().Err(err).Msgf("failed to set primary DNS: %s", string(output))
+		return err
+	}
+	if ipVer == "ipv4" && ctrldnet.SupportsIPv6ListenLocal() {
+		// Disable IPv6 DNS, so the query will be fallback to IPv4.
+		_, _ = netsh("interface", "ipv6", "set", "dnsserver", idx, "static", "::1", "primary")
+	}
+
+	return nil
+}
+
+func addSecondaryDNS(iface *net.Interface, dns string) error {
+	ipVer := "ipv4"
+	if ctrldnet.IsIPv6(dns) {
+		ipVer = "ipv6"
+	}
+	output, err := netsh("interface", ipVer, "add", "dns", strconv.Itoa(iface.Index), dns, "index=2")
+	if err != nil {
+		mainLog.Load().Warn().Err(err).Msgf("failed to add secondary DNS: %s", string(output))
+	}
+	return nil
+}
+
+func netsh(args ...string) ([]byte, error) {
+	return exec.Command("netsh", args...).Output()
+}
+
+func currentDNS(iface *net.Interface) []string {
+	luid, err := winipcfg.LUIDFromIndex(uint32(iface.Index))
+	if err != nil {
+		mainLog.Load().Error().Err(err).Msg("failed to get interface LUID")
+		return nil
+	}
+	nameservers, err := luid.DNS()
+	if err != nil {
+		mainLog.Load().Error().Err(err).Msg("failed to get interface DNS")
+		return nil
+	}
+	ns := make([]string, 0, len(nameservers))
+	for _, nameserver := range nameservers {
+		ns = append(ns, nameserver.String())
+	}
+	return ns
+}

cmd/cli/prog.go

@@ -0,0 +1,390 @@
+package cli
+
+import (
+	"errors"
+	"fmt"
+	"math/rand"
+	"net"
+	"net/url"
+	"os"
+	"runtime"
+	"strconv"
+	"sync"
+	"syscall"
+
+	"github.com/kardianos/service"
+	"tailscale.com/net/interfaces"
+
+	"github.com/Control-D-Inc/ctrld"
+	"github.com/Control-D-Inc/ctrld/internal/clientinfo"
+	"github.com/Control-D-Inc/ctrld/internal/dnscache"
+	"github.com/Control-D-Inc/ctrld/internal/router"
+)
+
+const (
+	defaultSemaphoreCap  = 256
+	ctrldLogUnixSock     = "ctrld_start.sock"
+	ctrldControlUnixSock = "ctrld_control.sock"
+)
+
+var logf = func(format string, args ...any) {
+	mainLog.Load().Debug().Msgf(format, args...)
+}
+
+var svcConfig = &service.Config{
+	Name:        "ctrld",
+	DisplayName: "Control-D Helper Service",
+	Option:      service.KeyValue{},
+}
+
+var useSystemdResolved = false
+
+type prog struct {
+	mu      sync.Mutex
+	waitCh  chan struct{}
+	stopCh  chan struct{}
+	logConn net.Conn
+	cs      *controlServer
+
+	cfg     *ctrld.Config
+	cache   dnscache.Cacher
+	sema    semaphore
+	ciTable *clientinfo.Table
+	router  router.Router
+
+	started       chan struct{}
+	onStartedDone chan struct{}
+	onStarted     []func()
+	onStopped     []func()
+}
+
+func (p *prog) Start(s service.Service) error {
+	p.cfg = &cfg
+	go p.run()
+	return nil
+}
+
+func (p *prog) preRun() {
+	if !service.Interactive() {
+		p.setDNS()
+	}
+	if runtime.GOOS == "darwin" {
+		p.onStopped = append(p.onStopped, func() {
+			if !service.Interactive() {
+				p.resetDNS()
+			}
+		})
+	}
+}
+
+func (p *prog) run() {
+	// Wait the caller to signal that we can do our logic.
+	<-p.waitCh
+	p.preRun()
+	numListeners := len(p.cfg.Listener)
+	p.started = make(chan struct{}, numListeners)
+	p.onStartedDone = make(chan struct{})
+	if p.cfg.Service.CacheEnable {
+		cacher, err := dnscache.NewLRUCache(p.cfg.Service.CacheSize)
+		if err != nil {
+			mainLog.Load().Error().Err(err).Msg("failed to create cacher, caching is disabled")
+		} else {
+			p.cache = cacher
+		}
+	}
+	p.sema = &chanSemaphore{ready: make(chan struct{}, defaultSemaphoreCap)}
+	if mcr := p.cfg.Service.MaxConcurrentRequests; mcr != nil {
+		n := *mcr
+		if n == 0 {
+			p.sema = &noopSemaphore{}
+		} else {
+			p.sema = &chanSemaphore{ready: make(chan struct{}, n)}
+		}
+	}
+	var wg sync.WaitGroup
+	wg.Add(len(p.cfg.Listener))
+
+	for _, nc := range p.cfg.Network {
+		for _, cidr := range nc.Cidrs {
+			_, ipNet, err := net.ParseCIDR(cidr)
+			if err != nil {
+				mainLog.Load().Error().Err(err).Str("network", nc.Name).Str("cidr", cidr).Msg("invalid cidr")
+				continue
+			}
+			nc.IPNets = append(nc.IPNets, ipNet)
+		}
+	}
+	for n := range p.cfg.Upstream {
+		uc := p.cfg.Upstream[n]
+		uc.Init()
+		if uc.BootstrapIP == "" {
+			uc.SetupBootstrapIP()
+			mainLog.Load().Info().Msgf("bootstrap IPs for upstream.%s: %q", n, uc.BootstrapIPs())
+		} else {
+			mainLog.Load().Info().Str("bootstrap_ip", uc.BootstrapIP).Msgf("using bootstrap IP for upstream.%s", n)
+		}
+		uc.SetCertPool(rootCertPool)
+		go uc.Ping()
+	}
+
+	p.ciTable = clientinfo.NewTable(&cfg, defaultRouteIP(), cdUID)
+	if leaseFile := p.cfg.Service.DHCPLeaseFile; leaseFile != "" {
+		mainLog.Load().Debug().Msgf("watching custom lease file: %s", leaseFile)
+		format := ctrld.LeaseFileFormat(p.cfg.Service.DHCPLeaseFileFormat)
+		p.ciTable.AddLeaseFile(leaseFile, format)
+	}
+
+	go func() {
+		p.ciTable.Init()
+		p.ciTable.RefreshLoop(p.stopCh)
+	}()
+	go p.watchLinkState()
+
+	for listenerNum := range p.cfg.Listener {
+		p.cfg.Listener[listenerNum].Init()
+		go func(listenerNum string) {
+			defer wg.Done()
+			listenerConfig := p.cfg.Listener[listenerNum]
+			upstreamConfig := p.cfg.Upstream[listenerNum]
+			if upstreamConfig == nil {
+				mainLog.Load().Warn().Msgf("no default upstream for: [listener.%s]", listenerNum)
+			}
+			addr := net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port))
+			mainLog.Load().Info().Msgf("starting DNS server on listener.%s: %s", listenerNum, addr)
+			if err := p.serveDNS(listenerNum); err != nil {
+				mainLog.Load().Fatal().Err(err).Msgf("unable to start dns proxy on listener.%s", listenerNum)
+			}
+		}(listenerNum)
+	}
+
+	for i := 0; i < numListeners; i++ {
+		<-p.started
+	}
+	for _, f := range p.onStarted {
+		f()
+	}
+	close(p.onStartedDone)
+
+	// Stop writing log to unix socket.
+	consoleWriter.Out = os.Stdout
+	initLoggingWithBackup(false)
+	if p.logConn != nil {
+		_ = p.logConn.Close()
+	}
+	if p.cs != nil {
+		p.registerControlServerHandler()
+		if err := p.cs.start(); err != nil {
+			mainLog.Load().Warn().Err(err).Msg("could not start control server")
+		}
+	}
+	wg.Wait()
+}
+
+func (p *prog) Stop(s service.Service) error {
+	mainLog.Load().Info().Msg("Service stopped")
+	close(p.stopCh)
+	if err := p.deAllocateIP(); err != nil {
+		mainLog.Load().Error().Err(err).Msg("de-allocate ip failed")
+		return err
+	}
+	return nil
+}
+
+func (p *prog) allocateIP(ip string) error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if !p.cfg.Service.AllocateIP {
+		return nil
+	}
+	return allocateIP(ip)
+}
+
+func (p *prog) deAllocateIP() error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if !p.cfg.Service.AllocateIP {
+		return nil
+	}
+	for _, lc := range p.cfg.Listener {
+		if err := deAllocateIP(lc.IP); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *prog) setDNS() {
+	if cfg.Listener == nil {
+		return
+	}
+	if iface == "" {
+		return
+	}
+	if iface == "auto" {
+		iface = defaultIfaceName()
+	}
+	lc := cfg.FirstListener()
+	if lc == nil {
+		return
+	}
+	logger := mainLog.Load().With().Str("iface", iface).Logger()
+	netIface, err := netInterface(iface)
+	if err != nil {
+		logger.Error().Err(err).Msg("could not get interface")
+		return
+	}
+	if err := setupNetworkManager(); err != nil {
+		logger.Error().Err(err).Msg("could not patch NetworkManager")
+		return
+	}
+
+	logger.Debug().Msg("setting DNS for interface")
+	ns := lc.IP
+	switch {
+	case lc.IsDirectDnsListener():
+		// If ctrld is direct listener, use 127.0.0.1 as nameserver.
+		ns = "127.0.0.1"
+	case lc.Port != 53:
+		ns = "127.0.0.1"
+		if resolver := router.LocalResolverIP(); resolver != "" {
+			ns = resolver
+		}
+	default:
+		// If we ever reach here, it means ctrld is running on lc.IP port 53,
+		// so we could just use lc.IP as nameserver.
+	}
+
+	nameservers := []string{ns}
+	if needRFC1918Listeners(lc) {
+		nameservers = append(nameservers, rfc1918Addresses()...)
+	}
+	if err := setDNS(netIface, nameservers); err != nil {
+		logger.Error().Err(err).Msgf("could not set DNS for interface")
+		return
+	}
+	logger.Debug().Msg("setting DNS successfully")
+}
+
+func (p *prog) resetDNS() {
+	if iface == "" {
+		return
+	}
+	if iface == "auto" {
+		iface = defaultIfaceName()
+	}
+	logger := mainLog.Load().With().Str("iface", iface).Logger()
+	netIface, err := netInterface(iface)
+	if err != nil {
+		logger.Error().Err(err).Msg("could not get interface")
+		return
+	}
+	if err := restoreNetworkManager(); err != nil {
+		logger.Error().Err(err).Msg("could not restore NetworkManager")
+		return
+	}
+	logger.Debug().Msg("Restoring DNS for interface")
+	if err := resetDNS(netIface); err != nil {
+		logger.Error().Err(err).Msgf("could not reset DNS")
+		return
+	}
+	logger.Debug().Msg("Restoring DNS successfully")
+}
+
+func randomLocalIP() string {
+	n := rand.Intn(254-2) + 2
+	return fmt.Sprintf("127.0.0.%d", n)
+}
+
+func randomPort() int {
+	max := 1<<16 - 1
+	min := 1025
+	n := rand.Intn(max-min) + min
+	return n
+}
+
+// runLogServer starts a unix listener, use by startCmd to gather log from runCmd.
+func runLogServer(sockPath string) net.Conn {
+	addr, err := net.ResolveUnixAddr("unix", sockPath)
+	if err != nil {
+		mainLog.Load().Warn().Err(err).Msg("invalid log sock path")
+		return nil
+	}
+	ln, err := net.ListenUnix("unix", addr)
+	if err != nil {
+		mainLog.Load().Warn().Err(err).Msg("could not listen log socket")
+		return nil
+	}
+	defer ln.Close()
+
+	server, err := ln.Accept()
+	if err != nil {
+		mainLog.Load().Warn().Err(err).Msg("could not accept connection")
+		return nil
+	}
+	return server
+}
+
+func errAddrInUse(err error) bool {
+	var opErr *net.OpError
+	if errors.As(err, &opErr) {
+		return errors.Is(opErr.Err, syscall.EADDRINUSE) || errors.Is(opErr.Err, windowsEADDRINUSE)
+	}
+	return false
+}
+
+// https://learn.microsoft.com/en-us/windows/win32/winsock/windows-sockets-error-codes-2
+var (
+	windowsECONNREFUSED = syscall.Errno(10061)
+	windowsENETUNREACH  = syscall.Errno(10051)
+	windowsEINVAL       = syscall.Errno(10022)
+	windowsEADDRINUSE   = syscall.Errno(10048)
+)
+
+func errUrlNetworkError(err error) bool {
+	var urlErr *url.Error
+	if errors.As(err, &urlErr) {
+		var opErr *net.OpError
+		if errors.As(urlErr.Err, &opErr) {
+			if opErr.Temporary() {
+				return true
+			}
+			switch {
+			case errors.Is(opErr.Err, syscall.ECONNREFUSED),
+				errors.Is(opErr.Err, syscall.EINVAL),
+				errors.Is(opErr.Err, syscall.ENETUNREACH),
+				errors.Is(opErr.Err, windowsENETUNREACH),
+				errors.Is(opErr.Err, windowsEINVAL),
+				errors.Is(opErr.Err, windowsECONNREFUSED):
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// defaultRouteIP returns IP string of the default route if present, prefer IPv4 over IPv6.
+func defaultRouteIP() string {
+	if dr, err := interfaces.DefaultRoute(); err == nil {
+		if netIface, err := netInterface(dr.InterfaceName); err == nil {
+			addrs, _ := netIface.Addrs()
+			do := func(v4 bool) net.IP {
+				for _, addr := range addrs {
+					if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.IsPrivate() {
+						if v4 {
+							return netIP.IP.To4()
+						}
+						return netIP.IP
+					}
+				}
+				return nil
+			}
+			if ip := do(true); ip != nil {
+				return ip.String()
+			}
+			if ip := do(false); ip != nil {
+				return ip.String()
+			}
+		}
+	}
+	return ""
+}

cmd/cli/prog_darwin.go

@@ -0,0 +1,11 @@
+package cli
+
+import (
+	"github.com/kardianos/service"
+)
+
+func setDependencies(svc *service.Config) {}
+
+func setWorkingDirectory(svc *service.Config, dir string) {
+	svc.WorkingDirectory = dir
+}

cmd/cli/prog_freebsd.go

@@ -0,0 +1,14 @@
+package cli
+
+import (
+	"os"
+
+	"github.com/kardianos/service"
+)
+
+func setDependencies(svc *service.Config) {
+	// TODO(cuonglm): remove once https://github.com/kardianos/service/issues/359 fixed.
+	_ = os.MkdirAll("/usr/local/etc/rc.d", 0755)
+}
+
+func setWorkingDirectory(svc *service.Config, dir string) {}

cmd/cli/prog_linux.go

@@ -0,0 +1,32 @@
+package cli
+
+import (
+	"github.com/kardianos/service"
+
+	"github.com/Control-D-Inc/ctrld/internal/dns"
+	"github.com/Control-D-Inc/ctrld/internal/router"
+)
+
+func init() {
+	if r, err := dns.NewOSConfigurator(func(format string, args ...any) {}, "lo"); err == nil {
+		useSystemdResolved = r.Mode() == "systemd-resolved"
+	}
+}
+
+func setDependencies(svc *service.Config) {
+	svc.Dependencies = []string{
+		"Wants=network-online.target",
+		"After=network-online.target",
+		"Wants=NetworkManager-wait-online.service",
+		"After=NetworkManager-wait-online.service",
+		"Wants=systemd-networkd-wait-online.service",
+		"After=systemd-networkd-wait-online.service",
+	}
+	if routerDeps := router.ServiceDependencies(); len(routerDeps) > 0 {
+		svc.Dependencies = append(svc.Dependencies, routerDeps...)
+	}
+}
+
+func setWorkingDirectory(svc *service.Config, dir string) {
+	svc.WorkingDirectory = dir
+}

cmd/cli/prog_others.go

@@ -0,0 +1,12 @@
+//go:build !linux && !freebsd && !darwin
+
+package cli
+
+import "github.com/kardianos/service"
+
+func setDependencies(svc *service.Config) {}
+
+func setWorkingDirectory(svc *service.Config, dir string) {
+	// WorkingDirectory is not supported on Windows.
+	svc.WorkingDirectory = dir
+}

cmd/cli/sema.go

@@ -0,0 +1,24 @@
+package cli
+
+type semaphore interface {
+	acquire()
+	release()
+}
+
+type noopSemaphore struct{}
+
+func (n noopSemaphore) acquire() {}
+
+func (n noopSemaphore) release() {}
+
+type chanSemaphore struct {
+	ready chan struct{}
+}
+
+func (c *chanSemaphore) acquire() {
+	c.ready <- struct{}{}
+}
+
+func (c *chanSemaphore) release() {
+	<-c.ready
+}

cmd/cli/service.go

@@ -0,0 +1,167 @@
+package cli
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/kardianos/service"
+
+	"github.com/Control-D-Inc/ctrld/internal/router"
+)
+
+// newService wraps service.New call to return service.Service
+// wrapper which is suitable for the current platform.
+func newService(i service.Interface, c *service.Config) (service.Service, error) {
+	s, err := service.New(i, c)
+	if err != nil {
+		return nil, err
+	}
+	switch {
+	case router.IsOldOpenwrt():
+		return &procd{&sysV{s}}, nil
+	case router.IsGLiNet():
+		return &sysV{s}, nil
+	case s.Platform() == "unix-systemv":
+		return &sysV{s}, nil
+	case s.Platform() == "linux-systemd":
+		return &systemd{s}, nil
+	}
+	return s, nil
+}
+
+// sysV wraps a service.Service, and provide start/stop/status command
+// base on "/etc/init.d/<service_name>".
+//
+// Use this on system where "service" command is not available, like GL.iNET router.
+type sysV struct {
+	service.Service
+}
+
+func (s *sysV) installed() bool {
+	fi, err := os.Stat("/etc/init.d/ctrld")
+	if err != nil {
+		return false
+	}
+	mode := fi.Mode()
+	return mode.IsRegular() && (mode&0111) != 0
+}
+
+func (s *sysV) Start() error {
+	if !s.installed() {
+		return service.ErrNotInstalled
+	}
+	_, err := exec.Command("/etc/init.d/ctrld", "start").CombinedOutput()
+	return err
+}
+
+func (s *sysV) Stop() error {
+	if !s.installed() {
+		return service.ErrNotInstalled
+	}
+	_, err := exec.Command("/etc/init.d/ctrld", "stop").CombinedOutput()
+	return err
+}
+
+func (s *sysV) Restart() error {
+	if !s.installed() {
+		return service.ErrNotInstalled
+	}
+	// We don't care about error returned by s.Stop,
+	// because the service may already be stopped.
+	_ = s.Stop()
+	return s.Start()
+}
+
+func (s *sysV) Status() (service.Status, error) {
+	if !s.installed() {
+		return service.StatusUnknown, service.ErrNotInstalled
+	}
+	return unixSystemVServiceStatus()
+}
+
+// procd wraps a service.Service, and provide start/stop command
+// base on "/etc/init.d/<service_name>", status command base on parsing "ps" command output.
+//
+// Use this on system where "/etc/init.d/<service_name> status" command is not available,
+// like old GL.iNET Opal router.
+type procd struct {
+	*sysV
+}
+
+func (s *procd) Status() (service.Status, error) {
+	if !s.installed() {
+		return service.StatusUnknown, service.ErrNotInstalled
+	}
+	exe, err := os.Executable()
+	if err != nil {
+		return service.StatusUnknown, nil
+	}
+	// Looking for something like "/sbin/ctrld run ".
+	shellCmd := fmt.Sprintf("ps | grep -q %q", exe+" [r]un ")
+	if err := exec.Command("sh", "-c", shellCmd).Run(); err != nil {
+		return service.StatusStopped, nil
+	}
+	return service.StatusRunning, nil
+}
+
+// procd wraps a service.Service, and provide status command to
+// report the status correctly.
+type systemd struct {
+	service.Service
+}
+
+func (s *systemd) Status() (service.Status, error) {
+	out, _ := exec.Command("systemctl", "status", "ctrld").CombinedOutput()
+	if bytes.Contains(out, []byte("/FAILURE)")) {
+		return service.StatusStopped, nil
+	}
+	return s.Service.Status()
+}
+
+type task struct {
+	f            func() error
+	abortOnError bool
+}
+
+func doTasks(tasks []task) bool {
+	var prevErr error
+	for _, task := range tasks {
+		if err := task.f(); err != nil {
+			if task.abortOnError {
+				mainLog.Load().Error().Msg(errors.Join(prevErr, err).Error())
+				return false
+			}
+			prevErr = err
+		}
+	}
+	return true
+}
+
+func checkHasElevatedPrivilege() {
+	ok, err := hasElevatedPrivilege()
+	if err != nil {
+		mainLog.Load().Error().Msgf("could not detect user privilege: %v", err)
+		return
+	}
+	if !ok {
+		mainLog.Load().Error().Msg("Please relaunch process with admin/root privilege.")
+		os.Exit(1)
+	}
+}
+
+func unixSystemVServiceStatus() (service.Status, error) {
+	out, err := exec.Command("/etc/init.d/ctrld", "status").CombinedOutput()
+	if err != nil {
+		return service.StatusUnknown, nil
+	}
+
+	switch string(bytes.ToLower(bytes.TrimSpace(out))) {
+	case "running":
+		return service.StatusRunning, nil
+	default:
+		return service.StatusStopped, nil
+	}
+}

cmd/cli/service_others.go

@@ -0,0 +1,11 @@
+//go:build !windows
+
+package cli
+
+import (
+	"os"
+)
+
+func hasElevatedPrivilege() (bool, error) {
+	return os.Geteuid() == 0, nil
+}

cmd/cli/service_windows.go

@@ -0,0 +1,24 @@
+package cli
+
+import "golang.org/x/sys/windows"
+
+func hasElevatedPrivilege() (bool, error) {
+	var sid *windows.SID
+	if err := windows.AllocateAndInitializeSid(
+		&windows.SECURITY_NT_AUTHORITY,
+		2,
+		windows.SECURITY_BUILTIN_DOMAIN_RID,
+		windows.DOMAIN_ALIAS_RID_ADMINS,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		&sid,
+	); err != nil {
+		return false, err
+	}
+	token := windows.Token(0)
+	return token.IsMember(sid)
+}