cmd/ctrld: make setDNS works on system using systemd-networkd

On Ubuntu 18.04 VM with some cloud provider, using dbus call to set DNS
is forbidden. A possible solution is stopping networkd entirely then
using systemd-resolve to set DNS when ctrld starts.

While at it, only set DNS during start command on Windows. On other
platforms, "ctrld run" does set DNS in service mode already.

When using systemd-resolved, only change listener address to default
route interface address if a loopback address is used.

Also fixing a bug in upstream tailscale code for checking in container.
See tailscale/tailscale#8444

cmd/ctrld/cli.go

@@ -361,7 +361,12 @@ func initCLI() {
 					uninstall(p, s)
 					os.Exit(1)
 				}
-				p.setDNS()
+				// On Linux, Darwin, Freebsd, ctrld set DNS on startup, because the DNS setting could be
+				// reset after rebooting. On windows, we only need to set once here. See prog.preRun in
+				// prog_*.go file for dedicated code on each platforms.
+				if runtime.GOOS == "windows" {
+					p.setDNS()
+				}
 			}
 		},
 	}
@@ -781,13 +786,17 @@ func processCDFlags() {
 			}
 		case useSystemdResolved:
 			if lc := cfg.Listener["0"]; lc != nil {
-				// systemd-resolved does not allow forwarding DNS queries from 127.0.0.53 to loopback
-				// ip address, so trying to listen on default route interface address instead.
-				if netIface, _ := net.InterfaceByName(defaultIfaceName()); netIface != nil {
-					addrs, _ := netIface.Addrs()
-					for _, addr := range addrs {
-						if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil {
-							lc.IP = netIP.IP.To4().String()
+				if ip := net.ParseIP(lc.IP); ip != nil && ip.IsLoopback() {
+					mainLog.Warn().Msg("using loopback interface do not work with systemd-resolved")
+					// systemd-resolved does not allow forwarding DNS queries from 127.0.0.53 to loopback
+					// ip address, so trying to listen on default route interface address instead.
+					if netIface, _ := net.InterfaceByName(defaultIfaceName()); netIface != nil {
+						addrs, _ := netIface.Addrs()
+						for _, addr := range addrs {
+							if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil {
+								lc.IP = netIP.IP.To4().String()
+								mainLog.Warn().Msgf("use %s as listener address", lc.IP)
+							}
 						}
 					}
 				}

cmd/ctrld/dns_proxy.go

@@ -501,7 +501,7 @@ func inContainer() bool {
 		return nil
 	})
 	lineread.File("/proc/mounts", func(line []byte) error {
-		if mem.Contains(mem.B(line), mem.S("fuse.lxcfs")) {
+		if mem.Contains(mem.B(line), mem.S("lxcfs /proc/cpuinfo fuse.lxcfs")) {
 			ret = true
 			return io.EOF
 		}

cmd/ctrld/os_linux.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"net"
 	"net/netip"
 	"os/exec"
@@ -63,8 +64,15 @@ func setDNS(iface *net.Interface, nameservers []string) error {
 		SearchDomains: []dnsname.FQDN{},
 	}
 
+	trySystemdResolve := false
 	for i := 0; i < maxSetDNSAttempts; i++ {
 		if err := r.SetDNS(osConfig); err != nil {
+			if strings.Contains(err.Error(), "Rejected send message") &&
+				strings.Contains(err.Error(), "org.freedesktop.network1.Manager") {
+				mainLog.Warn().Msg("Interfaces are managed by systemd-networkd, switch to systemd-resolve for setting DNS")
+				trySystemdResolve = true
+				break
+			}
 			return err
 		}
 		currentNS := currentDNS(iface)
@@ -72,6 +80,26 @@ func setDNS(iface *net.Interface, nameservers []string) error {
 			return nil
 		}
 	}
+	if trySystemdResolve {
+		// Stop systemd-networkd and retry setting DNS.
+		if out, err := exec.Command("systemctl", "stop", "systemd-networkd").CombinedOutput(); err != nil {
+			return fmt.Errorf("%s: %w", string(out), err)
+		}
+		args := []string{"--interface=" + iface.Name, "--set-domain=~"}
+		for _, nameserver := range nameservers {
+			args = append(args, "--set-dns="+nameserver)
+		}
+		for i := 0; i < maxSetDNSAttempts; i++ {
+			if out, err := exec.Command("systemd-resolve", args...).CombinedOutput(); err != nil {
+				return fmt.Errorf("%s: %w", string(out), err)
+			}
+			currentNS := currentDNS(iface)
+			if reflect.DeepEqual(currentNS, nameservers) {
+				return nil
+			}
+			time.Sleep(time.Second)
+		}
+	}
 	mainLog.Debug().Msg("DNS was not set for some reason")
 	return nil
 }
@@ -81,6 +109,10 @@ func resetDNS(iface *net.Interface) (err error) {
 		if err == nil {
 			return
 		}
+		// Start systemd-networkd if present.
+		if exe, _ := exec.LookPath("/lib/systemd/systemd-networkd"); exe != "" {
+			_ = exec.Command("systemctl", "restart", "systemd-networkd").Run()
+		}
 		if r, oerr := dns.NewOSConfigurator(logf, iface.Name); oerr == nil {
 			_ = r.SetDNS(dns.OSConfig{})
 			if err := r.Close(); err != nil {
@@ -139,7 +171,7 @@ func resetDNS(iface *net.Interface) (err error) {
 }
 
 func currentDNS(iface *net.Interface) []string {
-	for _, fn := range []getDNS{getDNSByResolvectl, getDNSByNmcli, resolvconffile.NameServers} {
+	for _, fn := range []getDNS{getDNSByResolvectl, getDNSBySystemdResolved, getDNSByNmcli, resolvconffile.NameServers} {
 		if ns := fn(iface.Name); len(ns) > 0 {
 			return ns
 		}
@@ -160,6 +192,36 @@ func getDNSByResolvectl(iface string) []string {
 	return nil
 }
 
+func getDNSBySystemdResolved(iface string) []string {
+	b, err := exec.Command("systemd-resolve", "--status", iface).Output()
+	if err != nil {
+		return nil
+	}
+	return getDNSBySystemdResolvedFromReader(bytes.NewReader(b))
+}
+
+func getDNSBySystemdResolvedFromReader(r io.Reader) []string {
+	scanner := bufio.NewScanner(r)
+	var ret []string
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if len(ret) > 0 {
+			if net.ParseIP(line) != nil {
+				ret = append(ret, line)
+			}
+			continue
+		}
+		after, found := strings.CutPrefix(line, "DNS Servers: ")
+		if !found {
+			continue
+		}
+		if net.ParseIP(after) != nil {
+			ret = append(ret, after)
+		}
+	}
+	return ret
+}
+
 func getDNSByNmcli(iface string) []string {
 	b, err := exec.Command("nmcli", "dev", "show", iface).Output()
 	if err != nil {

cmd/ctrld/os_linux_test.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func Test_getDNSBySystemdResolvedFromReader(t *testing.T) {
+	r := strings.NewReader(`Link 2 (eth0)
+      Current Scopes: DNS
+       LLMNR setting: yes
+MulticastDNS setting: no
+      DNSSEC setting: no
+    DNSSEC supported: no
+         DNS Servers: 8.8.8.8
+                      8.8.4.4`)
+	want := []string{"8.8.8.8", "8.8.4.4"}
+	ns := getDNSBySystemdResolvedFromReader(r)
+	if !reflect.DeepEqual(ns, want) {
+		t.Logf("unexpected result, want: %v, got: %v", want, ns)
+	}
+}

cmd/ctrld/prog_linux.go

@@ -25,6 +25,8 @@ func setDependencies(svc *service.Config) {
 		"After=network-online.target",
 		"Wants=NetworkManager-wait-online.service",
 		"After=NetworkManager-wait-online.service",
+		"Wants=systemd-networkd-wait-online.service",
+		"After=systemd-networkd-wait-online.service",
 	}
 	// On EdeOS, ctrld needs to start after vyatta-dhcpd, so it can read leases file.
 	if router.Name() == router.EdgeOS {