From a4c1983657905b4037e0f57d79401ea50141987f Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 26 Jun 2023 22:07:03 +0700 Subject: [PATCH] cmd/ctrld: make setDNS works on system using systemd-networkd On Ubuntu 18.04 VM with some cloud provider, using dbus call to set DNS is forbidden. A possible solution is stopping networkd entirely then using systemd-resolve to set DNS when ctrld starts. While at it, only set DNS during start command on Windows. On other platforms, "ctrld run" does set DNS in service mode already. When using systemd-resolved, only change listener address to default route interface address if a loopback address is used. Also fixing a bug in upstream tailscale code for checking in container. See tailscale/tailscale#8444 --- cmd/ctrld/cli.go | 25 ++++++++++----- cmd/ctrld/dns_proxy.go | 2 +- cmd/ctrld/os_linux.go | 64 +++++++++++++++++++++++++++++++++++++- cmd/ctrld/os_linux_test.go | 23 ++++++++++++++ cmd/ctrld/prog_linux.go | 2 ++ 5 files changed, 106 insertions(+), 10 deletions(-) create mode 100644 cmd/ctrld/os_linux_test.go diff --git a/cmd/ctrld/cli.go b/cmd/ctrld/cli.go index 1b0262b..080faec 100644 --- a/cmd/ctrld/cli.go +++ b/cmd/ctrld/cli.go @@ -361,7 +361,12 @@ func initCLI() { uninstall(p, s) os.Exit(1) } - p.setDNS() + // On Linux, Darwin, Freebsd, ctrld set DNS on startup, because the DNS setting could be + // reset after rebooting. On windows, we only need to set once here. See prog.preRun in + // prog_*.go file for dedicated code on each platforms. + if runtime.GOOS == "windows" { + p.setDNS() + } } }, } @@ -781,13 +786,17 @@ func processCDFlags() { } case useSystemdResolved: if lc := cfg.Listener["0"]; lc != nil { - // systemd-resolved does not allow forwarding DNS queries from 127.0.0.53 to loopback - // ip address, so trying to listen on default route interface address instead. - if netIface, _ := net.InterfaceByName(defaultIfaceName()); netIface != nil { - addrs, _ := netIface.Addrs() - for _, addr := range addrs { - if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil { - lc.IP = netIP.IP.To4().String() + if ip := net.ParseIP(lc.IP); ip != nil && ip.IsLoopback() { + mainLog.Warn().Msg("using loopback interface do not work with systemd-resolved") + // systemd-resolved does not allow forwarding DNS queries from 127.0.0.53 to loopback + // ip address, so trying to listen on default route interface address instead. + if netIface, _ := net.InterfaceByName(defaultIfaceName()); netIface != nil { + addrs, _ := netIface.Addrs() + for _, addr := range addrs { + if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil { + lc.IP = netIP.IP.To4().String() + mainLog.Warn().Msgf("use %s as listener address", lc.IP) + } } } } diff --git a/cmd/ctrld/dns_proxy.go b/cmd/ctrld/dns_proxy.go index 38a0ba4..fad8e9c 100644 --- a/cmd/ctrld/dns_proxy.go +++ b/cmd/ctrld/dns_proxy.go @@ -501,7 +501,7 @@ func inContainer() bool { return nil }) lineread.File("/proc/mounts", func(line []byte) error { - if mem.Contains(mem.B(line), mem.S("fuse.lxcfs")) { + if mem.Contains(mem.B(line), mem.S("lxcfs /proc/cpuinfo fuse.lxcfs")) { ret = true return io.EOF } diff --git a/cmd/ctrld/os_linux.go b/cmd/ctrld/os_linux.go index 307ee3a..55f6e7c 100644 --- a/cmd/ctrld/os_linux.go +++ b/cmd/ctrld/os_linux.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "fmt" + "io" "net" "net/netip" "os/exec" @@ -63,8 +64,15 @@ func setDNS(iface *net.Interface, nameservers []string) error { SearchDomains: []dnsname.FQDN{}, } + trySystemdResolve := false for i := 0; i < maxSetDNSAttempts; i++ { if err := r.SetDNS(osConfig); err != nil { + if strings.Contains(err.Error(), "Rejected send message") && + strings.Contains(err.Error(), "org.freedesktop.network1.Manager") { + mainLog.Warn().Msg("Interfaces are managed by systemd-networkd, switch to systemd-resolve for setting DNS") + trySystemdResolve = true + break + } return err } currentNS := currentDNS(iface) @@ -72,6 +80,26 @@ func setDNS(iface *net.Interface, nameservers []string) error { return nil } } + if trySystemdResolve { + // Stop systemd-networkd and retry setting DNS. + if out, err := exec.Command("systemctl", "stop", "systemd-networkd").CombinedOutput(); err != nil { + return fmt.Errorf("%s: %w", string(out), err) + } + args := []string{"--interface=" + iface.Name, "--set-domain=~"} + for _, nameserver := range nameservers { + args = append(args, "--set-dns="+nameserver) + } + for i := 0; i < maxSetDNSAttempts; i++ { + if out, err := exec.Command("systemd-resolve", args...).CombinedOutput(); err != nil { + return fmt.Errorf("%s: %w", string(out), err) + } + currentNS := currentDNS(iface) + if reflect.DeepEqual(currentNS, nameservers) { + return nil + } + time.Sleep(time.Second) + } + } mainLog.Debug().Msg("DNS was not set for some reason") return nil } @@ -81,6 +109,10 @@ func resetDNS(iface *net.Interface) (err error) { if err == nil { return } + // Start systemd-networkd if present. + if exe, _ := exec.LookPath("/lib/systemd/systemd-networkd"); exe != "" { + _ = exec.Command("systemctl", "restart", "systemd-networkd").Run() + } if r, oerr := dns.NewOSConfigurator(logf, iface.Name); oerr == nil { _ = r.SetDNS(dns.OSConfig{}) if err := r.Close(); err != nil { @@ -139,7 +171,7 @@ func resetDNS(iface *net.Interface) (err error) { } func currentDNS(iface *net.Interface) []string { - for _, fn := range []getDNS{getDNSByResolvectl, getDNSByNmcli, resolvconffile.NameServers} { + for _, fn := range []getDNS{getDNSByResolvectl, getDNSBySystemdResolved, getDNSByNmcli, resolvconffile.NameServers} { if ns := fn(iface.Name); len(ns) > 0 { return ns } @@ -160,6 +192,36 @@ func getDNSByResolvectl(iface string) []string { return nil } +func getDNSBySystemdResolved(iface string) []string { + b, err := exec.Command("systemd-resolve", "--status", iface).Output() + if err != nil { + return nil + } + return getDNSBySystemdResolvedFromReader(bytes.NewReader(b)) +} + +func getDNSBySystemdResolvedFromReader(r io.Reader) []string { + scanner := bufio.NewScanner(r) + var ret []string + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if len(ret) > 0 { + if net.ParseIP(line) != nil { + ret = append(ret, line) + } + continue + } + after, found := strings.CutPrefix(line, "DNS Servers: ") + if !found { + continue + } + if net.ParseIP(after) != nil { + ret = append(ret, after) + } + } + return ret +} + func getDNSByNmcli(iface string) []string { b, err := exec.Command("nmcli", "dev", "show", iface).Output() if err != nil { diff --git a/cmd/ctrld/os_linux_test.go b/cmd/ctrld/os_linux_test.go new file mode 100644 index 0000000..671f1b4 --- /dev/null +++ b/cmd/ctrld/os_linux_test.go @@ -0,0 +1,23 @@ +package main + +import ( + "reflect" + "strings" + "testing" +) + +func Test_getDNSBySystemdResolvedFromReader(t *testing.T) { + r := strings.NewReader(`Link 2 (eth0) + Current Scopes: DNS + LLMNR setting: yes +MulticastDNS setting: no + DNSSEC setting: no + DNSSEC supported: no + DNS Servers: 8.8.8.8 + 8.8.4.4`) + want := []string{"8.8.8.8", "8.8.4.4"} + ns := getDNSBySystemdResolvedFromReader(r) + if !reflect.DeepEqual(ns, want) { + t.Logf("unexpected result, want: %v, got: %v", want, ns) + } +} diff --git a/cmd/ctrld/prog_linux.go b/cmd/ctrld/prog_linux.go index 2c070a6..38cd1a5 100644 --- a/cmd/ctrld/prog_linux.go +++ b/cmd/ctrld/prog_linux.go @@ -25,6 +25,8 @@ func setDependencies(svc *service.Config) { "After=network-online.target", "Wants=NetworkManager-wait-online.service", "After=NetworkManager-wait-online.service", + "Wants=systemd-networkd-wait-online.service", + "After=systemd-networkd-wait-online.service", } // On EdeOS, ctrld needs to start after vyatta-dhcpd, so it can read leases file. if router.Name() == router.EdgeOS {