Merge pull request #26 from Control-D-Inc/release-branch-v1.1.1

Release branch v1.1.1

.github/workflows/ci.yml

@@ -9,18 +9,18 @@ jobs:
       fail-fast: false
       matrix:
         os: ["windows-latest", "ubuntu-latest", "macOS-latest"]
-        go: ["1.19.x"]
+        go: ["1.20.x"]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 1
-      - uses: WillAbides/setup-go-faster@v1.7.0
+      - uses: WillAbides/setup-go-faster@v1.8.0
         with:
           go-version: ${{ matrix.go }}
       - run: "go test -race ./..."
       - uses: dominikh/staticcheck-action@v1.2.0
         with:
-          version: "2022.1.1"
+          version: "2023.1.2"
           install-go: false
           cache-key: ${{ matrix.go }}

.gitignore

@@ -1,2 +1,3 @@
 
 dist/
+gon.hcl

.goreleaser-darwin.yaml

@@ -0,0 +1,33 @@
+before:
+  hooks:
+    - go mod tidy
+builds:
+  - id: ctrld-darwin
+    env:
+      - CGO_ENABLED=0
+    flags:
+      - -trimpath
+    ldflags:
+      - -s -w
+    goos:
+      - darwin
+    goarch:
+      - amd64
+      - arm64
+    main: ./cmd/ctrld
+    hooks:
+      post: gon gon.hcl
+archives:
+  - strip_parent_binary_folder: true
+    wrap_in_directory: true
+    files:
+      - README.md
+checksum:
+  name_template: 'checksums.txt'
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'

.goreleaser.yaml

@@ -2,7 +2,8 @@ before:
   hooks:
     - go mod tidy
 builds:
-  - env:
+  - id: ctrld
+    env:
       - CGO_ENABLED=0
     flags:
       - -trimpath
@@ -10,8 +11,8 @@ builds:
       - -s -w
     goos:
       - linux
+      - freebsd
       - windows
-      - darwin
     goarch:
       - 386
       - arm
@@ -25,6 +26,12 @@ builds:
     gomips:
       - softfloat
     main: ./cmd/ctrld
+    hooks:
+      post: /bin/sh ./scripts/upx.sh {{ .Path }}
+    ignore:
+      - goos: freebsd
+        goarch: arm
+        goarm: 5
 archives:
   - format_overrides:
       - goos: windows

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Control D Inc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,4 +1,9 @@
 # ctrld
+
+![Test](https://github.com/Control-D-Inc/ctrld/actions/workflows/ci.yml/badge.svg)
+[![Go Reference](https://pkg.go.dev/badge/github.com/Control-D-Inc/ctrld.svg)](https://pkg.go.dev/github.com/Control-D-Inc/ctrld)
+[![Go Report Card](https://goreportcard.com/badge/github.com/Control-D-Inc/ctrld)](https://goreportcard.com/report/github.com/Control-D-Inc/ctrld)
+
 A highly configurable DNS forwarding proxy with support for:
 - Multiple listeners for incoming queries
 - Multiple upstreams with fallbacks
@@ -15,51 +20,59 @@ All DNS protocols are supported, including:
 ## Use Cases
 1. Use secure DNS protocols on networks and devices that don't natively support them (legacy routers, legacy OSes, TVs, smart toasters).
 2. Create source IP based DNS routing policies with variable secure DNS upstreams. Subnet 1 (admin) uses upstream resolver A, while Subnet 2 (employee) uses upstream resolver B.
-3. Create destination IP based DNS routing policies with variable secure DNS upstreams. Listener 1 uses upstream resolver C, while Listener 2 uses upstream resolver D. 
+3. Create destination IP based DNS routing policies with variable secure DNS upstreams. Listener 1 uses upstream resolver C, while Listener 2 uses upstream resolver D.
 4. Create domain level "split horizon" DNS routing policies to send internal domains (*.company.int) to a local DNS server, while everything else goes to another upstream.
 
 
 ## OS Support
 - Windows (386, amd64, arm)
-- Mac (amd64, arm)
+- Mac (amd64, arm64)
 - Linux (386, amd64, arm, mips)
 
 ## Download
-Download pre-compiled binaries from the [Releases](#) section.
+Download pre-compiled binaries from the [Releases](https://github.com/Control-D-Inc/ctrld/releases) section.
 
 ## Build
 `ctrld` requires `go1.19+`:
 
 ```shell
-$ go build
+$ go build ./cmd/ctrld
 ```
 
-or 
+or
 
 ```shell
-$ go install <path_to_repo>
+$ go install github.com/Control-D-Inc/ctrld/cmd/ctrld@latest
 ```
 
 ## Arguments
 ```
+        __         .__       .___
+  _____/  |________|  |    __| _/
+_/ ___\   __\_  __ \  |   / __ |
+\  \___|  |  |  | \/  |__/ /_/ |
+ \___  >__|  |__|  |____/\____ |
+     \/ dns forwarding proxy  \/
+
 Usage:
   ctrld [command]
 
 Available Commands:
-  help        Help about any command
-  interfaces  Manage Interface DNS settings
   run         Run the DNS proxy server
+  service     Manage ctrld service
+  start       Quick start service and configure DNS on default interface
+  stop        Quick stop service and remove DNS from default interface
 
 Flags:
-  -h, --help      help for ctrld
-  -j, --json      json output
-  -v, --verbose   verbose log output
-      --version   version for ctrld
+  -h, --help            help for ctrld
+  -v, --verbose count   verbose log output, "-v" basic logging, "-vv" debug level logging
+      --version         version for ctrld
 
 Use "ctrld [command] --help" for more information about a command.
 ```
+
 ## Usage
-To start the server with default configuration, simply run: `ctrld run`. This will create a generic `config.toml` file in the working directory and start the service. 
+To start the server with default configuration, simply run: `./ctrld run`. This will create a generic `ctrld.toml` file in the **working directory** and start the application in foreground. 
 1. Start the server
   ```
   $ sudo ./ctrld run
@@ -74,15 +87,67 @@ To start the server with default configuration, simply run: `ctrld run`. This wi
 
 If `verify.controld.com` resolves, you're successfully using the default Control D upstream.
 
+### Service Mode
+To run the application in service mode, simply run: `./ctrld start` as system/root user. This will create a generic `ctrld.toml` file in the **user home** directory, start the system service, and configure the listener on the default interface. Service will start on OS boot.
+
+In order to stop the service, and restore your DNS to original state, simply run `./ctrld stop`.
+
+For granular control of the service, run the `service` command. Each sub-command has its own help section so you can see what arguments you can supply.
+
+```
+  Manage ctrld service
+
+  Usage:
+    ctrld service [command]
+
+  Available Commands:
+    interfaces  Manage network interfaces
+    restart     Restart the ctrld service
+    start       Start the ctrld service
+    status      Show status of the ctrld service
+    stop        Stop the ctrld service
+    uninstall   Uninstall the ctrld service
+
+  Flags:
+    -h, --help   help for service
+
+  Global Flags:
+    -v, --verbose count   verbose log output, "-v" basic logging, "-vv" debug level logging
+
+  Use "ctrld service [command] --help" for more information about a command.
+```
+
+### Control D Auto Configuration
+Application can be started with a specific resolver config, instead of the default one. Simply supply your resolver ID with a `--cd` flag, when using the `run` (foreground) or `start` (service) modes. 
+
+The following command will start the application in foreground mode, using the free "p2" resolver, which blocks Ads & Trackers. 
+
+```shell
+./ctrld run --cd p2
+```
+
+Alternatively, you can use your own personal Control D Device resolver, and start the application in service mode. Your resolver ID is the part after the slash of your DNS-over-HTTPS resolver. ie. https://dns.controld.com/abcd1234
+
+```shell
+./ctrld start --cd abcd1234
+```
+
+Once you run the above command, the following things will happen:
+- You resolver configuration will be fetched from the API, and config file templated with the resolver data
+- Application will start as a service, and keep running (even after reboot) until you run the `stop` or `service uninstall` sub-commands
+- Your default network interface will be updated to use the listener started by the service
+- All OS DNS queries will be sent to the listener
 
 ## Configuration
+See [Configuration Docs](docs/config.md).
+
 ### Example 
 - Start `listener.0` on 127.0.0.1:53
 - Accept queries from any source address
 - Send all queries to `upstream.0` via DoH protocol
 
 ### Default Config
-```toml                                               
+```toml
 [listener]
 
   [listener.0]
@@ -118,8 +183,10 @@ If `verify.controld.com` resolves, you're successfully using the default Control
 
 ```
 
-### Advanced 
-The above is the most basic example, which will work out of the box. If you're looking to do advanced configurations using policies, see [Configuration Docs](docs/config.md) for complete documentation of the config file. 
+### Advanced
+The above is the most basic example, which will work out of the box. If you're looking to do advanced configurations using policies, see [Configuration Docs](docs/config.md) for complete documentation of the config file.
+
+You can also supply configuration via launch argeuments, in [Ephemeral Mode](docs/ephemeral_mode.md).
 
 ## Contributing
 
@@ -127,6 +194,6 @@ See [Contribution Guideline](./docs/contributing.md)
 
 ## Roadmap
 The following functionality is on the roadmap and will be available in future releases. 
+- Router self-installation
+- Client hostname/MAC passthrough
 - Prometheus metrics exporter 
-- Local caching
-- Service self-installation 

cmd/ctrld/cli.go

@@ -1,34 +1,85 @@
 package main
 
 import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
 	"fmt"
 	"log"
+	"net"
+	"net/netip"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"runtime"
+	"strconv"
+	"strings"
+	"time"
 
+	"github.com/fsnotify/fsnotify"
+
+	"github.com/go-playground/validator/v10"
 	"github.com/kardianos/service"
-	"github.com/pelletier/go-toml"
+	"github.com/miekg/dns"
+	"github.com/pelletier/go-toml/v2"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"tailscale.com/logtail/backoff"
+	"tailscale.com/net/interfaces"
+
+	"github.com/Control-D-Inc/ctrld"
+	"github.com/Control-D-Inc/ctrld/internal/controld"
+	ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
 )
 
+const selfCheckFQDN = "verify.controld.com"
+
 var (
 	v                    = viper.NewWithOptions(viper.KeyDelimiter("::"))
 	defaultConfigWritten = false
+	defaultConfigFile    = "ctrld.toml"
 )
 
+var basicModeFlags = []string{"listen", "primary_upstream", "secondary_upstream", "domains"}
+
+func isNoConfigStart(cmd *cobra.Command) bool {
+	for _, flagName := range basicModeFlags {
+		if cmd.Flags().Lookup(flagName).Changed {
+			return true
+		}
+	}
+	return false
+}
+
+const rootShortDesc = `
+        __         .__       .___
+  _____/  |________|  |    __| _/
+_/ ___\   __\_  __ \  |   / __ |
+\  \___|  |  |  | \/  |__/ /_/ |
+ \___  >__|  |__|  |____/\____ |
+     \/ dns forwarding proxy  \/
+`
+
 func initCLI() {
 	// Enable opening via explorer.exe on Windows.
 	// See: https://github.com/spf13/cobra/issues/844.
 	cobra.MousetrapHelpText = ""
+	cobra.EnableCommandSorting = false
 
 	rootCmd := &cobra.Command{
 		Use:     "ctrld",
-		Short:   "Running Control-D DNS proxy server",
-		Version: "1.0.0",
+		Short:   strings.TrimLeft(rootShortDesc, "\n"),
+		Version: "1.1.1",
 	}
-	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose log output")
+	rootCmd.PersistentFlags().CountVarP(
+		&verbose,
+		"verbose",
+		"v",
+		`verbose log output, "-v" basic logging, "-vv" debug level logging`,
+	)
+	rootCmd.SetHelpCommand(&cobra.Command{Hidden: true})
+	rootCmd.CompletionOptions.HiddenDefaultCmd = true
 
 	runCmd := &cobra.Command{
 		Use:   "run",
@@ -38,21 +89,43 @@ func initCLI() {
 			if daemon && runtime.GOOS == "windows" {
 				log.Fatal("Cannot run in daemon mode. Please install a Windows service.")
 			}
-			if configPath != "" {
-				v.SetConfigFile(configPath)
+			noConfigStart := isNoConfigStart(cmd)
+			writeDefaultConfig := !noConfigStart && configBase64 == ""
+			configs := []struct {
+				name    string
+				written bool
+			}{
+				// For compatibility, we check for config.toml first, but only read it if exists.
+				{"config", false},
+				{"ctrld", writeDefaultConfig},
 			}
-			if err := v.ReadInConfig(); err != nil {
-				if _, ok := err.(viper.ConfigFileNotFoundError); ok {
-					writeConfigFile()
-					defaultConfigWritten = true
-				} else {
-					log.Fatalf("failed to decode config file: %v", err)
+			for _, config := range configs {
+				ctrld.SetConfigName(v, config.name)
+				v.SetConfigFile(configPath)
+				if readConfigFile(config.written) {
+					break
 				}
 			}
+
+			readBase64Config()
+			processNoConfigFlags(noConfigStart)
 			if err := v.Unmarshal(&cfg); err != nil {
 				log.Fatalf("failed to unmarshal config: %v", err)
 			}
+			// Wait for network up.
+			if !ctrldnet.Up() {
+				log.Fatal("network is not up yet")
+			}
+			processLogAndCacheFlags()
+			// Log config do not have thing to validate, so it's safe to init log here,
+			// so it's able to log information in processCDFlags.
 			initLogging()
+			processCDFlags()
+			if err := ctrld.ValidateConfig(validator.New(), &cfg); err != nil {
+				log.Fatalf("invalid config: %v", err)
+			}
+			initCache()
+
 			if daemon {
 				exe, err := os.Executable()
 				if err != nil {
@@ -95,22 +168,570 @@ func initCLI() {
 	}
 	runCmd.Flags().BoolVarP(&daemon, "daemon", "d", false, "Run as daemon")
 	runCmd.Flags().StringVarP(&configPath, "config", "c", "", "Path to config file")
+	runCmd.Flags().StringVarP(&configBase64, "base64_config", "", "", "Base64 encoded config")
+	runCmd.Flags().StringVarP(&listenAddress, "listen", "", "", "Listener address and port, in format: address:port")
+	runCmd.Flags().StringVarP(&primaryUpstream, "primary_upstream", "", "", "Primary upstream endpoint")
+	runCmd.Flags().StringVarP(&secondaryUpstream, "secondary_upstream", "", "", "Secondary upstream endpoint")
+	runCmd.Flags().StringSliceVarP(&domains, "domains", "", nil, "List of domain to apply in a split DNS policy")
+	runCmd.Flags().StringVarP(&logPath, "log", "", "", "Path to log file")
+	runCmd.Flags().IntVarP(&cacheSize, "cache_size", "", 0, "Enable cache with size items")
+	runCmd.Flags().StringVarP(&cdUID, "cd", "", "", "Control D resolver uid")
+	runCmd.Flags().StringVarP(&homedir, "homedir", "", "", "")
+	_ = runCmd.Flags().MarkHidden("homedir")
+	runCmd.Flags().StringVarP(&iface, "iface", "", "", `Update DNS setting for iface, "auto" means the default interface gateway`)
+	_ = runCmd.Flags().MarkHidden("iface")
 
 	rootCmd.AddCommand(runCmd)
 
+	startCmd := &cobra.Command{
+		PreRun: checkHasElevatedPrivilege,
+		Use:    "start",
+		Short:  "Install and start the ctrld service",
+		Args:   cobra.NoArgs,
+		Run: func(cmd *cobra.Command, args []string) {
+			sc := &service.Config{}
+			*sc = *svcConfig
+			osArgs := os.Args[2:]
+			if os.Args[1] == "service" {
+				osArgs = os.Args[3:]
+			}
+			setDependencies(sc)
+			sc.Arguments = append([]string{"run"}, osArgs...)
+
+			// No config path, generating config in HOME directory.
+			noConfigStart := isNoConfigStart(cmd)
+			writeDefaultConfig := !noConfigStart && configBase64 == ""
+			if configPath != "" {
+				v.SetConfigFile(configPath)
+			}
+			if dir, err := os.UserHomeDir(); err == nil {
+				setWorkingDirectory(sc, dir)
+				if configPath == "" && writeDefaultConfig {
+					defaultConfigFile = filepath.Join(dir, defaultConfigFile)
+					v.SetConfigFile(defaultConfigFile)
+				}
+				sc.Arguments = append(sc.Arguments, "--homedir="+dir)
+			}
+
+			readConfigFile(writeDefaultConfig && cdUID == "")
+			if err := v.Unmarshal(&cfg); err != nil {
+				log.Fatalf("failed to unmarshal config: %v", err)
+			}
+
+			logPath := cfg.Service.LogPath
+			cfg.Service.LogPath = ""
+			initLogging()
+			cfg.Service.LogPath = logPath
+
+			processCDFlags()
+			// On Windows, the service will be run as SYSTEM, so if ctrld start as Admin,
+			// the user home dir is different, so pass specific arguments that relevant here.
+			if runtime.GOOS == "windows" {
+				if configPath == "" {
+					sc.Arguments = append(sc.Arguments, "--config="+defaultConfigFile)
+				}
+			}
+
+			prog := &prog{}
+			s, err := service.New(prog, sc)
+			if err != nil {
+				stderrMsg(err.Error())
+				return
+			}
+			tasks := []task{
+				{s.Stop, false},
+				{s.Uninstall, false},
+				{s.Install, false},
+				{s.Start, true},
+			}
+			if doTasks(tasks) {
+				status, err := s.Status()
+				if err != nil {
+					mainLog.Warn().Err(err).Msg("could not get service status")
+					return
+				}
+
+				status = selfCheckStatus(status)
+				switch status {
+				case service.StatusRunning:
+					mainLog.Info().Msg("Service started")
+				default:
+					mainLog.Error().Msg("Service did not start, please check system/service log for details error")
+					if runtime.GOOS == "linux" {
+						prog.resetDNS()
+					}
+					os.Exit(1)
+				}
+				prog.setDNS()
+			}
+		},
+	}
+	// Keep these flags in sync with runCmd above, except for "-d".
+	startCmd.Flags().StringVarP(&configPath, "config", "c", "", "Path to config file")
+	startCmd.Flags().StringVarP(&configBase64, "base64_config", "", "", "Base64 encoded config")
+	startCmd.Flags().StringVarP(&listenAddress, "listen", "", "", "Listener address and port, in format: address:port")
+	startCmd.Flags().StringVarP(&primaryUpstream, "primary_upstream", "", "", "Primary upstream endpoint")
+	startCmd.Flags().StringVarP(&secondaryUpstream, "secondary_upstream", "", "", "Secondary upstream endpoint")
+	startCmd.Flags().StringSliceVarP(&domains, "domains", "", nil, "List of domain to apply in a split DNS policy")
+	startCmd.Flags().StringVarP(&logPath, "log", "", "", "Path to log file")
+	startCmd.Flags().IntVarP(&cacheSize, "cache_size", "", 0, "Enable cache with size items")
+	startCmd.Flags().StringVarP(&cdUID, "cd", "", "", "Control D resolver uid")
+	startCmd.Flags().StringVarP(&iface, "iface", "", "", `Update DNS setting for iface, "auto" means the default interface gateway`)
+
+	stopCmd := &cobra.Command{
+		PreRun: checkHasElevatedPrivilege,
+		Use:    "stop",
+		Short:  "Stop the ctrld service",
+		Args:   cobra.NoArgs,
+		Run: func(cmd *cobra.Command, args []string) {
+			prog := &prog{}
+			s, err := service.New(prog, svcConfig)
+			if err != nil {
+				stderrMsg(err.Error())
+				return
+			}
+			initLogging()
+			if doTasks([]task{{s.Stop, true}}) {
+				prog.resetDNS()
+				mainLog.Info().Msg("Service stopped")
+			}
+		},
+	}
+	stopCmd.Flags().StringVarP(&iface, "iface", "", "", `Reset DNS setting for iface, "auto" means the default interface gateway`)
+
+	restartCmd := &cobra.Command{
+		PreRun: checkHasElevatedPrivilege,
+		Use:    "restart",
+		Short:  "Restart the ctrld service",
+		Args:   cobra.NoArgs,
+		Run: func(cmd *cobra.Command, args []string) {
+			s, err := service.New(&prog{}, svcConfig)
+			if err != nil {
+				stderrMsg(err.Error())
+				return
+			}
+			initLogging()
+			if doTasks([]task{{s.Restart, true}}) {
+				stdoutMsg("Service restarted")
+			}
+		},
+	}
+
+	statusCmd := &cobra.Command{
+		Use:   "status",
+		Short: "Show status of the ctrld service",
+		Args:  cobra.NoArgs,
+		Run: func(cmd *cobra.Command, args []string) {
+			s, err := service.New(&prog{}, svcConfig)
+			if err != nil {
+				stderrMsg(err.Error())
+				return
+			}
+			status, err := s.Status()
+			if err != nil {
+				stderrMsg(err.Error())
+				os.Exit(1)
+			}
+			switch status {
+			case service.StatusUnknown:
+				stdoutMsg("Unknown status")
+				os.Exit(2)
+			case service.StatusRunning:
+				stdoutMsg("Service is running")
+				os.Exit(0)
+			case service.StatusStopped:
+				stdoutMsg("Service is stopped")
+				os.Exit(1)
+			}
+		},
+	}
+
+	uninstallCmd := &cobra.Command{
+		PreRun: checkHasElevatedPrivilege,
+		Use:    "uninstall",
+		Short:  "Stop and uninstall the ctrld service",
+		Args:   cobra.NoArgs,
+		Run: func(cmd *cobra.Command, args []string) {
+			prog := &prog{}
+			s, err := service.New(prog, svcConfig)
+			if err != nil {
+				stderrMsg(err.Error())
+				return
+			}
+			tasks := []task{
+				{s.Stop, false},
+				{s.Uninstall, true},
+			}
+			initLogging()
+			if doTasks(tasks) {
+				prog.resetDNS()
+				mainLog.Info().Msg("Service uninstalled")
+				return
+			}
+		},
+	}
+	uninstallCmd.Flags().StringVarP(&iface, "iface", "", "auto", `Reset DNS setting for iface, "auto" means the default interface gateway`)
+
+	listIfacesCmd := &cobra.Command{
+		Use:   "list",
+		Short: "List network interfaces of the host",
+		Args:  cobra.NoArgs,
+		Run: func(cmd *cobra.Command, args []string) {
+			err := interfaces.ForeachInterface(func(i interfaces.Interface, prefixes []netip.Prefix) {
+				fmt.Printf("Index : %d\n", i.Index)
+				fmt.Printf("Name  : %s\n", i.Name)
+				addrs, _ := i.Addrs()
+				for i, ipaddr := range addrs {
+					if i == 0 {
+						fmt.Printf("Addrs : %v\n", ipaddr)
+						continue
+					}
+					fmt.Printf("        %v\n", ipaddr)
+				}
+				for i, dns := range currentDNS(i.Interface) {
+					if i == 0 {
+						fmt.Printf("DNS   : %s\n", dns)
+						continue
+					}
+					fmt.Printf("      : %s\n", dns)
+				}
+				println()
+			})
+			if err != nil {
+				stderrMsg(err.Error())
+			}
+		},
+	}
+	interfacesCmd := &cobra.Command{
+		Use:   "interfaces",
+		Short: "Manage network interfaces",
+		Args:  cobra.OnlyValidArgs,
+		ValidArgs: []string{
+			listIfacesCmd.Use,
+		},
+	}
+	interfacesCmd.AddCommand(listIfacesCmd)
+
+	serviceCmd := &cobra.Command{
+		Use:   "service",
+		Short: "Manage ctrld service",
+		Args:  cobra.OnlyValidArgs,
+		ValidArgs: []string{
+			statusCmd.Use,
+			stopCmd.Use,
+			restartCmd.Use,
+			statusCmd.Use,
+			uninstallCmd.Use,
+			interfacesCmd.Use,
+		},
+	}
+	serviceCmd.AddCommand(startCmd)
+	serviceCmd.AddCommand(stopCmd)
+	serviceCmd.AddCommand(restartCmd)
+	serviceCmd.AddCommand(statusCmd)
+	serviceCmd.AddCommand(uninstallCmd)
+	serviceCmd.AddCommand(interfacesCmd)
+	rootCmd.AddCommand(serviceCmd)
+	startCmdAlias := &cobra.Command{
+		PreRun: checkHasElevatedPrivilege,
+		Use:    "start",
+		Short:  "Quick start service and configure DNS on interface",
+		Run: func(cmd *cobra.Command, args []string) {
+			if !cmd.Flags().Changed("iface") {
+				os.Args = append(os.Args, "--iface="+ifaceStartStop)
+			}
+			iface = ifaceStartStop
+			startCmd.Run(cmd, args)
+		},
+	}
+	startCmdAlias.Flags().StringVarP(&ifaceStartStop, "iface", "", "auto", `Update DNS setting for iface, "auto" means the default interface gateway`)
+	startCmdAlias.Flags().AddFlagSet(startCmd.Flags())
+	rootCmd.AddCommand(startCmdAlias)
+	stopCmdAlias := &cobra.Command{
+		PreRun: checkHasElevatedPrivilege,
+		Use:    "stop",
+		Short:  "Quick stop service and remove DNS from interface",
+		Run: func(cmd *cobra.Command, args []string) {
+			if !cmd.Flags().Changed("iface") {
+				os.Args = append(os.Args, "--iface="+ifaceStartStop)
+			}
+			iface = ifaceStartStop
+			stopCmd.Run(cmd, args)
+		},
+	}
+	stopCmdAlias.Flags().StringVarP(&ifaceStartStop, "iface", "", "auto", `Reset DNS setting for iface, "auto" means the default interface gateway`)
+	stopCmdAlias.Flags().AddFlagSet(stopCmd.Flags())
+	rootCmd.AddCommand(stopCmdAlias)
+
 	if err := rootCmd.Execute(); err != nil {
-		fmt.Println(err)
+		stderrMsg(err.Error())
 		os.Exit(1)
 	}
 }
 
-func writeConfigFile() {
-	c := v.AllSettings()
-	bs, err := toml.Marshal(c)
-	if err != nil {
-		log.Fatalf("unable to marshal config to toml: %v", err)
+func writeConfigFile() error {
+	if cfu := v.ConfigFileUsed(); cfu != "" {
+		defaultConfigFile = cfu
+	} else if configPath != "" {
+		defaultConfigFile = configPath
 	}
-	if err := os.WriteFile("config.toml", bs, 0600); err != nil {
-		log.Printf("failed to write config file: %v\n", err)
+	f, err := os.OpenFile(defaultConfigFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(0o644))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if cdUID != "" {
+		if _, err := f.WriteString("# AUTO-GENERATED VIA CD FLAG - DO NOT MODIFY\n\n"); err != nil {
+			return err
+		}
+	}
+	enc := toml.NewEncoder(f).SetIndentTables(true)
+	if err := enc.Encode(&cfg); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func readConfigFile(writeDefaultConfig bool) bool {
+	// If err == nil, there's a config supplied via `--config`, no default config written.
+	err := v.ReadInConfig()
+	if err == nil {
+		fmt.Println("loading config file from:", v.ConfigFileUsed())
+		defaultConfigFile = v.ConfigFileUsed()
+		v.OnConfigChange(func(in fsnotify.Event) {
+			if err := v.UnmarshalKey("listener", &cfg.Listener); err != nil {
+				log.Printf("failed to unmarshal listener config: %v", err)
+				return
+			}
+		})
+		v.WatchConfig()
+		return true
+	}
+
+	if !writeDefaultConfig {
+		return false
+	}
+
+	// If error is viper.ConfigFileNotFoundError, write default config.
+	if _, ok := err.(viper.ConfigFileNotFoundError); ok {
+		if err := writeConfigFile(); err != nil {
+			log.Fatalf("failed to write default config file: %v", err)
+		} else {
+			fmt.Println("writing default config file to: " + defaultConfigFile)
+		}
+		defaultConfigWritten = true
+		return false
+	}
+	// Otherwise, report fatal error and exit.
+	log.Fatalf("failed to decode config file: %v", err)
+	return false
+}
+
+func readBase64Config() {
+	if configBase64 == "" {
+		return
+	}
+	configStr, err := base64.StdEncoding.DecodeString(configBase64)
+	if err != nil {
+		log.Fatalf("invalid base64 config: %v", err)
+	}
+	if err := v.ReadConfig(bytes.NewReader(configStr)); err != nil {
+		log.Fatalf("failed to read base64 config: %v", err)
 	}
 }
+
+func processNoConfigFlags(noConfigStart bool) {
+	if !noConfigStart {
+		return
+	}
+	if listenAddress == "" || primaryUpstream == "" {
+		log.Fatal(`"listen" and "primary_upstream" flags must be set in no config mode`)
+	}
+	processListenFlag()
+
+	upstream := map[string]*ctrld.UpstreamConfig{
+		"0": {
+			Name:     primaryUpstream,
+			Endpoint: primaryUpstream,
+			Type:     ctrld.ResolverTypeDOH,
+		},
+	}
+	if secondaryUpstream != "" {
+		upstream["1"] = &ctrld.UpstreamConfig{
+			Name:     secondaryUpstream,
+			Endpoint: secondaryUpstream,
+			Type:     ctrld.ResolverTypeLegacy,
+		}
+		rules := make([]ctrld.Rule, 0, len(domains))
+		for _, domain := range domains {
+			rules = append(rules, ctrld.Rule{domain: []string{"upstream.1"}})
+		}
+		lc := v.Get("listener").(map[string]*ctrld.ListenerConfig)["0"]
+		lc.Policy = &ctrld.ListenerPolicyConfig{Name: "My Policy", Rules: rules}
+	}
+	v.Set("upstream", upstream)
+}
+
+func processCDFlags() {
+	if cdUID == "" {
+		return
+	}
+	if iface == "" {
+		iface = "auto"
+	}
+	logger := mainLog.With().Str("mode", "cd").Logger()
+	logger.Info().Msgf("fetching Controld D configuration from API: %s", cdUID)
+	resolverConfig, err := controld.FetchResolverConfig(cdUID)
+	if uer, ok := err.(*controld.UtilityErrorResponse); ok && uer.ErrorField.Code == controld.InvalidConfigCode {
+		s, err := service.New(&prog{}, svcConfig)
+		if err != nil {
+			logger.Warn().Err(err).Msg("failed to create new service")
+			return
+		}
+
+		if netIface, _ := netInterface(iface); netIface != nil {
+			if err := restoreNetworkManager(); err != nil {
+				logger.Error().Err(err).Msg("could not restore NetworkManager")
+				return
+			}
+			logger.Debug().Str("iface", netIface.Name).Msg("Restoring DNS for interface")
+			if err := resetDNS(netIface); err != nil {
+				logger.Warn().Err(err).Msg("something went wrong while restoring DNS")
+			} else {
+				logger.Debug().Str("iface", netIface.Name).Msg("Restoring DNS successfully")
+			}
+		}
+
+		tasks := []task{{s.Uninstall, true}}
+		if doTasks(tasks) {
+			logger.Info().Msg("uninstalled service")
+		}
+		logger.Fatal().Err(uer).Msg("failed to fetch resolver config")
+	}
+	if err != nil {
+		logger.Warn().Err(err).Msg("could not fetch resolver config")
+		return
+	}
+
+	logger.Info().Msg("generating ctrld config from Controld-D configuration")
+	cfg = ctrld.Config{}
+	cfg.Network = make(map[string]*ctrld.NetworkConfig)
+	cfg.Network["0"] = &ctrld.NetworkConfig{
+		Name:  "Network 0",
+		Cidrs: []string{"0.0.0.0/0"},
+	}
+	cfg.Upstream = make(map[string]*ctrld.UpstreamConfig)
+	cfg.Upstream["0"] = &ctrld.UpstreamConfig{
+		Endpoint: resolverConfig.DOH,
+		Type:     ctrld.ResolverTypeDOH,
+		Timeout:  5000,
+	}
+	rules := make([]ctrld.Rule, 0, len(resolverConfig.Exclude))
+	for _, domain := range resolverConfig.Exclude {
+		rules = append(rules, ctrld.Rule{domain: []string{}})
+	}
+	cfg.Listener = make(map[string]*ctrld.ListenerConfig)
+	cfg.Listener["0"] = &ctrld.ListenerConfig{
+		IP:   "127.0.0.1",
+		Port: 53,
+		Policy: &ctrld.ListenerPolicyConfig{
+			Name:  "My Policy",
+			Rules: rules,
+		},
+	}
+
+	processLogAndCacheFlags()
+	if err := writeConfigFile(); err != nil {
+		logger.Fatal().Err(err).Msg("failed to write config file")
+	} else {
+		logger.Info().Msg("writing config file to: " + defaultConfigFile)
+	}
+}
+
+func processListenFlag() {
+	if listenAddress == "" {
+		return
+	}
+	host, portStr, err := net.SplitHostPort(listenAddress)
+	if err != nil {
+		log.Fatalf("invalid listener address: %v", err)
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		log.Fatalf("invalid port number: %v", err)
+	}
+	lc := &ctrld.ListenerConfig{
+		IP:   host,
+		Port: port,
+	}
+	v.Set("listener", map[string]*ctrld.ListenerConfig{
+		"0": lc,
+	})
+}
+
+func processLogAndCacheFlags() {
+	if logPath != "" {
+		cfg.Service.LogLevel = "debug"
+		cfg.Service.LogPath = logPath
+	}
+
+	if cacheSize != 0 {
+		cfg.Service.CacheEnable = true
+		cfg.Service.CacheSize = cacheSize
+	}
+	v.Set("service", cfg.Service)
+}
+
+func netInterface(ifaceName string) (*net.Interface, error) {
+	if ifaceName == "auto" {
+		ifaceName = defaultIfaceName()
+	}
+	var iface *net.Interface
+	err := interfaces.ForeachInterface(func(i interfaces.Interface, prefixes []netip.Prefix) {
+		if i.Name == ifaceName {
+			iface = i.Interface
+		}
+	})
+	if iface == nil {
+		return nil, errors.New("interface not found")
+	}
+	if err := patchNetIfaceName(iface); err != nil {
+		return nil, err
+	}
+	return iface, err
+}
+
+func defaultIfaceName() string {
+	dri, err := interfaces.DefaultRouteInterface()
+	if err != nil {
+		mainLog.Fatal().Err(err).Msg("failed to get default route interface")
+	}
+	return dri
+}
+
+func selfCheckStatus(status service.Status) service.Status {
+	c := new(dns.Client)
+	bo := backoff.NewBackoff("self-check", logf, 10*time.Second)
+	bo.LogLongerThan = 500 * time.Millisecond
+	ctx := context.Background()
+	err := errors.New("query failed")
+	maxAttempts := 20
+	mainLog.Debug().Msg("Performing self-check")
+	for i := 0; i < maxAttempts; i++ {
+		lc := cfg.Listener["0"]
+		m := new(dns.Msg)
+		m.SetQuestion(selfCheckFQDN+".", dns.TypeA)
+		m.RecursionDesired = true
+		r, _, _ := c.ExchangeContext(ctx, m, net.JoinHostPort(lc.IP, strconv.Itoa(lc.Port)))
+		if r != nil && r.Rcode == dns.RcodeSuccess && len(r.Answer) > 0 {
+			mainLog.Debug().Msgf("self-check against %q succeeded", selfCheckFQDN)
+			return status
+		}
+		bo.BackOff(ctx, err)
+	}
+	mainLog.Debug().Msgf("self-check against %q failed", selfCheckFQDN)
+	return service.StatusUnknown
+}

cmd/ctrld/cli_test.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_writeConfigFile(t *testing.T) {
+	tmpdir := t.TempDir()
+	// simulate --config CLI flag by setting configPath manually.
+	configPath = filepath.Join(tmpdir, "ctrld.toml")
+	_, err := os.Stat(configPath)
+	assert.True(t, os.IsNotExist(err))
+
+	assert.NoError(t, writeConfigFile())
+
+	_, err = os.Stat(configPath)
+	require.NoError(t, err)
+}

cmd/ctrld/dns.go

@@ -0,0 +1,4 @@
+package main
+
+//lint:ignore U1000 use in os_linux.go
+type getDNS func(iface string) []string

cmd/ctrld/dns_proxy.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"net"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -13,8 +14,12 @@ import (
 	"github.com/miekg/dns"
 
 	"github.com/Control-D-Inc/ctrld"
+	"github.com/Control-D-Inc/ctrld/internal/dnscache"
+	ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
 )
 
+const staleTTL = 60 * time.Second
+
 func (p *prog) serveUDP(listenerNum string) error {
 	listenerConfig := p.cfg.Listener[listenerNum]
 	// make sure ip is allocated
@@ -22,14 +27,18 @@ func (p *prog) serveUDP(listenerNum string) error {
 		mainLog.Error().Err(allocErr).Str("ip", listenerConfig.IP).Msg("serveUDP: failed to allocate listen ip")
 		return allocErr
 	}
-
+	var failoverRcodes []int
+	if listenerConfig.Policy != nil {
+		failoverRcodes = listenerConfig.Policy.FailoverRcodeNumbers
+	}
 	handler := dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) {
-		domain := canonicalName(m.Question[0].Name)
+		q := m.Question[0]
+		domain := canonicalName(q.Name)
 		reqId := requestID()
 		fmtSrcToDest := fmtRemoteToLocal(listenerNum, w.RemoteAddr().String(), w.LocalAddr().String())
 		t := time.Now()
 		ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, reqId)
-		ctrld.Log(ctx, proxyLog.Debug(), "%s received query: %s", fmtSrcToDest, domain)
+		ctrld.Log(ctx, mainLog.Debug(), "%s received query: %s %s", fmtSrcToDest, dns.TypeToString[q.Qtype], domain)
 		upstreams, matched := p.upstreamFor(ctx, listenerNum, listenerConfig, w.RemoteAddr(), domain)
 		var answer *dns.Msg
 		if !matched && listenerConfig.Restricted {
@@ -37,14 +46,30 @@ func (p *prog) serveUDP(listenerNum string) error {
 			answer.SetRcode(m, dns.RcodeRefused)
 
 		} else {
-			answer = p.proxy(ctx, upstreams, m)
+			answer = p.proxy(ctx, upstreams, failoverRcodes, m)
 			rtt := time.Since(t)
-			ctrld.Log(ctx, proxyLog.Debug(), "received response of %d bytes in %s", answer.Len(), rtt)
+			ctrld.Log(ctx, mainLog.Debug(), "received response of %d bytes in %s", answer.Len(), rtt)
 		}
 		if err := w.WriteMsg(answer); err != nil {
 			ctrld.Log(ctx, mainLog.Error().Err(err), "serveUDP: failed to send DNS response to client")
 		}
 	})
+
+	// On Windows, there's no easy way for disabling/removing IPv6 DNS resolver, so we check whether we can
+	// listen on ::1, then spawn a listener for receiving DNS requests.
+	if runtime.GOOS == "windows" && ctrldnet.SupportsIPv6ListenLocal() {
+		go func() {
+			s := &dns.Server{
+				Addr:    net.JoinHostPort("::1", strconv.Itoa(listenerConfig.Port)),
+				Net:     "udp",
+				Handler: handler,
+			}
+			if err := s.ListenAndServe(); err != nil {
+				mainLog.Error().Err(err).Msg("could not serving on ::1")
+			}
+		}()
+	}
+
 	s := &dns.Server{
 		Addr:    net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port)),
 		Net:     "udp",
@@ -62,10 +87,10 @@ func (p *prog) upstreamFor(ctx context.Context, defaultUpstreamNum string, lc *c
 
 	defer func() {
 		if !matched && lc.Restricted {
-			ctrld.Log(ctx, proxyLog.Info(), "query refused, %s does not match any network policy", addr.String())
+			ctrld.Log(ctx, mainLog.Info(), "query refused, %s does not match any network policy", addr.String())
 			return
 		}
-		ctrld.Log(ctx, proxyLog.Info(), "%s, %s, %s -> %v", matchedPolicy, matchedNetwork, matchedRule, upstreams)
+		ctrld.Log(ctx, mainLog.Info(), "%s, %s, %s -> %v", matchedPolicy, matchedNetwork, matchedRule, upstreams)
 	}()
 
 	if lc.Policy == nil {
@@ -119,38 +144,106 @@ func (p *prog) upstreamFor(ctx context.Context, defaultUpstreamNum string, lc *c
 	return upstreams, matched
 }
 
-func (p *prog) proxy(ctx context.Context, upstreams []string, msg *dns.Msg) *dns.Msg {
+func (p *prog) proxy(ctx context.Context, upstreams []string, failoverRcodes []int, msg *dns.Msg) *dns.Msg {
+	var staleAnswer *dns.Msg
+	serveStaleCache := p.cache != nil && p.cfg.Service.CacheServeStale
 	upstreamConfigs := p.upstreamConfigsFromUpstreamNumbers(upstreams)
-	resolve := func(n int, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) *dns.Msg {
-		ctrld.Log(ctx, proxyLog.Debug(), "sending query to %s: %s", upstreams[n], upstreamConfig.Name)
+	if len(upstreamConfigs) == 0 {
+		upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig}
+		upstreams = []string{"upstream.os"}
+	}
+	// Inverse query should not be cached: https://www.rfc-editor.org/rfc/rfc1035#section-7.4
+	if p.cache != nil && msg.Question[0].Qtype != dns.TypePTR {
+		for _, upstream := range upstreams {
+			cachedValue := p.cache.Get(dnscache.NewKey(msg, upstream))
+			if cachedValue == nil {
+				continue
+			}
+			answer := cachedValue.Msg.Copy()
+			answer.SetRcode(msg, answer.Rcode)
+			now := time.Now()
+			if cachedValue.Expire.After(now) {
+				ctrld.Log(ctx, mainLog.Debug(), "hit cached response")
+				setCachedAnswerTTL(answer, now, cachedValue.Expire)
+				return answer
+			}
+			staleAnswer = answer
+		}
+	}
+	resolve1 := func(n int, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) (*dns.Msg, error) {
+		ctrld.Log(ctx, mainLog.Debug(), "sending query to %s: %s", upstreams[n], upstreamConfig.Name)
 		dnsResolver, err := ctrld.NewResolver(upstreamConfig)
 		if err != nil {
-			ctrld.Log(ctx, proxyLog.Error().Err(err), "failed to create resolver")
-			return nil
+			ctrld.Log(ctx, mainLog.Error().Err(err), "failed to create resolver")
+			return nil, err
 		}
+		resolveCtx, cancel := context.WithCancel(ctx)
+		defer cancel()
 		if upstreamConfig.Timeout > 0 {
-			timeoutCtx, cancel := context.WithTimeout(ctx, time.Millisecond*time.Duration(upstreamConfig.Timeout))
+			timeoutCtx, cancel := context.WithTimeout(resolveCtx, time.Millisecond*time.Duration(upstreamConfig.Timeout))
 			defer cancel()
-			ctx = timeoutCtx
+			resolveCtx = timeoutCtx
 		}
-		answer, err := dnsResolver.Resolve(ctx, msg)
+		return dnsResolver.Resolve(resolveCtx, msg)
+	}
+	resolve := func(n int, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) *dns.Msg {
+		answer, err := resolve1(n, upstreamConfig, msg)
 		if err != nil {
-			ctrld.Log(ctx, proxyLog.Error().Err(err), "failed to resolve query")
+			ctrld.Log(ctx, mainLog.Debug().Err(err), "could not resolve query on first attempt, retrying...")
+			// If any error occurred, re-bootstrap transport/ip, retry the request.
+			upstreamConfig.ReBootstrap()
+			answer, err = resolve1(n, upstreamConfig, msg)
+			if err == nil {
+				return answer
+			}
+			ctrld.Log(ctx, mainLog.Error().Err(err), "failed to resolve query")
 			return nil
 		}
 		return answer
 	}
 	for n, upstreamConfig := range upstreamConfigs {
-		if answer := resolve(n, upstreamConfig, msg); answer != nil {
-			return answer
+		answer := resolve(n, upstreamConfig, msg)
+		if answer == nil {
+			if serveStaleCache && staleAnswer != nil {
+				ctrld.Log(ctx, mainLog.Debug(), "serving stale cached response")
+				now := time.Now()
+				setCachedAnswerTTL(staleAnswer, now, now.Add(staleTTL))
+				return staleAnswer
+			}
+			continue
 		}
+		if answer.Rcode != dns.RcodeSuccess && len(upstreamConfigs) > 1 && containRcode(failoverRcodes, answer.Rcode) {
+			ctrld.Log(ctx, mainLog.Debug(), "failover rcode matched, process to next upstream")
+			continue
+		}
+		if p.cache != nil {
+			ttl := ttlFromMsg(answer)
+			now := time.Now()
+			expired := now.Add(time.Duration(ttl) * time.Second)
+			if cachedTTL := p.cfg.Service.CacheTTLOverride; cachedTTL > 0 {
+				expired = now.Add(time.Duration(cachedTTL) * time.Second)
+			}
+			setCachedAnswerTTL(answer, now, expired)
+			p.cache.Add(dnscache.NewKey(msg, upstreams[n]), dnscache.NewValue(answer, expired))
+			ctrld.Log(ctx, mainLog.Debug(), "add cached response")
+		}
+		return answer
 	}
-	ctrld.Log(ctx, proxyLog.Error(), "all upstreams failed")
+	ctrld.Log(ctx, mainLog.Error(), "all upstreams failed")
 	answer := new(dns.Msg)
 	answer.SetRcode(msg, dns.RcodeServerFailure)
 	return answer
 }
 
+func (p *prog) upstreamConfigsFromUpstreamNumbers(upstreams []string) []*ctrld.UpstreamConfig {
+	upstreamConfigs := make([]*ctrld.UpstreamConfig, 0, len(upstreams))
+	for _, upstream := range upstreams {
+		upstreamNum := strings.TrimPrefix(upstream, "upstream.")
+		upstreamConfigs = append(upstreamConfigs, p.cfg.Upstream[upstreamNum])
+	}
+	return upstreamConfigs
+}
+
 // canonicalName returns canonical name from FQDN with "." trimmed.
 func canonicalName(fqdn string) string {
 	q := strings.TrimSpace(fqdn)
@@ -189,18 +282,6 @@ func fmtRemoteToLocal(listenerNum, remote, local string) string {
 	return fmt.Sprintf("%s -> listener.%s: %s:", remote, listenerNum, local)
 }
 
-func (p *prog) upstreamConfigsFromUpstreamNumbers(upstreams []string) []*ctrld.UpstreamConfig {
-	upstreamConfigs := make([]*ctrld.UpstreamConfig, 0, len(upstreams))
-	for _, upstream := range upstreams {
-		upstreamNum := strings.TrimPrefix(upstream, "upstream.")
-		upstreamConfigs = append(upstreamConfigs, p.cfg.Upstream[upstreamNum])
-	}
-	if len(upstreamConfigs) == 0 {
-		upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig}
-	}
-	return upstreamConfigs
-}
-
 func requestID() string {
 	b := make([]byte, 3) // 6 chars
 	if _, err := rand.Read(b); err != nil {
@@ -209,7 +290,46 @@ func requestID() string {
 	return hex.EncodeToString(b)
 }
 
+func containRcode(rcodes []int, rcode int) bool {
+	for i := range rcodes {
+		if rcodes[i] == rcode {
+			return true
+		}
+	}
+	return false
+}
+
+func setCachedAnswerTTL(answer *dns.Msg, now, expiredTime time.Time) {
+	ttlSecs := expiredTime.Sub(now).Seconds()
+	if ttlSecs < 0 {
+		return
+	}
+
+	ttl := uint32(ttlSecs)
+	for _, rr := range answer.Answer {
+		rr.Header().Ttl = ttl
+	}
+	for _, rr := range answer.Ns {
+		rr.Header().Ttl = ttl
+	}
+	for _, rr := range answer.Extra {
+		if rr.Header().Rrtype != dns.TypeOPT {
+			rr.Header().Ttl = ttl
+		}
+	}
+}
+
+func ttlFromMsg(msg *dns.Msg) uint32 {
+	for _, rr := range msg.Answer {
+		return rr.Header().Ttl
+	}
+	for _, rr := range msg.Ns {
+		return rr.Header().Ttl
+	}
+	return 0
+}
+
 var osUpstreamConfig = &ctrld.UpstreamConfig{
 	Name: "OS resolver",
-	Type: "os",
+	Type: ctrld.ResolverTypeOS,
 }

cmd/ctrld/dns_proxy_test.go

@@ -4,11 +4,14 @@ import (
 	"context"
 	"net"
 	"testing"
+	"time"
 
+	"github.com/miekg/dns"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/Control-D-Inc/ctrld"
+	"github.com/Control-D-Inc/ctrld/internal/dnscache"
 	"github.com/Control-D-Inc/ctrld/testhelper"
 )
 
@@ -115,3 +118,37 @@ func Test_prog_upstreamFor(t *testing.T) {
 		})
 	}
 }
+
+func TestCache(t *testing.T) {
+	cfg := testhelper.SampleConfig(t)
+	prog := &prog{cfg: cfg}
+	for _, nc := range prog.cfg.Network {
+		for _, cidr := range nc.Cidrs {
+			_, ipNet, err := net.ParseCIDR(cidr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			nc.IPNets = append(nc.IPNets, ipNet)
+		}
+	}
+	cacher, err := dnscache.NewLRUCache(4096)
+	require.NoError(t, err)
+	prog.cache = cacher
+
+	msg := new(dns.Msg)
+	msg.SetQuestion("example.com", dns.TypeA)
+	msg.MsgHdr.RecursionDesired = true
+	answer1 := new(dns.Msg)
+	answer1.SetRcode(msg, dns.RcodeSuccess)
+
+	prog.cache.Add(dnscache.NewKey(msg, "upstream.1"), dnscache.NewValue(answer1, time.Now().Add(time.Minute)))
+	answer2 := new(dns.Msg)
+	answer2.SetRcode(msg, dns.RcodeRefused)
+	prog.cache.Add(dnscache.NewKey(msg, "upstream.0"), dnscache.NewValue(answer2, time.Now().Add(time.Minute)))
+
+	got1 := prog.proxy(context.Background(), []string{"upstream.1"}, nil, msg)
+	got2 := prog.proxy(context.Background(), []string{"upstream.0"}, nil, msg)
+	assert.NotSame(t, got1, got2)
+	assert.Equal(t, answer1.Rcode, got1.Rcode)
+	assert.Equal(t, answer2.Rcode, got2.Rcode)
+}

cmd/ctrld/main.go

@@ -1,65 +1,109 @@
 package main
 
 import (
-	"fmt"
 	"io"
+	"log"
 	"os"
+	"path/filepath"
 	"time"
 
+	"github.com/kardianos/service"
 	"github.com/rs/zerolog"
 
 	"github.com/Control-D-Inc/ctrld"
 )
 
 var (
-	configPath string
-	daemon     bool
-	cfg        ctrld.Config
-	verbose    bool
-
-	bootstrapDNS = "76.76.2.0"
+	configPath        string
+	configBase64      string
+	daemon            bool
+	listenAddress     string
+	primaryUpstream   string
+	secondaryUpstream string
+	domains           []string
+	logPath           string
+	homedir           string
+	cacheSize         int
+	cfg               ctrld.Config
+	verbose           int
 
 	rootLogger = zerolog.New(io.Discard)
 	mainLog    = rootLogger
-	proxyLog   = rootLogger
+
+	cdUID          string
+	iface          string
+	ifaceStartStop string
 )
 
 func main() {
-	ctrld.InitConfig(v, "config")
+	ctrld.InitConfig(v, "ctrld")
 	initCLI()
 }
 
+func normalizeLogFilePath(logFilePath string) string {
+	if logFilePath == "" || filepath.IsAbs(logFilePath) || service.Interactive() {
+		return logFilePath
+	}
+	if homedir != "" {
+		return filepath.Join(homedir, logFilePath)
+	}
+	dir, _ := os.UserHomeDir()
+	if dir == "" {
+		return logFilePath
+	}
+	return filepath.Join(dir, logFilePath)
+}
+
 func initLogging() {
 	writers := []io.Writer{io.Discard}
-	isLog := cfg.Service.LogLevel != ""
-	if logPath := cfg.Service.LogPath; logPath != "" {
-		logFile, err := os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0600)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "failed to creating log file: %v", err)
+	if logFilePath := normalizeLogFilePath(cfg.Service.LogPath); logFilePath != "" {
+		// Create parent directory if necessary.
+		if err := os.MkdirAll(filepath.Dir(logFilePath), 0750); err != nil {
+			log.Printf("failed to create log path: %v", err)
+			os.Exit(1)
+		}
+		// Backup old log file with .1 suffix.
+		if err := os.Rename(logFilePath, logFilePath+".1"); err != nil && !os.IsNotExist(err) {
+			log.Printf("could not backup old log file: %v", err)
+		}
+		logFile, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_RDWR, os.FileMode(0o600))
+		if err != nil {
+			log.Printf("failed to create log file: %v", err)
 			os.Exit(1)
 		}
-		isLog = true
 		writers = append(writers, logFile)
 	}
 	zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs
-	if verbose || isLog {
-		consoleWriter := zerolog.NewConsoleWriter(func(w *zerolog.ConsoleWriter) {
-			w.TimeFormat = time.StampMilli
-		})
-		writers = append(writers, consoleWriter)
-		multi := zerolog.MultiLevelWriter(writers...)
-		mainLog = mainLog.Output(multi).With().Timestamp().Str("prefix", "main").Logger()
-		proxyLog = proxyLog.Output(multi).With().Timestamp().Logger()
-		// TODO: find a better way.
-		ctrld.ProxyLog = proxyLog
+	consoleWriter := zerolog.NewConsoleWriter(func(w *zerolog.ConsoleWriter) {
+		w.TimeFormat = time.StampMilli
+	})
+	writers = append(writers, consoleWriter)
+	multi := zerolog.MultiLevelWriter(writers...)
+	mainLog = mainLog.Output(multi).With().Timestamp().Logger()
+	// TODO: find a better way.
+	ctrld.ProxyLog = mainLog
+
+	zerolog.SetGlobalLevel(zerolog.InfoLevel)
+	logLevel := cfg.Service.LogLevel
+	if verbose > 1 {
+		logLevel = "debug"
 	}
-	if cfg.Service.LogLevel == "" {
+	if logLevel == "" {
 		return
 	}
-	level, err := zerolog.ParseLevel(cfg.Service.LogLevel)
+	level, err := zerolog.ParseLevel(logLevel)
 	if err != nil {
 		mainLog.Warn().Err(err).Msg("could not set log level")
 		return
 	}
 	zerolog.SetGlobalLevel(level)
 }
+
+func initCache() {
+	if !cfg.Service.CacheEnable {
+		return
+	}
+	if cfg.Service.CacheSize == 0 {
+		cfg.Service.CacheSize = 4096
+	}
+}

cmd/ctrld/net_darwin.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"net"
+	"os/exec"
+	"strings"
+)
+
+func patchNetIfaceName(iface *net.Interface) error {
+	b, err := exec.Command("networksetup", "-listnetworkserviceorder").Output()
+	if err != nil {
+		return err
+	}
+
+	if name := networkServiceName(iface.Name, bytes.NewReader(b)); name != "" {
+		iface.Name = name
+		mainLog.Debug().Str("network_service", name).Msg("found network service name for interface")
+	}
+	return nil
+}
+
+func networkServiceName(ifaceName string, r io.Reader) string {
+	scanner := bufio.NewScanner(r)
+	prevLine := ""
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.Contains(line, "*") {
+			// Network services is disabled.
+			continue
+		}
+		if !strings.Contains(line, "Device: "+ifaceName) {
+			prevLine = line
+			continue
+		}
+		parts := strings.SplitN(prevLine, " ", 2)
+		if len(parts) == 2 {
+			return strings.TrimSpace(parts[1])
+		}
+	}
+	return ""
+}

cmd/ctrld/net_darwin_test.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const listnetworkserviceorderOutput = `
+(1) USB 10/100/1000 LAN 2
+(Hardware Port: USB 10/100/1000 LAN, Device: en7)
+
+(2) Ethernet
+(Hardware Port: Ethernet, Device: en0)
+
+(3) Wi-Fi
+(Hardware Port: Wi-Fi, Device: en1)
+
+(4) Bluetooth PAN
+(Hardware Port: Bluetooth PAN, Device: en4)
+
+(5) Thunderbolt Bridge
+(Hardware Port: Thunderbolt Bridge, Device: bridge0)
+
+(6) kernal
+(Hardware Port: com.wireguard.macos, Device: )
+
+(7) WS BT
+(Hardware Port: com.wireguard.macos, Device: )
+
+(8) ca-001-stg
+(Hardware Port: com.wireguard.macos, Device: )
+
+(9) ca-001-stg-2
+(Hardware Port: com.wireguard.macos, Device: )
+
+`
+
+func Test_networkServiceName(t *testing.T) {
+	tests := []struct {
+		ifaceName          string
+		networkServiceName string
+	}{
+		{"en7", "USB 10/100/1000 LAN 2"},
+		{"en0", "Ethernet"},
+		{"en1", "Wi-Fi"},
+		{"en4", "Bluetooth PAN"},
+		{"bridge0", "Thunderbolt Bridge"},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.ifaceName, func(t *testing.T) {
+			t.Parallel()
+			name := networkServiceName(tc.ifaceName, strings.NewReader(listnetworkserviceorderOutput))
+			assert.Equal(t, tc.networkServiceName, name)
+		})
+	}
+}

cmd/ctrld/net_others.go

@@ -0,0 +1,7 @@
+//go:build !darwin
+
+package main
+
+import "net"
+
+func patchNetIfaceName(iface *net.Interface) error { return nil }

cmd/ctrld/network_manager_linux.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/coreos/go-systemd/v22/dbus"
+)
+
+const (
+	nmConfDir           = "/etc/NetworkManager/conf.d"
+	nmCtrldConfFilename = "99-ctrld.conf"
+	nmCtrldConfContent  = `[main]
+dns=none
+systemd-resolved=false
+`
+	nmSystemdUnitName   = "NetworkManager.service"
+	systemdEnabledState = "enabled"
+)
+
+var networkManagerCtrldConfFile = filepath.Join(nmConfDir, nmCtrldConfFilename)
+
+func setupNetworkManager() error {
+	if content, _ := os.ReadFile(nmCtrldConfContent); string(content) == nmCtrldConfContent {
+		mainLog.Debug().Msg("NetworkManager already setup, nothing to do")
+		return nil
+	}
+	err := os.WriteFile(networkManagerCtrldConfFile, []byte(nmCtrldConfContent), os.FileMode(0644))
+	if os.IsNotExist(err) {
+		mainLog.Debug().Msg("NetworkManager is not available")
+		return nil
+	}
+	if err != nil {
+		mainLog.Debug().Err(err).Msg("could not write NetworkManager ctrld config file")
+		return err
+	}
+
+	reloadNetworkManager()
+	mainLog.Debug().Msg("setup NetworkManager done")
+	return nil
+}
+
+func restoreNetworkManager() error {
+	err := os.Remove(networkManagerCtrldConfFile)
+	if os.IsNotExist(err) {
+		mainLog.Debug().Msg("NetworkManager is not available")
+		return nil
+	}
+	if err != nil {
+		mainLog.Debug().Err(err).Msg("could not remove NetworkManager ctrld config file")
+		return err
+	}
+
+	reloadNetworkManager()
+	mainLog.Debug().Msg("restore NetworkManager done")
+	return nil
+}
+
+func reloadNetworkManager() {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+	conn, err := dbus.NewSystemConnectionContext(ctx)
+	if err != nil {
+		mainLog.Error().Err(err).Msg("could not create new system connection")
+		return
+	}
+	defer conn.Close()
+
+	waitCh := make(chan string)
+	if _, err := conn.ReloadUnitContext(ctx, nmSystemdUnitName, "ignore-dependencies", waitCh); err != nil {
+		mainLog.Debug().Err(err).Msg("could not reload NetworkManager")
+	}
+	<-waitCh
+}

cmd/ctrld/network_manager_others.go

@@ -0,0 +1,15 @@
+//go:build !linux
+
+package main
+
+func setupNetworkManager() error {
+	reloadNetworkManager()
+	return nil
+}
+
+func restoreNetworkManager() error {
+	reloadNetworkManager()
+	return nil
+}
+
+func reloadNetworkManager() {}

cmd/ctrld/os_darwin.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"net"
+	"os/exec"
+
+	"github.com/Control-D-Inc/ctrld/internal/resolvconffile"
+)
+
+// allocate loopback ip
+// sudo ifconfig lo0 alias 127.0.0.2 up
+func allocateIP(ip string) error {
+	cmd := exec.Command("ifconfig", "lo0", "alias", ip, "up")
+	if err := cmd.Run(); err != nil {
+		mainLog.Error().Err(err).Msg("allocateIP failed")
+		return err
+	}
+	return nil
+}
+
+func deAllocateIP(ip string) error {
+	cmd := exec.Command("ifconfig", "lo0", "-alias", ip)
+	if err := cmd.Run(); err != nil {
+		mainLog.Error().Err(err).Msg("deAllocateIP failed")
+		return err
+	}
+	return nil
+}
+
+// set the dns server for the provided network interface
+// networksetup -setdnsservers Wi-Fi 8.8.8.8 1.1.1.1
+// TODO(cuonglm): use system API
+func setDNS(iface *net.Interface, nameservers []string) error {
+	cmd := "networksetup"
+	args := []string{"-setdnsservers", iface.Name}
+	args = append(args, nameservers...)
+
+	if err := exec.Command(cmd, args...).Run(); err != nil {
+		mainLog.Error().Err(err).Msgf("setDNS failed, ips = %q", nameservers)
+		return err
+	}
+	return nil
+}
+
+// TODO(cuonglm): use system API
+func resetDNS(iface *net.Interface) error {
+	cmd := "networksetup"
+	args := []string{"-setdnsservers", iface.Name, "empty"}
+
+	if err := exec.Command(cmd, args...).Run(); err != nil {
+		mainLog.Error().Err(err).Msgf("resetDNS failed")
+		return err
+	}
+	return nil
+}
+
+func currentDNS(_ *net.Interface) []string {
+	return resolvconffile.NameServers("")
+}

cmd/ctrld/os_freebsd.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"net"
+	"net/netip"
+	"os/exec"
+
+	"github.com/Control-D-Inc/ctrld/internal/dns"
+	"github.com/Control-D-Inc/ctrld/internal/resolvconffile"
+)
+
+// allocate loopback ip
+// sudo ifconfig lo0 127.0.0.53 alias
+func allocateIP(ip string) error {
+	cmd := exec.Command("ifconfig", "lo0", ip, "alias")
+	if err := cmd.Run(); err != nil {
+		mainLog.Error().Err(err).Msg("allocateIP failed")
+		return err
+	}
+	return nil
+}
+
+func deAllocateIP(ip string) error {
+	cmd := exec.Command("ifconfig", "lo0", ip, "-alias")
+	if err := cmd.Run(); err != nil {
+		mainLog.Error().Err(err).Msg("deAllocateIP failed")
+		return err
+	}
+	return nil
+}
+
+// set the dns server for the provided network interface
+func setDNS(iface *net.Interface, nameservers []string) error {
+	r, err := dns.NewOSConfigurator(logf, iface.Name)
+	if err != nil {
+		mainLog.Error().Err(err).Msg("failed to create DNS OS configurator")
+		return err
+	}
+
+	ns := make([]netip.Addr, 0, len(nameservers))
+	for _, nameserver := range nameservers {
+		ns = append(ns, netip.MustParseAddr(nameserver))
+	}
+
+	if err := r.SetDNS(dns.OSConfig{Nameservers: ns}); err != nil {
+		mainLog.Error().Err(err).Msg("failed to set DNS")
+		return err
+	}
+	return nil
+}
+
+func resetDNS(iface *net.Interface) error {
+	r, err := dns.NewOSConfigurator(logf, iface.Name)
+	if err != nil {
+		mainLog.Error().Err(err).Msg("failed to create DNS OS configurator")
+		return err
+	}
+
+	if err := r.Close(); err != nil {
+		mainLog.Error().Err(err).Msg("failed to rollback DNS setting")
+		return err
+	}
+	return nil
+}
+
+func currentDNS(_ *net.Interface) []string {
+	return resolvconffile.NameServers("")
+}

cmd/ctrld/os_linux.go

@@ -1,15 +1,34 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
 	"os/exec"
+	"reflect"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
+	"github.com/insomniacslk/dhcp/dhcpv6"
+	"github.com/insomniacslk/dhcp/dhcpv6/client6"
+	"tailscale.com/util/dnsname"
+
+	"github.com/Control-D-Inc/ctrld/internal/dns"
+	ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
+	"github.com/Control-D-Inc/ctrld/internal/resolvconffile"
 )
 
 // allocate loopback ip
 // sudo ip a add 127.0.0.2/24 dev lo
 func allocateIP(ip string) error {
 	cmd := exec.Command("ip", "a", "add", ip+"/24", "dev", "lo")
-	if err := cmd.Run(); err != nil {
-		mainLog.Error().Err(err).Msg("allocateIP failed")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		mainLog.Error().Err(err).Msgf("allocateIP failed: %s", string(out))
 		return err
 	}
 	return nil
@@ -23,3 +42,154 @@ func deAllocateIP(ip string) error {
 	}
 	return nil
 }
+
+const maxSetDNSAttempts = 5
+
+// set the dns server for the provided network interface
+func setDNS(iface *net.Interface, nameservers []string) error {
+	r, err := dns.NewOSConfigurator(logf, iface.Name)
+	if err != nil {
+		mainLog.Error().Err(err).Msg("failed to create DNS OS configurator")
+		return err
+	}
+
+	ns := make([]netip.Addr, 0, len(nameservers))
+	for _, nameserver := range nameservers {
+		ns = append(ns, netip.MustParseAddr(nameserver))
+	}
+
+	osConfig := dns.OSConfig{
+		Nameservers:   ns,
+		SearchDomains: []dnsname.FQDN{},
+	}
+
+	for i := 0; i < maxSetDNSAttempts; i++ {
+		if err := r.SetDNS(osConfig); err != nil {
+			return err
+		}
+		currentNS := currentDNS(iface)
+		if reflect.DeepEqual(currentNS, nameservers) {
+			return nil
+		}
+	}
+	mainLog.Debug().Msg("DNS was not set for some reason")
+	return nil
+}
+
+func resetDNS(iface *net.Interface) (err error) {
+	defer func() {
+		if err == nil {
+			return
+		}
+		if r, oerr := dns.NewOSConfigurator(logf, iface.Name); oerr == nil {
+			_ = r.SetDNS(dns.OSConfig{})
+			if err := r.Close(); err != nil {
+				mainLog.Error().Err(err).Msg("failed to rollback DNS setting")
+				return
+			}
+			err = nil
+		}
+	}()
+
+	var ns []string
+	c, err := nclient4.New(iface.Name)
+	if err != nil {
+		return fmt.Errorf("nclient4.New: %w", err)
+	}
+	defer c.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	lease, err := c.Request(ctx)
+	if err != nil {
+		return fmt.Errorf("nclient4.Request: %w", err)
+	}
+	for _, nameserver := range lease.ACK.DNS() {
+		if nameserver.Equal(net.IPv4zero) {
+			continue
+		}
+		ns = append(ns, nameserver.String())
+	}
+
+	// TODO(cuonglm): handle DHCPv6 properly.
+	if ctrldnet.SupportsIPv6() {
+		c := client6.NewClient()
+		conversation, err := c.Exchange(iface.Name)
+		if err != nil {
+			mainLog.Debug().Err(err).Msg("could not exchange DHCPv6")
+		}
+		for _, packet := range conversation {
+			if packet.Type() == dhcpv6.MessageTypeReply {
+				msg, err := packet.GetInnerMessage()
+				if err != nil {
+					mainLog.Debug().Err(err).Msg("could not get inner DHCPv6 message")
+					return nil
+				}
+				nameservers := msg.Options.DNS()
+				for _, nameserver := range nameservers {
+					ns = append(ns, nameserver.String())
+				}
+			}
+		}
+	}
+
+	return ignoringEINTR(func() error {
+		return setDNS(iface, ns)
+	})
+}
+
+func currentDNS(iface *net.Interface) []string {
+	for _, fn := range []getDNS{getDNSByResolvectl, getDNSByNmcli, resolvconffile.NameServers} {
+		if ns := fn(iface.Name); len(ns) > 0 {
+			return ns
+		}
+	}
+	return nil
+}
+
+func getDNSByResolvectl(iface string) []string {
+	b, err := exec.Command("resolvectl", "dns", "-i", iface).Output()
+	if err != nil {
+		return nil
+	}
+
+	parts := strings.Fields(strings.SplitN(string(b), "%", 2)[0])
+	if len(parts) > 2 {
+		return parts[3:]
+	}
+	return nil
+}
+
+func getDNSByNmcli(iface string) []string {
+	b, err := exec.Command("nmcli", "dev", "show", iface).Output()
+	if err != nil {
+		return nil
+	}
+	s := bufio.NewScanner(bytes.NewReader(b))
+	var dns []string
+	do := func(line string) {
+		parts := strings.SplitN(line, ":", 2)
+		if len(parts) > 1 {
+			dns = append(dns, strings.TrimSpace(parts[1]))
+		}
+	}
+	for s.Scan() {
+		line := s.Text()
+		switch {
+		case strings.HasPrefix(line, "IP4.DNS"):
+			fallthrough
+		case strings.HasPrefix(line, "IP6.DNS"):
+			do(line)
+		}
+	}
+	return dns
+}
+
+func ignoringEINTR(fn func() error) error {
+	for {
+		err := fn()
+		if err != syscall.EINTR {
+			return err
+		}
+	}
+}

cmd/ctrld/os_mac.go

@@ -1,28 +0,0 @@
-//go:build darwin
-// +build darwin
-
-package main
-
-import (
-	"os/exec"
-)
-
-// allocate loopback ip
-// sudo ifconfig lo0 alias 127.0.0.2 up
-func allocateIP(ip string) error {
-	cmd := exec.Command("ifconfig", "lo0", "alias", ip, "up")
-	if err := cmd.Run(); err != nil {
-		mainLog.Error().Err(err).Msg("allocateIP failed")
-		return err
-	}
-	return nil
-}
-
-func deAllocateIP(ip string) error {
-	cmd := exec.Command("ifconfig", "lo0", "-alias", ip)
-	if err := cmd.Run(); err != nil {
-		mainLog.Error().Err(err).Msg("deAllocateIP failed")
-		return err
-	}
-	return nil
-}

cmd/ctrld/os_others.go

@@ -0,0 +1,13 @@
+//go:build !linux && !darwin && !freebsd
+
+package main
+
+// TODO(cuonglm): implement.
+func allocateIP(ip string) error {
+	return nil
+}
+
+// TODO(cuonglm): implement.
+func deAllocateIP(ip string) error {
+	return nil
+}

cmd/ctrld/os_windows.go

@@ -1,14 +1,95 @@
-//go:build windows
-// +build windows
-
 package main
 
-// TODO(cuonglm): implement.
-func allocateIP(ip string) error {
+import (
+	"errors"
+	"net"
+	"os/exec"
+	"strconv"
+
+	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
+
+	ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
+)
+
+func setDNS(iface *net.Interface, nameservers []string) error {
+	if len(nameservers) == 0 {
+		return errors.New("empty DNS nameservers")
+	}
+	primaryDNS := nameservers[0]
+	if err := setPrimaryDNS(iface, primaryDNS); err != nil {
+		return err
+	}
+	if len(nameservers) > 1 {
+		secondaryDNS := nameservers[1]
+		_ = addSecondaryDNS(iface, secondaryDNS)
+	}
 	return nil
 }
 
-// TODO(cuonglm): implement.
-func deAllocateIP(ip string) error {
+// TODO(cuonglm): should we use system API?
+func resetDNS(iface *net.Interface) error {
+	if ctrldnet.SupportsIPv6ListenLocal() {
+		if output, err := netsh("interface", "ipv6", "set", "dnsserver", strconv.Itoa(iface.Index), "dhcp"); err != nil {
+			mainLog.Warn().Err(err).Msgf("failed to reset ipv6 DNS: %s", string(output))
+		}
+	}
+	output, err := netsh("interface", "ipv4", "set", "dnsserver", strconv.Itoa(iface.Index), "dhcp")
+	if err != nil {
+		mainLog.Error().Err(err).Msgf("failed to reset ipv4 DNS: %s", string(output))
+		return err
+	}
 	return nil
 }
+
+func setPrimaryDNS(iface *net.Interface, dns string) error {
+	ipVer := "ipv4"
+	if ctrldnet.IsIPv6(dns) {
+		ipVer = "ipv6"
+	}
+	idx := strconv.Itoa(iface.Index)
+	output, err := netsh("interface", ipVer, "set", "dnsserver", idx, "static", dns)
+	if err != nil {
+		mainLog.Error().Err(err).Msgf("failed to set primary DNS: %s", string(output))
+		return err
+	}
+	if ipVer == "ipv4" && ctrldnet.SupportsIPv6ListenLocal() {
+		// Disable IPv6 DNS, so the query will be fallback to IPv4.
+		_, _ = netsh("interface", "ipv6", "set", "dnsserver", idx, "static", "::1", "primary")
+	}
+
+	return nil
+}
+
+func addSecondaryDNS(iface *net.Interface, dns string) error {
+	ipVer := "ipv4"
+	if ctrldnet.IsIPv6(dns) {
+		ipVer = "ipv6"
+	}
+	output, err := netsh("interface", ipVer, "add", "dns", strconv.Itoa(iface.Index), dns, "index=2")
+	if err != nil {
+		mainLog.Warn().Err(err).Msgf("failed to add secondary DNS: %s", string(output))
+	}
+	return nil
+}
+
+func netsh(args ...string) ([]byte, error) {
+	return exec.Command("netsh", args...).Output()
+}
+
+func currentDNS(iface *net.Interface) []string {
+	luid, err := winipcfg.LUIDFromIndex(uint32(iface.Index))
+	if err != nil {
+		mainLog.Error().Err(err).Msg("failed to get interface LUID")
+		return nil
+	}
+	nameservers, err := luid.DNS()
+	if err != nil {
+		mainLog.Error().Err(err).Msg("failed to get interface DNS")
+		return nil
+	}
+	ns := make([]string, 0, len(nameservers))
+	for _, nameserver := range nameservers {
+		ns = append(ns, nameserver.String())
+	}
+	return ns
+}

cmd/ctrld/prog.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"errors"
+	"fmt"
+	"math/rand"
 	"net"
 	"os"
 	"strconv"
@@ -9,11 +11,15 @@ import (
 	"syscall"
 
 	"github.com/kardianos/service"
-	"github.com/miekg/dns"
 
 	"github.com/Control-D-Inc/ctrld"
+	"github.com/Control-D-Inc/ctrld/internal/dnscache"
 )
 
+var logf = func(format string, args ...any) {
+	mainLog.Debug().Msgf(format, args...)
+}
+
 var errWindowsAddrInUse = syscall.Errno(0x2740)
 
 var svcConfig = &service.Config{
@@ -22,7 +28,8 @@ var svcConfig = &service.Config{
 }
 
 type prog struct {
-	cfg *ctrld.Config
+	cfg   *ctrld.Config
+	cache dnscache.Cacher
 }
 
 func (p *prog) Start(s service.Service) error {
@@ -32,6 +39,15 @@ func (p *prog) Start(s service.Service) error {
 }
 
 func (p *prog) run() {
+	p.preRun()
+	if p.cfg.Service.CacheEnable {
+		cacher, err := dnscache.NewLRUCache(p.cfg.Service.CacheSize)
+		if err != nil {
+			mainLog.Error().Err(err).Msg("failed to create cacher, caching is disabled")
+		} else {
+			p.cache = cacher
+		}
+	}
 	var wg sync.WaitGroup
 	wg.Add(len(p.cfg.Listener))
 
@@ -39,7 +55,7 @@ func (p *prog) run() {
 		for _, cidr := range nc.Cidrs {
 			_, ipNet, err := net.ParseCIDR(cidr)
 			if err != nil {
-				proxyLog.Error().Err(err).Str("network", nc.Name).Str("cidr", cidr).Msg("invalid cidr")
+				mainLog.Error().Err(err).Str("network", nc.Name).Str("cidr", cidr).Msg("invalid cidr")
 				continue
 			}
 			nc.IPNets = append(nc.IPNets, ipNet)
@@ -48,83 +64,58 @@ func (p *prog) run() {
 	for n := range p.cfg.Upstream {
 		uc := p.cfg.Upstream[n]
 		uc.Init()
-
 		if uc.BootstrapIP == "" {
-			// resolve it manually and set the bootstrap ip
-			c := new(dns.Client)
-			m := new(dns.Msg)
-			m.SetQuestion(uc.Domain+".", dns.TypeA)
-			m.RecursionDesired = true
-			r, _, err := c.Exchange(m, net.JoinHostPort(bootstrapDNS, "53"))
-			if err != nil {
-				proxyLog.Error().Err(err).Msgf("could not resolve domain %s for upstream.%s", uc.Domain, n)
-			} else {
-				if r.Rcode != dns.RcodeSuccess {
-					proxyLog.Error().Msgf("could not resolve domain return code: %d, upstream.%s", r.Rcode, n)
-				} else {
-					for _, a := range r.Answer {
-						if ar, ok := a.(*dns.A); ok {
-							uc.BootstrapIP = ar.A.String()
-							proxyLog.Info().Str("bootstrap_ip", uc.BootstrapIP).Msgf("Setting bootstrap IP for upstream.%s", n)
-						}
-					}
-				}
-			}
+			uc.SetupBootstrapIP()
+			mainLog.Info().Str("bootstrap_ip", uc.BootstrapIP).Msgf("Setting bootstrap IP for upstream.%s", n)
+		} else {
+			mainLog.Info().Str("bootstrap_ip", uc.BootstrapIP).Msgf("Using bootstrap IP for upstream.%s", n)
 		}
+		uc.SetupTransport()
 	}
 
 	for listenerNum := range p.cfg.Listener {
+		p.cfg.Listener[listenerNum].Init()
 		go func(listenerNum string) {
 			defer wg.Done()
 			listenerConfig := p.cfg.Listener[listenerNum]
 			upstreamConfig := p.cfg.Upstream[listenerNum]
 			if upstreamConfig == nil {
-				proxyLog.Error().Msgf("missing upstream config for: [listener.%s]", listenerNum)
+				mainLog.Error().Msgf("missing upstream config for: [listener.%s]", listenerNum)
 				return
 			}
 			addr := net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port))
-			proxyLog.Info().Msgf("Starting DNS server on listener.%s: %s", listenerNum, addr)
+			mainLog.Info().Msgf("Starting DNS server on listener.%s: %s", listenerNum, addr)
 			err := p.serveUDP(listenerNum)
-			if err != nil && !defaultConfigWritten {
-				proxyLog.Error().Err(err).Msgf("Unable to start dns proxy on listener.%s", listenerNum)
+			if err != nil && !defaultConfigWritten && cdUID == "" {
+				mainLog.Fatal().Err(err).Msgf("Unable to start dns proxy on listener.%s", listenerNum)
+				return
+			}
+			if err == nil {
 				return
 			}
 
-			if opErr, ok := err.(*net.OpError); ok {
+			if opErr, ok := err.(*net.OpError); ok && listenerNum == "0" {
 				if sErr, ok := opErr.Err.(*os.SyscallError); ok && errors.Is(opErr.Err, syscall.EADDRINUSE) || errors.Is(sErr.Err, errWindowsAddrInUse) {
-					proxyLog.Warn().Msgf("Address %s already in used, pick a random one", addr)
-					pc, err := net.ListenPacket("udp", net.JoinHostPort(listenerConfig.IP, "0"))
-					if err != nil {
-						proxyLog.Error().Err(err).Msg("failed to listen packet")
-						return
-					}
-					_, portStr, _ := net.SplitHostPort(pc.LocalAddr().String())
-					port, err := strconv.Atoi(portStr)
-					if err != nil {
-						proxyLog.Error().Err(err).Msg("malformed port")
-						return
-					}
-					listenerConfig.Port = port
-					v.Set("listener", map[string]*ctrld.ListenerConfig{
-						"0": {
-							IP:   "127.0.0.1",
-							Port: port,
-						},
-					})
-					writeConfigFile()
-					proxyLog.Info().Msgf("Starting DNS server on listener.%s: %s", listenerNum, pc.LocalAddr())
-					// There can be a race between closing the listener and start our own UDP server, but it's
-					// rare, and we only do this once, so let conservative here.
-					if err := pc.Close(); err != nil {
-						proxyLog.Error().Err(err).Msg("failed to close packet conn")
-						return
+					mainLog.Warn().Msgf("Address %s already in used, pick a random one", addr)
+					ip := randomLocalIP()
+					listenerConfig.IP = ip
+					port := listenerConfig.Port
+					cfg.Upstream = map[string]*ctrld.UpstreamConfig{"0": cfg.Upstream["0"]}
+					if err := writeConfigFile(); err != nil {
+						mainLog.Fatal().Err(err).Msg("failed to write config file")
+					} else {
+						mainLog.Info().Msg("writing config file to: " + defaultConfigFile)
 					}
+					p.cfg.Service.AllocateIP = true
+					p.preRun()
+					mainLog.Info().Msgf("Starting DNS server on listener.%s: %s", listenerNum, net.JoinHostPort(ip, strconv.Itoa(port)))
 					if err := p.serveUDP(listenerNum); err != nil {
-						proxyLog.Error().Err(err).Msgf("Unable to start dns proxy on listener.%s", listenerNum)
+						mainLog.Fatal().Err(err).Msgf("Unable to start dns proxy on listener.%s", listenerNum)
 						return
 					}
 				}
 			}
+			mainLog.Fatal().Err(err).Msgf("Unable to start dns proxy on listener.%s", listenerNum)
 		}(listenerNum)
 	}
 
@@ -136,6 +127,7 @@ func (p *prog) Stop(s service.Service) error {
 		mainLog.Error().Err(err).Msg("de-allocate ip failed")
 		return err
 	}
+	mainLog.Info().Msg("Service stopped")
 	return nil
 }
 
@@ -157,3 +149,61 @@ func (p *prog) deAllocateIP() error {
 	}
 	return nil
 }
+
+func (p *prog) setDNS() {
+	if cfg.Listener == nil || cfg.Listener["0"] == nil {
+		return
+	}
+	if iface == "" {
+		return
+	}
+	if iface == "auto" {
+		iface = defaultIfaceName()
+	}
+	logger := mainLog.With().Str("iface", iface).Logger()
+	netIface, err := netInterface(iface)
+	if err != nil {
+		logger.Error().Err(err).Msg("could not get interface")
+		return
+	}
+	if err := setupNetworkManager(); err != nil {
+		logger.Error().Err(err).Msg("could not patch NetworkManager")
+		return
+	}
+	logger.Debug().Msg("setting DNS for interface")
+	if err := setDNS(netIface, []string{cfg.Listener["0"].IP}); err != nil {
+		logger.Error().Err(err).Msgf("could not set DNS for interface")
+		return
+	}
+	logger.Debug().Msg("setting DNS successfully")
+}
+
+func (p *prog) resetDNS() {
+	if iface == "" {
+		return
+	}
+	if iface == "auto" {
+		iface = defaultIfaceName()
+	}
+	logger := mainLog.With().Str("iface", iface).Logger()
+	netIface, err := netInterface(iface)
+	if err != nil {
+		logger.Error().Err(err).Msg("could not get interface")
+		return
+	}
+	if err := restoreNetworkManager(); err != nil {
+		logger.Error().Err(err).Msg("could not restore NetworkManager")
+		return
+	}
+	logger.Debug().Msg("Restoring DNS for interface")
+	if err := resetDNS(netIface); err != nil {
+		logger.Error().Err(err).Msgf("could not reset DNS")
+		return
+	}
+	logger.Debug().Msg("Restoring DNS successfully")
+}
+
+func randomLocalIP() string {
+	n := rand.Intn(254-2) + 2
+	return fmt.Sprintf("127.0.0.%d", n)
+}

cmd/ctrld/prog_freebsd.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"os"
+
+	"github.com/kardianos/service"
+)
+
+func (p *prog) preRun() {
+	if !service.Interactive() {
+		p.setDNS()
+	}
+}
+
+func setDependencies(svc *service.Config) {
+	// TODO(cuonglm): remove once https://github.com/kardianos/service/issues/359 fixed.
+	_ = os.MkdirAll("/usr/local/etc/rc.d", 0755)
+}
+
+func setWorkingDirectory(svc *service.Config, dir string) {}

cmd/ctrld/prog_linux.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"github.com/kardianos/service"
+)
+
+func (p *prog) preRun() {
+	if !service.Interactive() {
+		p.setDNS()
+	}
+}
+
+func setDependencies(svc *service.Config) {
+	svc.Dependencies = []string{
+		"Wants=network-online.target",
+		"After=network-online.target",
+		"Wants=NetworkManager-wait-online.service",
+		"After=NetworkManager-wait-online.service",
+	}
+}
+
+func setWorkingDirectory(svc *service.Config, dir string) {
+	svc.WorkingDirectory = dir
+}

cmd/ctrld/prog_others.go

@@ -0,0 +1,14 @@
+//go:build !linux && !freebsd
+
+package main
+
+import "github.com/kardianos/service"
+
+func (p *prog) preRun() {}
+
+func setDependencies(svc *service.Config) {}
+
+func setWorkingDirectory(svc *service.Config, dir string) {
+	// WorkingDirectory is not supported on Windows.
+	svc.WorkingDirectory = dir
+}

cmd/ctrld/service.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+func stderrMsg(msg string) {
+	_, _ = fmt.Fprintln(os.Stderr, msg)
+}
+
+func stdoutMsg(msg string) {
+	_, _ = fmt.Fprintln(os.Stdout, msg)
+}
+
+type task struct {
+	f            func() error
+	abortOnError bool
+}
+
+func doTasks(tasks []task) bool {
+	for _, task := range tasks {
+		if err := task.f(); err != nil {
+			if task.abortOnError {
+				stderrMsg(err.Error())
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func checkHasElevatedPrivilege(cmd *cobra.Command, args []string) {
+	ok, err := hasElevatedPrivilege()
+	if err != nil {
+		fmt.Printf("could not detect user privilege: %v", err)
+		return
+	}
+	if !ok {
+		fmt.Println("Please relaunch process with admin/root privilege.")
+		os.Exit(1)
+	}
+}

cmd/ctrld/service_others.go

@@ -0,0 +1,11 @@
+//go:build !windows
+
+package main
+
+import (
+	"os"
+)
+
+func hasElevatedPrivilege() (bool, error) {
+	return os.Geteuid() == 0, nil
+}

cmd/ctrld/service_windows.go

@@ -0,0 +1,24 @@
+package main
+
+import "golang.org/x/sys/windows"
+
+func hasElevatedPrivilege() (bool, error) {
+	var sid *windows.SID
+	if err := windows.AllocateAndInitializeSid(
+		&windows.SECURITY_NT_AUTHORITY,
+		2,
+		windows.SECURITY_BUILTIN_DOMAIN_RID,
+		windows.DOMAIN_ALIAS_RID_ADMINS,
+		0,
+		0,
+		0,
+		0,
+		0,
+		0,
+		&sid,
+	); err != nil {
+		return false, err
+	}
+	token := windows.Token(0)
+	return token.IsMember(sid)
+}

config.go

@@ -1,24 +1,42 @@
 package ctrld
 
 import (
+	"context"
 	"net"
+	"net/http"
 	"net/url"
+	"os"
 	"strings"
+	"sync/atomic"
+	"time"
 
 	"github.com/go-playground/validator/v10"
+	"github.com/miekg/dns"
 	"github.com/spf13/viper"
+	"golang.org/x/sync/singleflight"
+
+	"github.com/Control-D-Inc/ctrld/internal/dnsrcode"
+	ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
 )
 
+// SetConfigName set the config name that ctrld will look for.
+func SetConfigName(v *viper.Viper, name string) {
+	v.SetConfigName(name)
+
+	configPath := "$HOME"
+	// viper has its own way to get user home directory:  https://github.com/spf13/viper/blob/v1.14.0/util.go#L134
+	// To be consistent, we prefer os.UserHomeDir instead.
+	if homeDir, err := os.UserHomeDir(); err == nil {
+		configPath = homeDir
+	}
+	v.AddConfigPath(configPath)
+	v.AddConfigPath(".")
+}
+
 // InitConfig initializes default config values for given *viper.Viper instance.
 func InitConfig(v *viper.Viper, name string) {
-	v.SetConfigName(name)
-	v.SetConfigType("toml")
-	v.AddConfigPath("$HOME/.ctrld")
-	v.AddConfigPath(".")
+	SetConfigName(v, name)
 
-	v.SetDefault("service", ServiceConfig{
-		LogLevel: "info",
-	})
 	v.SetDefault("listener", map[string]*ListenerConfig{
 		"0": {
 			IP:   "127.0.0.1",
@@ -35,14 +53,14 @@ func InitConfig(v *viper.Viper, name string) {
 		"0": {
 			BootstrapIP: "76.76.2.11",
 			Name:        "Control D - Anti-Malware",
-			Type:        "doh",
+			Type:        ResolverTypeDOH,
 			Endpoint:    "https://freedns.controld.com/p1",
 			Timeout:     5000,
 		},
 		"1": {
 			BootstrapIP: "76.76.2.11",
 			Name:        "Control D - No Ads",
-			Type:        "doq",
+			Type:        ResolverTypeDOQ,
 			Endpoint:    "p2.freedns.controld.com",
 			Timeout:     3000,
 		},
@@ -51,50 +69,62 @@ func InitConfig(v *viper.Viper, name string) {
 
 // Config represents ctrld supported configuration.
 type Config struct {
-	Service  ServiceConfig              `mapstructure:"service"`
+	Service  ServiceConfig              `mapstructure:"service" toml:"service,omitempty"`
+	Listener map[string]*ListenerConfig `mapstructure:"listener" toml:"listener" validate:"min=1,dive"`
 	Network  map[string]*NetworkConfig  `mapstructure:"network" toml:"network" validate:"min=1,dive"`
 	Upstream map[string]*UpstreamConfig `mapstructure:"upstream" toml:"upstream" validate:"min=1,dive"`
-	Listener map[string]*ListenerConfig `mapstructure:"listener" toml:"listener" validate:"min=1,dive"`
 }
 
 // ServiceConfig specifies the general ctrld config.
 type ServiceConfig struct {
-	LogLevel   string `mapstructure:"log_level" toml:"log_level"`
-	LogPath    string `mapstructure:"log_path" toml:"log_path"`
-	Daemon     bool   `mapstructure:"-" toml:"-"`
-	AllocateIP bool   `mapstructure:"-" toml:"-"`
+	LogLevel         string `mapstructure:"log_level" toml:"log_level,omitempty"`
+	LogPath          string `mapstructure:"log_path" toml:"log_path,omitempty"`
+	CacheEnable      bool   `mapstructure:"cache_enable" toml:"cache_enable,omitempty"`
+	CacheSize        int    `mapstructure:"cache_size" toml:"cache_size,omitempty"`
+	CacheTTLOverride int    `mapstructure:"cache_ttl_override" toml:"cache_ttl_override,omitempty"`
+	CacheServeStale  bool   `mapstructure:"cache_serve_stale" toml:"cache_serve_stale,omitempty"`
+	Daemon           bool   `mapstructure:"-" toml:"-"`
+	AllocateIP       bool   `mapstructure:"-" toml:"-"`
 }
 
 // NetworkConfig specifies configuration for networks where ctrld will handle requests.
 type NetworkConfig struct {
-	Name   string       `mapstructure:"name" toml:"name"`
-	Cidrs  []string     `mapstructure:"cidrs" toml:"cidrs" validate:"dive,cidr"`
+	Name   string       `mapstructure:"name" toml:"name,omitempty"`
+	Cidrs  []string     `mapstructure:"cidrs" toml:"cidrs,omitempty" validate:"dive,cidr"`
 	IPNets []*net.IPNet `mapstructure:"-" toml:"-"`
 }
 
 // UpstreamConfig specifies configuration for upstreams that ctrld will forward requests to.
 type UpstreamConfig struct {
-	Name        string `mapstructure:"name" toml:"name"`
-	Type        string `mapstructure:"type" toml:"type" validate:"oneof=doh doh3 dot doq os legacy"`
-	Endpoint    string `mapstructure:"endpoint" toml:"endpoint" validate:"required_unless=Type os"`
-	BootstrapIP string `mapstructure:"bootstrap_ip" toml:"bootstrap_ip"`
-	Domain      string `mapstructure:"-" toml:"-"`
-	Timeout     int    `mapstructure:"timeout" toml:"timeout" validate:"gte=0"`
+	Name              string            `mapstructure:"name" toml:"name,omitempty"`
+	Type              string            `mapstructure:"type" toml:"type,omitempty" validate:"oneof=doh doh3 dot doq os legacy"`
+	Endpoint          string            `mapstructure:"endpoint" toml:"endpoint,omitempty" validate:"required_unless=Type os"`
+	BootstrapIP       string            `mapstructure:"bootstrap_ip" toml:"bootstrap_ip,omitempty"`
+	Domain            string            `mapstructure:"-" toml:"-"`
+	Timeout           int               `mapstructure:"timeout" toml:"timeout,omitempty" validate:"gte=0"`
+	transport         *http.Transport   `mapstructure:"-" toml:"-"`
+	http3RoundTripper http.RoundTripper `mapstructure:"-" toml:"-"`
+
+	g               singleflight.Group
+	bootstrapIPs    []string
+	nextBootstrapIP atomic.Uint32
 }
 
 // ListenerConfig specifies the networks configuration that ctrld will run on.
 type ListenerConfig struct {
-	IP         string                `mapstructure:"ip" toml:"ip" validate:"ip"`
-	Port       int                   `mapstructure:"port" toml:"port" validate:"gt=0"`
-	Restricted bool                  `mapstructure:"restricted" toml:"restricted"`
-	Policy     *ListenerPolicyConfig `mapstructure:"policy" toml:"policy"`
+	IP         string                `mapstructure:"ip" toml:"ip,omitempty" validate:"ip"`
+	Port       int                   `mapstructure:"port" toml:"port,omitempty" validate:"gt=0"`
+	Restricted bool                  `mapstructure:"restricted" toml:"restricted,omitempty"`
+	Policy     *ListenerPolicyConfig `mapstructure:"policy" toml:"policy,omitempty"`
 }
 
 // ListenerPolicyConfig specifies the policy rules for ctrld to filter incoming requests.
 type ListenerPolicyConfig struct {
-	Name     string `mapstructure:"name" toml:"name"`
-	Networks []Rule `mapstructure:"networks" toml:"networks" validate:"dive,len=1"`
-	Rules    []Rule `mapstructure:"rules" toml:"rules" validate:"dive,len=1"`
+	Name                 string   `mapstructure:"name" toml:"name,omitempty"`
+	Networks             []Rule   `mapstructure:"networks" toml:"networks,omitempty,inline,multiline" validate:"dive,len=1"`
+	Rules                []Rule   `mapstructure:"rules" toml:"rules,omitempty,inline,multiline" validate:"dive,len=1"`
+	FailoverRcodes       []string `mapstructure:"failover_rcodes" toml:"failover_rcodes,omitempty" validate:"dive,dnsrcode"`
+	FailoverRcodeNumbers []int    `mapstructure:"-" toml:"-"`
 }
 
 // Rule is a map from source to list of upstreams.
@@ -122,18 +152,199 @@ func (uc *UpstreamConfig) Init() {
 	}
 }
 
+// SetupBootstrapIP manually find all available IPs of the upstream.
+// The first usable IP will be used as bootstrap IP of the upstream.
+func (uc *UpstreamConfig) SetupBootstrapIP() {
+	bootstrapIP := func(record dns.RR) string {
+		switch ar := record.(type) {
+		case *dns.A:
+			return ar.A.String()
+		case *dns.AAAA:
+			return ar.AAAA.String()
+		}
+		return ""
+	}
+
+	resolver := &osResolver{nameservers: nameservers()}
+	resolver.nameservers = append([]string{net.JoinHostPort(bootstrapDNS, "53")}, resolver.nameservers...)
+	ProxyLog.Debug().Msgf("Resolving %q using bootstrap DNS %q", uc.Domain, resolver.nameservers)
+	do := func(dnsType uint16) {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(uc.Timeout)*time.Millisecond)
+		defer cancel()
+		m := new(dns.Msg)
+		m.SetQuestion(uc.Domain+".", dnsType)
+		m.RecursionDesired = true
+
+		r, err := resolver.Resolve(ctx, m)
+		if err != nil {
+			ProxyLog.Error().Err(err).Str("type", dns.TypeToString[dnsType]).Msgf("could not resolve domain %s for upstream", uc.Domain)
+			return
+		}
+		if r.Rcode != dns.RcodeSuccess {
+			ProxyLog.Error().Msgf("could not resolve domain return code: %d, upstream", r.Rcode)
+			return
+		}
+		if len(r.Answer) == 0 {
+			ProxyLog.Error().Msg("no answer from bootstrap DNS server")
+			return
+		}
+		for _, a := range r.Answer {
+			ip := bootstrapIP(a)
+			if ip == "" {
+				continue
+			}
+
+			// Storing the ip to uc.bootstrapIPs list, so it can be selected later
+			// when retrying failed request due to network stack changed.
+			uc.bootstrapIPs = append(uc.bootstrapIPs, ip)
+			if uc.BootstrapIP == "" {
+				// Remember what's the current IP in bootstrap IPs list,
+				// so we can select next one upon re-bootstrapping.
+				uc.nextBootstrapIP.Add(1)
+
+				// If this is an ipv6, and ipv6 is not available, don't use it as bootstrap ip.
+				if !ctrldnet.IPv6Available(ctx) && ctrldnet.IsIPv6(ip) {
+					continue
+				}
+				uc.BootstrapIP = ip
+			}
+		}
+	}
+	// Find all A, AAAA records of the upstream.
+	for _, dnsType := range []uint16{dns.TypeAAAA, dns.TypeA} {
+		do(dnsType)
+	}
+	ProxyLog.Debug().Msgf("Bootstrap IPs: %v", uc.bootstrapIPs)
+}
+
+// ReBootstrap re-setup the bootstrap IP and the transport.
+func (uc *UpstreamConfig) ReBootstrap() {
+	switch uc.Type {
+	case ResolverTypeDOH, ResolverTypeDOH3:
+	default:
+		return
+	}
+	_, _, _ = uc.g.Do("rebootstrap", func() (any, error) {
+		ProxyLog.Debug().Msg("re-bootstrapping upstream ip")
+		n := uint32(len(uc.bootstrapIPs))
+
+		timeoutMs := 1000
+		if uc.Timeout > 0 && uc.Timeout < timeoutMs {
+			timeoutMs = uc.Timeout
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutMs)*time.Millisecond)
+		defer cancel()
+
+		hasIPv6 := ctrldnet.IPv6Available(ctx)
+		// Only attempt n times, because if there's no usable ip,
+		// the bootstrap ip will be kept as-is.
+		for i := uint32(0); i < n; i++ {
+			// Select the next ip in bootstrap ip list.
+			next := uc.nextBootstrapIP.Add(1)
+			ip := uc.bootstrapIPs[(next-1)%n]
+			if !hasIPv6 && ctrldnet.IsIPv6(ip) {
+				continue
+			}
+			uc.BootstrapIP = ip
+			break
+		}
+		uc.setupTransportWithoutPingUpstream()
+		return true, nil
+	})
+}
+
+func (uc *UpstreamConfig) setupTransportWithoutPingUpstream() {
+	switch uc.Type {
+	case ResolverTypeDOH:
+		uc.setupDOHTransportWithoutPingUpstream()
+	case ResolverTypeDOH3:
+		uc.setupDOH3TransportWithoutPingUpstream()
+	}
+}
+
+// SetupTransport initializes the network transport used to connect to upstream server.
+// For now, only DoH upstream is supported.
+func (uc *UpstreamConfig) SetupTransport() {
+	switch uc.Type {
+	case ResolverTypeDOH:
+		uc.setupDOHTransport()
+	case ResolverTypeDOH3:
+		uc.setupDOH3Transport()
+	}
+}
+
+func (uc *UpstreamConfig) setupDOHTransport() {
+	uc.setupDOHTransportWithoutPingUpstream()
+	uc.pingUpstream()
+}
+
+func (uc *UpstreamConfig) setupDOHTransportWithoutPingUpstream() {
+	uc.transport = http.DefaultTransport.(*http.Transport).Clone()
+	uc.transport.IdleConnTimeout = 5 * time.Second
+
+	dialerTimeoutMs := 2000
+	if uc.Timeout > 0 && uc.Timeout < dialerTimeoutMs {
+		dialerTimeoutMs = uc.Timeout
+	}
+	dialerTimeout := time.Duration(dialerTimeoutMs) * time.Millisecond
+	uc.transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+		dialer := &net.Dialer{
+			Timeout:   dialerTimeout,
+			KeepAlive: dialerTimeout,
+		}
+		// if we have a bootstrap ip set, use it to avoid DNS lookup
+		if uc.BootstrapIP != "" {
+			if _, port, _ := net.SplitHostPort(addr); port != "" {
+				addr = net.JoinHostPort(uc.BootstrapIP, port)
+			}
+		}
+		Log(ctx, ProxyLog.Debug(), "sending doh request to: %s", addr)
+		return dialer.DialContext(ctx, network, addr)
+	}
+}
+
+func (uc *UpstreamConfig) pingUpstream() {
+	// Warming up the transport by querying a test packet.
+	dnsResolver, err := NewResolver(uc)
+	if err != nil {
+		ProxyLog.Error().Err(err).Msgf("failed to create resolver for upstream: %s", uc.Name)
+		return
+	}
+	msg := new(dns.Msg)
+	msg.SetQuestion(".", dns.TypeNS)
+	msg.MsgHdr.RecursionDesired = true
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, _ = dnsResolver.Resolve(ctx, msg)
+}
+
+// Init initialized necessary values for an ListenerConfig.
+func (lc *ListenerConfig) Init() {
+	if lc.Policy != nil {
+		lc.Policy.FailoverRcodeNumbers = make([]int, len(lc.Policy.FailoverRcodes))
+		for i, rcode := range lc.Policy.FailoverRcodes {
+			lc.Policy.FailoverRcodeNumbers[i] = dnsrcode.FromString(rcode)
+		}
+	}
+}
+
 // ValidateConfig validates the given config.
 func ValidateConfig(validate *validator.Validate, cfg *Config) error {
+	_ = validate.RegisterValidation("dnsrcode", validateDnsRcode)
 	return validate.Struct(cfg)
 }
 
+func validateDnsRcode(fl validator.FieldLevel) bool {
+	return dnsrcode.FromString(fl.Field().String()) != -1
+}
+
 func defaultPortFor(typ string) string {
 	switch typ {
-	case resolverTypeDOH, resolverTypeDOH3:
+	case ResolverTypeDOH, ResolverTypeDOH3:
 		return "443"
-	case resolverTypeDOQ, resolverTypeDOT:
+	case ResolverTypeDOQ, ResolverTypeDOT:
 		return "853"
-	case resolverTypeLegacy:
+	case ResolverTypeLegacy:
 		return "53"
 	}
 	return "53"

config_quic.go

@@ -0,0 +1,44 @@
+//go:build !qf
+
+package ctrld
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+
+	"github.com/quic-go/quic-go"
+	"github.com/quic-go/quic-go/http3"
+)
+
+func (uc *UpstreamConfig) setupDOH3Transport() {
+	uc.setupDOH3TransportWithoutPingUpstream()
+	uc.pingUpstream()
+}
+
+func (uc *UpstreamConfig) setupDOH3TransportWithoutPingUpstream() {
+	rt := &http3.RoundTripper{}
+	rt.Dial = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error) {
+		host := addr
+		ProxyLog.Debug().Msgf("debug dial context D0H3 %s - %s", addr, bootstrapDNS)
+		// if we have a bootstrap ip set, use it to avoid DNS lookup
+		if uc.BootstrapIP != "" {
+			if _, port, _ := net.SplitHostPort(addr); port != "" {
+				addr = net.JoinHostPort(uc.BootstrapIP, port)
+			}
+			ProxyLog.Debug().Msgf("sending doh3 request to: %s", addr)
+		}
+		remoteAddr, err := net.ResolveUDPAddr("udp", addr)
+		if err != nil {
+			return nil, err
+		}
+
+		udpConn, err := net.ListenUDP("udp", nil)
+		if err != nil {
+			return nil, err
+		}
+		return quic.DialEarlyContext(ctx, udpConn, remoteAddr, host, tlsCfg, cfg)
+	}
+
+	uc.http3RoundTripper = rt
+}

config_quic_free.go

@@ -0,0 +1,7 @@
+//go:build qf
+
+package ctrld
+
+func (uc *UpstreamConfig) setupDOH3Transport() {}
+
+func (uc *UpstreamConfig) setupDOH3TransportWithoutPingUpstream() {}

config_test.go

@@ -69,6 +69,7 @@ func TestConfigValidation(t *testing.T) {
 		{"invalid listener port", invalidListenerPort(t), true},
 		{"os upstream", configWithOsUpstream(t), false},
 		{"invalid rules", configWithInvalidRules(t), true},
+		{"invalid dns rcodes", configWithInvalidRcodes(t), true},
 	}
 
 	for _, tc := range tests {
@@ -155,6 +156,16 @@ func configWithInvalidRules(t *testing.T) *ctrld.Config {
 	return cfg
 }
 
+func configWithInvalidRcodes(t *testing.T) *ctrld.Config {
+	cfg := defaultConfig(t)
+	cfg.Listener["0"].Policy = &ctrld.ListenerPolicyConfig{
+		Name:           "Policy with invalid Rcodes",
+		Networks:       []ctrld.Rule{{"*.com": []string{"upstream.0"}}},
+		FailoverRcodes: []string{"foo"},
+	}
+	return cfg
+}
+
 func TestUpstreamConfig_Init(t *testing.T) {
 	tests := []struct {
 		name     string

docs/config.md

@@ -25,7 +25,10 @@ The user can choose to override default value using command line `--config` or `
 ctrld run --config /path/to/myconfig.toml
 ```
 
-If no configuration files found, a default `config.toml` file will be created in the current directory.
+If no configuration files found, a default `ctrld.toml` file will be created in the current directory.
+
+In pre v1.1.0, `config.toml` file was used, so for compatibility, `ctrld` will still read `config.toml`
+if it's existed.
 
 # Example Config
 
@@ -33,6 +36,8 @@ If no configuration files found, a default `config.toml` file will be created in
 [service]
     log_level = "info"
     log_path = ""
+    cache_enable = true
+    cache_size = 4096
 
 [network.0]
     cidrs = ["0.0.0.0/0"]
@@ -109,6 +114,31 @@ Relative or absolute path of the log file.
 - Type: string
 - Required: no
 
+### cache_enable
+When `cache_enable = true`, all resolved DNS query responses will be cached for duration of the upstream record TTLs.
+
+- Type: boolean
+- Required: no
+
+### cache_size
+The number of cached records, must be a positive integer. Tweaking this value with care depends on your available RAM. 
+A minimum value `4096` should be enough for most use cases.
+
+An invalid `cache_size` value will disable the cache, regardless of `cache_enable` value.
+
+- Type: int
+- Required: no
+
+### cache_ttl_override
+When `cache_ttl_override` is set to a positive value (in seconds), TTLs are overridden to this value and cached for this long.
+
+- Type: int
+- Required: no
+
+### cache_serve_stale
+When `cache_serve_stale = true`, in cases of upstream failures (upstreams not reachable), `ctrld` will keep serving
+stale cached records (regardless of their TTLs) until upstream comes online.
+
 The above config will look like this at query time.
 
 ```
@@ -284,6 +314,17 @@ Above policy will:
 - Forward requests on `listener.0` for `test.com` to `upstream.2`. If timeout is reached, retry on `upstream.1`.
 - All other requests on `listener.0` that do not match above conditions will be forwarded to `upstream.0`.
 
+An empty upstream would not route the request to any defined upstreams, and use the OS default resolver.
+
+```toml
+[listener.0.policy]
+name = "OS Resolver"
+
+rules = [
+    {"*.local" = []},
+]
+```
+
 #### name
 `name` is the name for the policy.
 
@@ -300,4 +341,21 @@ Above policy will:
 
 - type: array of rule
 
+### failover_rcodes
+For non success response, `failover_rcodes` allows the request to be forwarded to next upstream, if the response `RCODE` matches any value defined in `failover_rcodes`. For example:
+
+```toml
+[listener.0.policy]
+name = "My Policy"
+failover_rcodes = ["NXDOMAIN", "SERVFAIL"]
+networks = [
+	{"network.0" = ["upstream.0", "upstream.1"]},
+]
+```
+
+If `upstream.0` returns a NXDOMAIN response, the request will be forwarded to `upstream.1` instead of returning immediately to the client.
+
+See all available DNS Rcodes value [here](rcode_link).
+
 [toml_link]: https://toml.io/en
+[rcode_link]: https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-6

docs/ephemeral_mode.md

@@ -0,0 +1,74 @@
+# Ephemeral Mode
+`ctrld` can operate in ephemeral mode which won't attempt to read or write any config file to disk. All necessary information is provided via command line flags.
+
+## Launch arguments
+```shell
+$ ctrld run --help
+Run the DNS proxy server
+
+Usage:
+  ctrld run [flags]
+
+Flags:
+      --base64_config string        base64 encoded config
+      --cache_size int              Enable cache defined amount of slots
+      --cd string                   Control D resolver uid
+  -c, --config string               Path to config file
+  -d, --daemon                      Run as daemon
+      --domains strings             list of domain to apply in a split DNS policy
+  -h, --help                        help for run
+      --listen string               listener address and port, in format: address:port
+      --log string                  path to log file
+      --primary_upstream string     primary upstream endpoint
+      --secondary_upstream string   secondary upstream endpoint
+
+Global Flags:
+  -v, --verbose count   verbose log output, "-v" means query logging enabled, "-vv" means debug level logging enabled
+```
+
+For example:
+
+```shell
+ctrld run --listen=127.0.0.1:53 --primary_upstream=https://freedns.controld.com/p2 --secondary_upstream=8.8.8.8:53 --domains=*.company.int,*.net --log /path/to/log.log
+```
+
+Above command will be translated roughly to this config:
+
+```toml
+[service]
+    log_level = "debug"
+    log_path = "/path/to/log.log"
+
+[network.0]
+    name = "Network 0"
+    cidrs = ["0.0.0.0/0"]
+
+[upstream.0]
+    name = "https://freedns.controld.com/p2"
+    endpoint = "https://freedns.controld.com/p2"
+    type = "doh"
+
+[upstream.1]
+    name = "8.8.8.8:53"
+    endpoint = "8.8.8.8:53"
+    type = "legacy"
+	
+[listener.0]
+    ip = "127.0.0.1"
+    port = 53
+
+    [listener.0.policy]
+        rules = [
+            {"*.company.int" = ["upstream.1"]},
+            {"*.net" = ["upstream.1"]},
+        ]
+```
+
+Only `listen` and `primary_upstream` flags are required. 
+
+## Base64 encoded config
+`ctrld` can read a complete base64 encoded config via command line flag. This allows you to supply complex configurations. 
+
+```shell
+ctrld run --base64_config="CltsaXN0ZW5lcl0KCiAgW2xpc3RlbmVyLjBdCiAgICBpcCA9ICIxMjcuMC4wLjEiCiAgICBwb3J0ID0gNTMKICAgIHJlc3RyaWN0ZWQgPSBmYWxzZQoKW25ldHdvcmtdCgogIFtuZXR3b3JrLjBdCiAgICBjaWRycyA9IFsiMC4wLjAuMC8wIl0KICAgIG5hbWUgPSAiTmV0d29yayAwIgoKW3Vwc3RyZWFtXQoKICBbdXBzdHJlYW0uMF0KICAgIGJvb3RzdHJhcF9pcCA9ICI3Ni43Ni4yLjExIgogICAgZW5kcG9pbnQgPSAiaHR0cHM6Ly9mcmVlZG5zLmNvbnRyb2xkLmNvbS9wMSIKICAgIG5hbWUgPSAiQ29udHJvbCBEIC0gQW50aS1NYWx3YXJlIgogICAgdGltZW91dCA9IDUwMDAKICAgIHR5cGUgPSAiZG9oIgoKICBbdXBzdHJlYW0uMV0KICAgIGJvb3RzdHJhcF9pcCA9ICI3Ni43Ni4yLjExIgogICAgZW5kcG9pbnQgPSAicDIuZnJlZWRucy5jb250cm9sZC5jb20iCiAgICBuYW1lID0gIkNvbnRyb2wgRCAtIE5vIEFkcyIKICAgIHRpbWVvdXQgPSAzMDAwCiAgICB0eXBlID0gImRvcSIK"
+```

doh.go

@@ -2,61 +2,30 @@ package ctrld
 
 import (
 	"context"
-	"crypto/tls"
 	"encoding/base64"
+	"errors"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
-	"time"
 
-	"github.com/lucas-clemente/quic-go"
-	"github.com/lucas-clemente/quic-go/http3"
 	"github.com/miekg/dns"
 )
 
 func newDohResolver(uc *UpstreamConfig) *dohResolver {
-	http.DefaultTransport.(*http.Transport).DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
-		dialer := &net.Dialer{
-			Timeout:   10 * time.Second,
-			KeepAlive: 10 * time.Second,
-		}
-		Log(ctx, ProxyLog.Debug(), "debug dial context %s - %s - %s", addr, network, bootstrapDNS)
-		// if we have a bootstrap ip set, use it to avoid DNS lookup
-		if uc.BootstrapIP != "" && addr == fmt.Sprintf("%s:443", uc.Domain) {
-			addr = fmt.Sprintf("%s:443", uc.BootstrapIP)
-			Log(ctx, ProxyLog.Debug(), "sending doh request to: %s", addr)
-		}
-		return dialer.DialContext(ctx, network, addr)
-	}
-	r := &dohResolver{endpoint: uc.Endpoint, isDoH3: uc.Type == resolverTypeDOH3}
-	if r.isDoH3 {
-		r.doh3DialFunc = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error) {
-			host := addr
-			Log(ctx, ProxyLog.Debug(), "debug dial context D0H3 %s - %s", addr, bootstrapDNS)
-			// if we have a bootstrap ip set, use it to avoid DNS lookup
-			if uc.BootstrapIP != "" && addr == fmt.Sprintf("%s:443", uc.Domain) {
-				addr = fmt.Sprintf("%s:443", uc.BootstrapIP)
-				Log(ctx, ProxyLog.Debug(), "sending doh3 request to: %s", addr)
-			}
-			remoteAddr, err := net.ResolveUDPAddr("udp", addr)
-			if err != nil {
-				return nil, err
-			}
-			udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
-			if err != nil {
-				return nil, err
-			}
-			return quic.DialEarlyContext(ctx, udpConn, remoteAddr, host, tlsCfg, cfg)
-		}
+	r := &dohResolver{
+		endpoint:          uc.Endpoint,
+		isDoH3:            uc.Type == ResolverTypeDOH3,
+		transport:         uc.transport,
+		http3RoundTripper: uc.http3RoundTripper,
 	}
 	return r
 }
 
 type dohResolver struct {
-	endpoint     string
-	isDoH3       bool
-	doh3DialFunc func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.EarlyConnection, error)
+	endpoint          string
+	isDoH3            bool
+	transport         *http.Transport
+	http3RoundTripper http.RoundTripper
 }
 
 func (r *dohResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) {
@@ -73,14 +42,20 @@ func (r *dohResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, erro
 	req.Header.Set("Content-Type", "application/dns-message")
 	req.Header.Set("Accept", "application/dns-message")
 
-	c := http.Client{}
+	c := http.Client{Transport: r.transport}
 	if r.isDoH3 {
-		c.Transport = &http3.RoundTripper{}
-		c.Transport.(*http3.RoundTripper).Dial = r.doh3DialFunc
-		defer c.Transport.(*http3.RoundTripper).Close()
+		if r.http3RoundTripper == nil {
+			return nil, errors.New("DoH3 is not supported")
+		}
+		c.Transport = r.http3RoundTripper
 	}
 	resp, err := c.Do(req)
 	if err != nil {
+		if r.isDoH3 {
+			if closer, ok := r.http3RoundTripper.(io.Closer); ok {
+				closer.Close()
+			}
+		}
 		return nil, fmt.Errorf("could not perform request: %w", err)
 	}
 	defer resp.Body.Close()

doq.go

@@ -1,3 +1,5 @@
+//go:build !qf
+
 package ctrld
 
 import (
@@ -7,8 +9,8 @@ import (
 	"net"
 	"time"
 
-	"github.com/lucas-clemente/quic-go"
 	"github.com/miekg/dns"
+	"github.com/quic-go/quic-go"
 )
 
 type doqResolver struct {
@@ -27,6 +29,22 @@ func (r *doqResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, erro
 }
 
 func resolve(ctx context.Context, msg *dns.Msg, endpoint string, tlsConfig *tls.Config) (*dns.Msg, error) {
+	// DoQ quic-go server returns io.EOF error after running for a long time,
+	// even for a good stream. So retrying the query for 5 times before giving up.
+	for i := 0; i < 5; i++ {
+		answer, err := doResolve(ctx, msg, endpoint, tlsConfig)
+		if err == io.EOF {
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		return answer, nil
+	}
+	return nil, &quic.ApplicationError{ErrorCode: quic.ApplicationErrorCode(quic.InternalError), ErrorMessage: quic.InternalError.Message()}
+}
+
+func doResolve(ctx context.Context, msg *dns.Msg, endpoint string, tlsConfig *tls.Config) (*dns.Msg, error) {
 	session, err := quic.DialAddr(endpoint, tlsConfig, nil)
 	if err != nil {
 		return nil, err
@@ -66,6 +84,14 @@ func resolve(ctx context.Context, msg *dns.Msg, endpoint string, tlsConfig *tls.
 
 	_ = stream.Close()
 
+	// io.ReadAll hide the io.EOF error returned by quic-go server.
+	// Once we figure out why quic-go server sends io.EOF after running
+	// for a long time, we can have a better way to handle this. For now,
+	// make sure io.EOF error returned, so the caller can handle it cleanly.
+	if len(buf) == 0 {
+		return nil, io.EOF
+	}
+
 	answer := new(dns.Msg)
 	if err := answer.Unpack(buf[2:]); err != nil {
 		return nil, err

doq_quic_free.go

@@ -0,0 +1,18 @@
+//go:build qf
+
+package ctrld
+
+import (
+	"context"
+	"errors"
+
+	"github.com/miekg/dns"
+)
+
+type doqResolver struct {
+	uc *UpstreamConfig
+}
+
+func (r *doqResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) {
+	return nil, errors.New("DoQ is not supported")
+}

errors.go

@@ -0,0 +1,43 @@
+package ctrld
+
+// TODO(cuonglm): use stdlib once we bump minimum version to 1.20
+
+func joinErrors(errs ...error) error {
+	n := 0
+	for _, err := range errs {
+		if err != nil {
+			n++
+		}
+	}
+	if n == 0 {
+		return nil
+	}
+	e := &joinError{
+		errs: make([]error, 0, n),
+	}
+	for _, err := range errs {
+		if err != nil {
+			e.errs = append(e.errs, err)
+		}
+	}
+	return e
+}
+
+type joinError struct {
+	errs []error
+}
+
+func (e *joinError) Error() string {
+	var b []byte
+	for i, err := range e.errs {
+		if i > 0 {
+			b = append(b, '\n')
+		}
+		b = append(b, err.Error()...)
+	}
+	return string(b)
+}
+
+func (e *joinError) Unwrap() []error {
+	return e.errs
+}

go.mod

@@ -3,51 +3,74 @@ module github.com/Control-D-Inc/ctrld
 go 1.19
 
 require (
+	github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534
+	github.com/frankban/quicktest v1.14.3
+	github.com/fsnotify/fsnotify v1.6.0
 	github.com/go-playground/validator/v10 v10.11.1
+	github.com/godbus/dbus/v5 v5.0.6
+	github.com/hashicorp/golang-lru/v2 v2.0.1
+	github.com/illarion/gonotify v1.0.1
+	github.com/insomniacslk/dhcp v0.0.0-20211209223715-7d93572ebe8e
 	github.com/kardianos/service v1.2.1
-	github.com/lucas-clemente/quic-go v0.29.1
 	github.com/miekg/dns v1.1.50
-	github.com/pelletier/go-toml v1.9.5
+	github.com/pelletier/go-toml/v2 v2.0.6
+	github.com/quic-go/quic-go v0.32.0
 	github.com/rs/zerolog v1.28.0
-	github.com/spf13/cobra v1.1.1
-	github.com/spf13/viper v1.7.0
-	github.com/stretchr/testify v1.7.1
+	github.com/spf13/cobra v1.4.0
+	github.com/spf13/viper v1.14.0
+	github.com/stretchr/testify v1.8.1
+	golang.org/x/sync v0.1.0
+	golang.org/x/sys v0.5.0
+	golang.zx2c4.com/wireguard/windows v0.5.3
+	tailscale.com v1.34.1
 )
 
 require (
+	github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/go-playground/locales v0.14.0 // indirect
 	github.com/go-playground/universal-translator v0.18.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
 	github.com/golang/mock v1.6.0 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/josharian/native v1.0.0 // indirect
+	github.com/jsimonetti/rtnetlink v1.1.2-0.20220408201609-d380b505068b // indirect
+	github.com/kr/pretty v0.3.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
-	github.com/magiconair/properties v1.8.1 // indirect
-	github.com/marten-seemann/qpack v0.2.1 // indirect
-	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
-	github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7 // indirect
+	github.com/mdlayher/netlink v1.6.0 // indirect
+	github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065 // indirect
+	github.com/mdlayher/socket v0.2.3 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/onsi/ginkgo v1.16.5 // indirect
+	github.com/onsi/ginkgo/v2 v2.2.0 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/spf13/afero v1.1.2 // indirect
-	github.com/spf13/cast v1.3.0 // indirect
-	github.com/spf13/jwalterweatherman v1.0.0 // indirect
+	github.com/quic-go/qpack v0.4.0 // indirect
+	github.com/quic-go/qtls-go1-18 v0.2.0 // indirect
+	github.com/quic-go/qtls-go1-19 v0.2.0 // indirect
+	github.com/quic-go/qtls-go1-20 v0.1.0 // indirect
+	github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 // indirect
+	github.com/spf13/afero v1.9.3 // indirect
+	github.com/spf13/cast v1.5.0 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.2.0 // indirect
-	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
-	golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/net v0.0.0-20221004154528-8021a29435af // indirect
-	golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.12 // indirect
-	gopkg.in/ini.v1 v1.51.0 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	github.com/subosito/gotenv v1.4.1 // indirect
+	github.com/u-root/uio v0.0.0-20220204230159-dac05f7d2cb4 // indirect
+	go4.org/mem v0.0.0-20210711025021-927187094b94 // indirect
+	golang.org/x/crypto v0.4.0 // indirect
+	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
+	golang.org/x/mod v0.6.0 // indirect
+	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/tools v0.2.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -3,51 +3,75 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
 cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
 cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
 cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
+cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
+github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/cilium/ebpf v0.8.1 h1:bLSSEbBLqGPXxls55pGr5qWZaTqcmfDJHhou7t254ao=
+github.com/cilium/ebpf v0.8.1/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 h1:rtAn27wIbmOGUs7RIbVgPEjb31ehTVniDwPGXyMxm5U=
 github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
-github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fanliao/go-promise v0.0.0-20141029170127-1890db352a72/go.mod h1:PjfxuH4FZdUyfMdtBio2lsRr1AKEaVPwelzuHuh8Lqc=
+github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
+github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
+github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
@@ -56,83 +80,106 @@ github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/j
 github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
 github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ=
 github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/godbus/dbus/v5 v5.0.6 h1:mkgN1ofwASrYnJ5W6U/BxG15eXXXjirgZc7CLqkcaro=
+github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
-github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
-github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4=
+github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8=
+github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/illarion/gonotify v1.0.1 h1:F1d+0Fgbq/sDWjj/r66ekjDG+IDeecQKUFH4wNwsoio=
+github.com/illarion/gonotify v1.0.1/go.mod h1:zt5pmDofZpU1f8aqlK0+95eQhoEAn/d4G4B/FjVW4jE=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/insomniacslk/dhcp v0.0.0-20211209223715-7d93572ebe8e h1:IQpunlq7T+NiJJMO7ODYV2YWBiv/KnObR3gofX0mWOo=
+github.com/insomniacslk/dhcp v0.0.0-20211209223715-7d93572ebe8e/go.mod h1:h+MxyHxRg9NH3terB1nfRIUaQEcI0XOVkdR9LNBlp8E=
+github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTxk=
+github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
+github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
+github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
+github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok=
+github.com/jsimonetti/rtnetlink v0.0.0-20201110080708-d2c240429e6c/go.mod h1:huN4d1phzjhlOsNIjFsw2SVRbwIHj3fJDMEU2SDPTmg=
+github.com/jsimonetti/rtnetlink v1.1.2-0.20220408201609-d380b505068b h1:Yws7RV6kZr2O7PPdT+RkbSmmOponA8i/1DuGHe8BRsM=
+github.com/jsimonetti/rtnetlink v1.1.2-0.20220408201609-d380b505068b/go.mod h1:TzDCVOZKUa79z6iXbbXqhtAflVgUKaFkZ21M5tK5tzY=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kardianos/service v1.2.1 h1:AYndMsehS+ywIS6RB9KOlcXzteWUzxgMgBymJD7+BYk=
 github.com/kardianos/service v1.2.1/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
@@ -143,141 +190,129 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
-github.com/lucas-clemente/quic-go v0.29.1 h1:Z+WMJ++qMLhvpFkRZA+jl3BTxUjm415YBmWanXB8zP0=
-github.com/lucas-clemente/quic-go v0.29.1/go.mod h1:CTcNfLYJS2UuRNB+zcNlgvkjBhxX6Hm3WUxxAQx2mgE=
-github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs=
-github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
-github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM=
-github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
-github.com/marten-seemann/qtls-go1-19 v0.1.0 h1:rLFKD/9mp/uq1SYGYuVZhm83wkmU95pK5df3GufyYYU=
-github.com/marten-seemann/qtls-go1-19 v0.1.0/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
+github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7 h1:lez6TS6aAau+8wXUP3G9I3TGlmPFEq2CTxBaRqY6AGE=
+github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y=
+github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA=
+github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M=
+github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY=
+github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o=
+github.com/mdlayher/netlink v1.6.0 h1:rOHX5yl7qnlpiVkFWoqccueppMtXzeziFjWAjLg6sz0=
+github.com/mdlayher/netlink v1.6.0/go.mod h1:0o3PlBmGst1xve7wQ7j/hwpNaFaH4qCRyWCdcZk8/vA=
+github.com/mdlayher/raw v0.0.0-20190606142536-fef19f00fc18/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg=
+github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065 h1:aFkJ6lx4FPip+S+Uw4aTegFMct9shDvP+79PsSxpm3w=
+github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg=
+github.com/mdlayher/socket v0.1.1/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs=
+github.com/mdlayher/socket v0.2.3 h1:XZA2X2TjdOwNoNPVPclRCURoX/hokBY8nkTmRZFEheM=
+github.com/mdlayher/socket v0.2.3/go.mod h1:bz12/FozYNH/VbvC3q7TRIK/Y6dH1kCKsXaUeXi/FmY=
 github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
 github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
-github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
-github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
-github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
-github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
-github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
-github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/onsi/ginkgo/v2 v2.2.0 h1:3ZNA3L1c5FYDFTTxbFeVGGD8jYvjYauHD30YgLxVsNI=
+github.com/onsi/ginkgo/v2 v2.2.0/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
+github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
+github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
+github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
+github.com/quic-go/qtls-go1-18 v0.2.0 h1:5ViXqBZ90wpUcZS0ge79rf029yx0dYB0McyPJwqqj7U=
+github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc=
+github.com/quic-go/qtls-go1-19 v0.2.0 h1:Cvn2WdhyViFUHoOqK52i51k4nDX8EwIh5VJiVM4nttk=
+github.com/quic-go/qtls-go1-19 v0.2.0/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI=
+github.com/quic-go/qtls-go1-20 v0.1.0 h1:d1PK3ErFy9t7zxKsG3NXBJXZjp/kMLoIb3y/kV54oAI=
+github.com/quic-go/qtls-go1-20 v0.1.0/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM=
+github.com/quic-go/quic-go v0.32.0 h1:lY02md31s1JgPiiyfqJijpu/UX/Iun304FI3yUqX7tA=
+github.com/quic-go/quic-go v0.32.0/go.mod h1:/fCsKANhQIeD5l76c2JFU+07gVE3KaA0FP+0zMWwfwo=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
-github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
+github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 h1:Ha8xCaq6ln1a+R91Km45Oq6lPXj2Mla6CRJYcuV2h1w=
+github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY=
 github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
-github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk=
+github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y=
+github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
+github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
+github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=
+github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
-github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU=
+github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
-github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs=
+github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
+github.com/u-root/uio v0.0.0-20210528114334-82958018845c/go.mod h1:LpEX5FO/cB+WF4TYGY1V5qktpaZLkKkSegbr0V4eYXA=
+github.com/u-root/uio v0.0.0-20220204230159-dac05f7d2cb4 h1:hl6sK6aFgTLISijk6xIzeqnPzQcsLqqvL6vEfTPinME=
+github.com/u-root/uio v0.0.0-20220204230159-dac05f7d2cb4/go.mod h1:LpEX5FO/cB+WF4TYGY1V5qktpaZLkKkSegbr0V4eYXA=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go4.org/mem v0.0.0-20210711025021-927187094b94 h1:OAAkygi2Js191AJP1Ds42MhJRgeofeKGjuoUqNp1QC4=
+go4.org/mem v0.0.0-20210711025021-927187094b94/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM=
-golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
+golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 h1:tnebWN09GYg9OLPss1KXj8txwZc6X6uMr6VFdcGNbHw=
-golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o=
+golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -287,92 +322,158 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
+golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190419010253-1f3472d942ba/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20221004154528-8021a29435af h1:wv66FM3rLZGPdxpYL+ApnDe2HzHcTFta3z5nsc13wI4=
-golang.org/x/net v0.0.0-20221004154528-8021a29435af/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
-golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc=
-golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -382,32 +483,81 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE=
+golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
+golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -417,41 +567,83 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
 google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
-gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+tailscale.com v1.34.1 h1:tqm9Ww4ltyYp3IPe7vCGch6tT6j5G/WXPQ6BrVZ6pdI=
+tailscale.com v1.34.1/go.mod h1:ZsBP7rjzzB2rp+UCOumr9DAe0EQ6OPivwSXcz/BrekQ=

internal/controld/config.go

@@ -0,0 +1,92 @@
+package controld
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
+)
+
+const (
+	resolverDataURL   = "https://api.controld.com/utility"
+	InvalidConfigCode = 40401
+)
+
+// ResolverConfig represents Control D resolver data.
+type ResolverConfig struct {
+	DOH     string   `json:"doh"`
+	Exclude []string `json:"exclude"`
+}
+
+type utilityResponse struct {
+	Success bool `json:"success"`
+	Body    struct {
+		Resolver ResolverConfig `json:"resolver"`
+	} `json:"body"`
+}
+
+type UtilityErrorResponse struct {
+	ErrorField struct {
+		Message string `json:"message"`
+		Code    int    `json:"code"`
+	} `json:"error"`
+}
+
+func (u UtilityErrorResponse) Error() string {
+	return u.ErrorField.Message
+}
+
+type utilityRequest struct {
+	UID string `json:"uid"`
+}
+
+// FetchResolverConfig fetch Control D config for given uid.
+func FetchResolverConfig(uid string) (*ResolverConfig, error) {
+	body, _ := json.Marshal(utilityRequest{UID: uid})
+	req, err := http.NewRequest("POST", resolverDataURL, bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("http.NewRequest: %w", err)
+	}
+	q := req.URL.Query()
+	q.Set("platform", "ctrld")
+	req.URL.RawQuery = q.Encode()
+	req.Header.Add("Content-Type", "application/json")
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+		// We experiment hanging in TLS handshake when connecting to ControlD API
+		// with ipv6. So prefer ipv4 if available.
+		proto := "tcp6"
+		if ctrldnet.SupportsIPv4() {
+			proto = "tcp4"
+		}
+		return ctrldnet.Dialer.DialContext(ctx, proto, addr)
+	}
+	client := http.Client{
+		Timeout:   10 * time.Second,
+		Transport: transport,
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("client.Do: %w", err)
+	}
+	defer resp.Body.Close()
+	d := json.NewDecoder(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		errResp := &UtilityErrorResponse{}
+		if err := d.Decode(errResp); err != nil {
+			return nil, err
+		}
+		return nil, errResp
+	}
+
+	ur := &utilityResponse{}
+	if err := d.Decode(ur); err != nil {
+		return nil, err
+	}
+	return &ur.Body.Resolver, nil
+}

internal/controld/config_test.go

@@ -0,0 +1,34 @@
+//go:build controld
+
+package controld
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const utilityURL = "https://api.controld.com/utility"
+
+func TestFetchResolverConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		uid     string
+		wantErr bool
+	}{
+		{"valid", "p2", false},
+		{"invalid uid", "abcd1234", true},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got, err := FetchResolverConfig(tc.uid)
+			require.False(t, (err != nil) != tc.wantErr, err)
+			if !tc.wantErr {
+				assert.NotEmpty(t, got.DOH)
+			}
+		})
+	}
+}

internal/dns/README.md

@@ -0,0 +1,2 @@
+This is a fork of https://pkg.go.dev/tailscale.com@v1.34.2/net/dns with modification
+to fit ctrld use case.

internal/dns/debian_resolvconf.go

@@ -0,0 +1,153 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux || freebsd || openbsd
+
+package dns
+
+import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"tailscale.com/atomicfile"
+	"tailscale.com/types/logger"
+)
+
+//go:embed resolvconf-workaround.sh
+var workaroundScript []byte
+
+// resolvconfConfigName is the name of the config submitted to
+// resolvconf.
+// The name starts with 'tun' in order to match the hardcoded
+// interface order in debian resolvconf, which will place this
+// configuration ahead of regular network links. In theory, this
+// doesn't matter because we then fix things up to ensure our config
+// is the only one in use, but in case that fails, this will make our
+// configuration slightly preferred.
+// The 'inet' suffix has no specific meaning, but conventionally
+// resolvconf implementations encourage adding a suffix roughly
+// indicating where the config came from, and "inet" is the "none of
+// the above" value (rather than, say, "ppp" or "dhcp").
+const resolvconfConfigName = "ctrld.inet"
+
+// resolvconfLibcHookPath is the directory containing libc update
+// scripts, which are run by Debian resolvconf when /etc/resolv.conf
+// has been updated.
+const resolvconfLibcHookPath = "/etc/resolvconf/update-libc.d"
+
+// resolvconfHookPath is the name of the libc hook script we install
+// to force Ctrld's DNS config to take effect.
+var resolvconfHookPath = filepath.Join(resolvconfLibcHookPath, "ctrld")
+
+// resolvconfManager manages DNS configuration using the Debian
+// implementation of the `resolvconf` program, written by Thomas Hood.
+type resolvconfManager struct {
+	logf            logger.Logf
+	listRecordsPath string
+	interfacesDir   string
+	scriptInstalled bool // libc update script has been installed
+}
+
+var _ OSConfigurator = (*resolvconfManager)(nil)
+
+func newDebianResolvconfManager(logf logger.Logf) (*resolvconfManager, error) {
+	ret := &resolvconfManager{
+		logf:            logf,
+		listRecordsPath: "/lib/resolvconf/list-records",
+		interfacesDir:   "/etc/resolvconf/run/interface", // panic fallback if nothing seems to work
+	}
+
+	if _, err := os.Stat(ret.listRecordsPath); os.IsNotExist(err) {
+		// This might be a Debian system from before the big /usr
+		// merge, try /usr instead.
+		ret.listRecordsPath = "/usr" + ret.listRecordsPath
+	}
+	// The runtime directory is currently (2020-04) canonically
+	// /etc/resolvconf/run, but the manpage is making noise about
+	// switching to /run/resolvconf and dropping the /etc path. So,
+	// let's probe the possible directories and use the first one
+	// that works.
+	for _, path := range []string{
+		"/etc/resolvconf/run/interface",
+		"/run/resolvconf/interface",
+		"/var/run/resolvconf/interface",
+	} {
+		if _, err := os.Stat(path); err == nil {
+			ret.interfacesDir = path
+			break
+		}
+	}
+	if ret.interfacesDir == "" {
+		// None of the paths seem to work, use the canonical location
+		// that the current manpage says to use.
+		ret.interfacesDir = "/etc/resolvconf/run/interfaces"
+	}
+
+	return ret, nil
+}
+
+func (m *resolvconfManager) deleteCtrldConfig() error {
+	cmd := exec.Command("resolvconf", "-d", resolvconfConfigName)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("running %s: %s", cmd, out)
+	}
+	return nil
+}
+
+func (m *resolvconfManager) SetDNS(config OSConfig) error {
+	if !m.scriptInstalled {
+		m.logf("injecting resolvconf workaround script")
+		if err := os.MkdirAll(resolvconfLibcHookPath, 0755); err != nil {
+			return err
+		}
+		if err := atomicfile.WriteFile(resolvconfHookPath, workaroundScript, 0755); err != nil {
+			return err
+		}
+		m.scriptInstalled = true
+	}
+
+	if config.IsZero() {
+		if err := m.deleteCtrldConfig(); err != nil {
+			return err
+		}
+	} else {
+		stdin := new(bytes.Buffer)
+		writeResolvConf(stdin, config.Nameservers, config.SearchDomains) // dns_direct.go
+
+		// This resolvconf implementation doesn't support exclusive
+		// mode or interface priorities, so it will end up blending
+		// our configuration with other sources. However, this will
+		// get fixed up by the script we injected above.
+		cmd := exec.Command("resolvconf", "-a", resolvconfConfigName)
+		cmd.Stdin = stdin
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("running %s: %s", cmd, out)
+		}
+	}
+
+	return nil
+}
+
+func (m *resolvconfManager) Close() error {
+	if err := m.deleteCtrldConfig(); err != nil {
+		return err
+	}
+
+	if m.scriptInstalled {
+		m.logf("removing resolvconf workaround script")
+		os.Remove(resolvconfHookPath) // Best-effort
+	}
+
+	return nil
+}
+
+func (m *resolvconfManager) Mode() string {
+	return "resolvconf"
+}

internal/dns/direct.go

@@ -0,0 +1,537 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//lint:file-ignore U1000 satisfy CI.
+
+package dns
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/netip"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	"tailscale.com/health"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/dnsname"
+	"tailscale.com/version/distro"
+
+	"github.com/Control-D-Inc/ctrld/internal/dns/resolvconffile"
+)
+
+const (
+	backupConf = "/etc/resolv.pre-ctrld-backup.conf"
+	resolvConf = "/etc/resolv.conf"
+)
+
+// writeResolvConf writes DNS configuration in resolv.conf format to the given writer.
+func writeResolvConf(w io.Writer, servers []netip.Addr, domains []dnsname.FQDN) error {
+	c := &resolvconffile.Config{
+		Nameservers:   servers,
+		SearchDomains: domains,
+	}
+	return c.Write(w)
+}
+
+func readResolv(r io.Reader) (OSConfig, error) {
+	c, err := resolvconffile.Parse(r)
+	if err != nil {
+		return OSConfig{}, err
+	}
+	return OSConfig{
+		Nameservers:   c.Nameservers,
+		SearchDomains: c.SearchDomains,
+	}, nil
+}
+
+// resolvOwner returns the apparent owner of the resolv.conf
+// configuration in bs - one of "resolvconf", "systemd-resolved" or
+// "NetworkManager", or "" if no known owner was found.
+func resolvOwner(bs []byte) string {
+	likely := ""
+	b := bytes.NewBuffer(bs)
+	for {
+		line, err := b.ReadString('\n')
+		if err != nil {
+			return likely
+		}
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		if line[0] != '#' {
+			// First non-empty, non-comment line. Assume the owner
+			// isn't hiding further down.
+			return likely
+		}
+
+		if strings.Contains(line, "systemd-resolved") {
+			likely = "systemd-resolved"
+		} else if strings.Contains(line, "NetworkManager") {
+			likely = "NetworkManager"
+		} else if strings.Contains(line, "resolvconf") {
+			likely = "resolvconf"
+		}
+	}
+}
+
+// isResolvedRunning reports whether systemd-resolved is running on the system,
+// even if it is not managing the system DNS settings.
+func isResolvedRunning() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+
+	// systemd-resolved is never installed without systemd.
+	_, err := exec.LookPath("systemctl")
+	if err != nil {
+		return false
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	err = exec.CommandContext(ctx, "systemctl", "is-active", "systemd-resolved.service").Run()
+
+	// is-active exits with code 3 if the service is not active.
+	return err == nil
+}
+
+func restartResolved() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	return exec.CommandContext(ctx, "systemctl", "restart", "systemd-resolved.service").Run()
+}
+
+// directManager is an OSConfigurator which replaces /etc/resolv.conf with a file
+// generated from the given configuration, creating a backup of its old state.
+//
+// This way of configuring DNS is precarious, since it does not react
+// to the disappearance of the Tailscale interface.
+// The caller must call Down before program shutdown
+// or as cleanup if the program terminates unexpectedly.
+type directManager struct {
+	logf logger.Logf
+	fs   wholeFileFS
+	// renameBroken is set if fs.Rename to or from /etc/resolv.conf
+	// fails. This can happen in some container runtimes, where
+	// /etc/resolv.conf is bind-mounted from outside the container,
+	// and therefore /etc and /etc/resolv.conf are different
+	// filesystems as far as rename(2) is concerned.
+	//
+	// In those situations, we fall back to emulating rename with file
+	// copies and truncations, which is not as good (opens up a race
+	// where a reader can see an empty or partial /etc/resolv.conf),
+	// but is better than having non-functioning DNS.
+	renameBroken bool
+
+	ctx      context.Context    // valid until Close
+	ctxClose context.CancelFunc // closes ctx
+
+	mu               sync.Mutex
+	wantResolvConf   []byte // if non-nil, what we expect /etc/resolv.conf to contain
+	lastWarnContents []byte // last resolv.conf contents that we warned about
+}
+
+func newDirectManager(logf logger.Logf) *directManager {
+	return newDirectManagerOnFS(logf, directFS{})
+}
+
+func newDirectManagerOnFS(logf logger.Logf, fs wholeFileFS) *directManager {
+	ctx, cancel := context.WithCancel(context.Background())
+	m := &directManager{
+		logf:     logf,
+		fs:       fs,
+		ctx:      ctx,
+		ctxClose: cancel,
+	}
+	go m.runFileWatcher()
+	return m
+}
+
+func (m *directManager) readResolvFile(path string) (OSConfig, error) {
+	b, err := m.fs.ReadFile(path)
+	if err != nil {
+		return OSConfig{}, err
+	}
+	return readResolv(bytes.NewReader(b))
+}
+
+// ownedByCtrld reports whether /etc/resolv.conf seems to be a
+// ctrld-managed file.
+func (m *directManager) ownedByCtrld() (bool, error) {
+	isRegular, err := m.fs.Stat(resolvConf)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	if !isRegular {
+		return false, nil
+	}
+	bs, err := m.fs.ReadFile(resolvConf)
+	if err != nil {
+		return false, err
+	}
+	if bytes.Contains(bs, []byte("generated by ctrld")) {
+		return true, nil
+	}
+	return false, nil
+}
+
+// backupConfig creates or updates a backup of /etc/resolv.conf, if
+// resolv.conf does not currently contain a Tailscale-managed config.
+func (m *directManager) backupConfig() error {
+	if _, err := m.fs.Stat(resolvConf); err != nil {
+		if os.IsNotExist(err) {
+			// No resolv.conf, nothing to back up. Also get rid of any
+			// existing backup file, to avoid restoring something old.
+			_ = m.fs.Remove(backupConf)
+			return nil
+		}
+		return err
+	}
+
+	owned, err := m.ownedByCtrld()
+	if err != nil {
+		return err
+	}
+	if owned {
+		return nil
+	}
+
+	return m.rename(resolvConf, backupConf)
+}
+
+func (m *directManager) restoreBackup() (restored bool, err error) {
+	if _, err := m.fs.Stat(backupConf); err != nil {
+		if os.IsNotExist(err) {
+			// No backup, nothing we can do.
+			return false, nil
+		}
+		return false, err
+	}
+	owned, err := m.ownedByCtrld()
+	if err != nil {
+		return false, err
+	}
+	_, err = m.fs.Stat(resolvConf)
+	if err != nil && !os.IsNotExist(err) {
+		return false, err
+	}
+	resolvConfExists := !os.IsNotExist(err)
+
+	if resolvConfExists && !owned {
+		// There's already a non-ctrld config in place, get rid of
+		// our backup.
+		_ = m.fs.Remove(backupConf)
+		return false, nil
+	}
+
+	// We own resolv.conf, and a backup exists.
+	if err := m.rename(backupConf, resolvConf); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
+// rename tries to rename old to new using m.fs.Rename, and falls back
+// to hand-copying bytes and truncating old if that fails.
+//
+// This is a workaround to /etc/resolv.conf being a bind-mounted file
+// some container environments, which cannot be moved elsewhere in
+// /etc (because that would be a cross-filesystem move) or deleted
+// (because that would break the bind in surprising ways).
+func (m *directManager) rename(old, new string) error {
+	if !m.renameBroken {
+		err := m.fs.Rename(old, new)
+		if err == nil {
+			return nil
+		}
+		if runtime.GOOS == "linux" && distro.Get() == distro.Synology {
+			// Fail fast. The fallback case below won't work anyway.
+			return err
+		}
+		m.logf("rename of %q to %q failed (%v), falling back to copy+delete", old, new, err)
+		m.renameBroken = true
+	}
+
+	bs, err := m.fs.ReadFile(old)
+	if err != nil {
+		return fmt.Errorf("reading %q to rename: %w", old, err)
+	}
+	if err := m.fs.WriteFile(new, bs, 0644); err != nil {
+		return fmt.Errorf("writing to %q in rename of %q: %w", new, old, err)
+	}
+
+	if err := m.fs.Remove(old); err != nil {
+		err2 := m.fs.Truncate(old)
+		if err2 != nil {
+			return fmt.Errorf("remove of %q failed (%w) and so did truncate: %v", old, err, err2)
+		}
+	}
+	return nil
+}
+
+// setWant sets the expected contents of /etc/resolv.conf, if any.
+//
+// A value of nil means no particular value is expected.
+//
+// m takes ownership of want.
+func (m *directManager) setWant(want []byte) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.wantResolvConf = want
+}
+
+var warnTrample = health.NewWarnable()
+
+// checkForFileTrample checks whether /etc/resolv.conf has been trampled
+// by another program on the system. (e.g. a DHCP client)
+func (m *directManager) checkForFileTrample() {
+	m.mu.Lock()
+	want := m.wantResolvConf
+	lastWarn := m.lastWarnContents
+	m.mu.Unlock()
+
+	if want == nil {
+		return
+	}
+
+	cur, err := m.fs.ReadFile(resolvConf)
+	if err != nil {
+		m.logf("trample: read error: %v", err)
+		return
+	}
+	if bytes.Equal(cur, want) {
+		warnTrample.Set(nil)
+		if lastWarn != nil {
+			m.mu.Lock()
+			m.lastWarnContents = nil
+			m.mu.Unlock()
+			m.logf("trample: resolv.conf again matches expected content")
+		}
+		return
+	}
+	if bytes.Equal(cur, lastWarn) {
+		// We already logged about this, so not worth doing it again.
+		return
+	}
+
+	m.mu.Lock()
+	m.lastWarnContents = cur
+	m.mu.Unlock()
+
+	show := cur
+	if len(show) > 1024 {
+		show = show[:1024]
+	}
+	m.logf("trample: resolv.conf changed from what we expected. did some other program interfere? current contents: %q", show)
+	//lint:ignore ST1005 This error is for human.
+	warnTrample.Set(errors.New("Linux DNS config not ideal. /etc/resolv.conf overwritten. See https://tailscale.com/s/dns-fight"))
+}
+
+func (m *directManager) SetDNS(config OSConfig) (err error) {
+	defer func() {
+		if err != nil && errors.Is(err, fs.ErrPermission) && runtime.GOOS == "linux" &&
+			distro.Get() == distro.Synology && os.Geteuid() != 0 {
+			// On Synology (notably DSM7 where we don't run as root), ignore all
+			// DNS configuration errors for now. We don't have permission.
+			// See https://github.com/tailscale/tailscale/issues/4017
+			m.logf("ignoring SetDNS permission error on Synology (Issue 4017); was: %v", err)
+			err = nil
+		}
+	}()
+	m.setWant(nil) // reset our expectations before any work
+	var changed bool
+	if config.IsZero() {
+		changed, err = m.restoreBackup()
+		if err != nil {
+			return err
+		}
+	} else {
+		changed = true
+		if err := m.backupConfig(); err != nil {
+			return err
+		}
+
+		buf := new(bytes.Buffer)
+		_ = writeResolvConf(buf, config.Nameservers, config.SearchDomains)
+		if err := m.atomicWriteFile(m.fs, resolvConf, buf.Bytes(), 0644); err != nil {
+			return err
+		}
+
+		// Now that we've successfully written to the file, lock it in.
+		// If we see /etc/resolv.conf with different contents, we know somebody
+		// else trampled on it.
+		m.setWant(buf.Bytes())
+	}
+
+	// We might have taken over a configuration managed by resolved,
+	// in which case it will notice this on restart and gracefully
+	// start using our configuration. This shouldn't happen because we
+	// try to manage DNS through resolved when it's around, but as a
+	// best-effort fallback if we messed up the detection, try to
+	// restart resolved to make the system configuration consistent.
+	//
+	// We take care to only kick systemd-resolved if we've made some
+	// change to the system's DNS configuration, because this codepath
+	// can end up running in cases where the user has manually
+	// configured /etc/resolv.conf to point to systemd-resolved (but
+	// it's not managed explicitly by systemd-resolved), *and* has
+	// --accept-dns=false, meaning we pass an empty configuration to
+	// the running DNS manager. In that very edge-case scenario, we
+	// cause a disruptive DNS outage each time we reset an empty
+	// OS configuration.
+	if changed && isResolvedRunning() && !runningAsGUIDesktopUser() {
+		t0 := time.Now()
+		err := restartResolved()
+		d := time.Since(t0).Round(time.Millisecond)
+		if err != nil {
+			m.logf("error restarting resolved after %v: %v", d, err)
+		} else {
+			m.logf("restarted resolved after %v", d)
+		}
+	}
+
+	return nil
+}
+
+func (m *directManager) Close() error {
+	// We used to keep a file for the ctrld config and symlinked
+	// to it, but then we stopped because /etc/resolv.conf being a
+	// symlink to surprising places breaks snaps and other sandboxing
+	// things. Clean it up if it's still there.
+	_ = m.fs.Remove("/etc/resolv.ctrld.conf")
+
+	if _, err := m.fs.Stat(backupConf); err != nil {
+		if os.IsNotExist(err) {
+			// No backup, nothing we can do.
+			return nil
+		}
+		return err
+	}
+	owned, err := m.ownedByCtrld()
+	if err != nil {
+		return err
+	}
+	_, err = m.fs.Stat(resolvConf)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	resolvConfExists := !os.IsNotExist(err)
+
+	if resolvConfExists && !owned {
+		// There's already a non-ctrld config in place, get rid of
+		// our backup.
+		_ = m.fs.Remove(backupConf)
+		return nil
+	}
+
+	// We own resolv.conf, and a backup exists.
+	if err := m.rename(backupConf, resolvConf); err != nil {
+		return err
+	}
+
+	if isResolvedRunning() && !runningAsGUIDesktopUser() {
+		m.logf("restarting systemd-resolved...")
+		if err := restartResolved(); err != nil {
+			m.logf("restart of systemd-resolved failed: %v", err)
+		} else {
+			m.logf("restarted systemd-resolved")
+		}
+	}
+
+	return nil
+}
+
+func (m *directManager) Mode() string {
+	return "direct"
+}
+
+func (m *directManager) atomicWriteFile(fs wholeFileFS, filename string, data []byte, perm os.FileMode) error {
+	var randBytes [12]byte
+	if _, err := rand.Read(randBytes[:]); err != nil {
+		return fmt.Errorf("atomicWriteFile: %w", err)
+	}
+
+	tmpName := fmt.Sprintf("%s.%x.tmp", filename, randBytes[:])
+	defer fs.Remove(tmpName)
+
+	if err := fs.WriteFile(tmpName, data, perm); err != nil {
+		return fmt.Errorf("atomicWriteFile: %w", err)
+	}
+	return m.rename(tmpName, filename)
+}
+
+// wholeFileFS is a high-level file system abstraction designed just for use
+// by directManager, with the goal that it is easy to implement over wsl.exe.
+//
+// All name parameters are absolute paths.
+type wholeFileFS interface {
+	Stat(name string) (isRegular bool, err error)
+	Rename(oldName, newName string) error
+	Remove(name string) error
+	ReadFile(name string) ([]byte, error)
+	Truncate(name string) error
+	WriteFile(name string, contents []byte, perm os.FileMode) error
+}
+
+// directFS is a wholeFileFS implemented directly on the OS.
+type directFS struct {
+	// prefix is file path prefix.
+	//
+	// All name parameters are absolute paths so this is typically a
+	// testing temporary directory like "/tmp".
+	prefix string
+}
+
+func (fs directFS) path(name string) string { return filepath.Join(fs.prefix, name) }
+
+func (fs directFS) Stat(name string) (isRegular bool, err error) {
+	fi, err := os.Stat(fs.path(name))
+	if err != nil {
+		return false, err
+	}
+	return fi.Mode().IsRegular(), nil
+}
+
+func (fs directFS) Rename(oldName, newName string) error {
+	return os.Rename(fs.path(oldName), fs.path(newName))
+}
+
+func (fs directFS) Remove(name string) error { return os.Remove(fs.path(name)) }
+
+func (fs directFS) ReadFile(name string) ([]byte, error) {
+	return os.ReadFile(fs.path(name))
+}
+
+func (fs directFS) Truncate(name string) error {
+	return os.Truncate(fs.path(name), 0)
+}
+
+func (fs directFS) WriteFile(name string, contents []byte, perm os.FileMode) error {
+	return os.WriteFile(fs.path(name), contents, perm)
+}
+
+// runningAsGUIDesktopUser reports whether it seems that this code is
+// being run as a regular user on a Linux desktop. This is a quick
+// hack to fix Issue 2672 where PolicyKit pops up a GUI dialog asking
+// to proceed we do a best effort attempt to restart
+// systemd-resolved.service. There's surely a better way.
+func runningAsGUIDesktopUser() bool {
+	return os.Getuid() != 0 && os.Getenv("DISPLAY") != ""
+}

internal/dns/direct_linux.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"context"
+
+	"github.com/illarion/gonotify"
+)
+
+func (m *directManager) runFileWatcher() {
+	in, err := gonotify.NewInotify()
+	if err != nil {
+		// Oh well, we tried. This is all best effort for now, to
+		// surface warnings to users.
+		m.logf("dns: inotify new: %v", err)
+		return
+	}
+	ctx, cancel := context.WithCancel(m.ctx)
+	defer cancel()
+	go m.closeInotifyOnDone(ctx, in)
+
+	const events = gonotify.IN_ATTRIB |
+		gonotify.IN_CLOSE_WRITE |
+		gonotify.IN_CREATE |
+		gonotify.IN_DELETE |
+		gonotify.IN_MODIFY |
+		gonotify.IN_MOVE
+
+	if err := in.AddWatch("/etc/", events); err != nil {
+		m.logf("dns: inotify addwatch: %v", err)
+		return
+	}
+	for {
+		events, err := in.Read()
+		if ctx.Err() != nil {
+			return
+		}
+		if err != nil {
+			m.logf("dns: inotify read: %v", err)
+			return
+		}
+		var match bool
+		for _, ev := range events {
+			if ev.Name == resolvConf {
+				match = true
+				break
+			}
+		}
+		if !match {
+			continue
+		}
+		m.checkForFileTrample()
+	}
+}
+
+func (m *directManager) closeInotifyOnDone(ctx context.Context, in *gonotify.Inotify) {
+	<-ctx.Done()
+	_ = in.Close()
+}

internal/dns/direct_notlinux.go

@@ -0,0 +1,11 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !linux
+
+package dns
+
+func (m *directManager) runFileWatcher() {
+	// Not implemented on other platforms. Maybe it could resort to polling.
+}

internal/dns/direct_test.go

@@ -0,0 +1,199 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+	"tailscale.com/util/dnsname"
+)
+
+func TestDirectManager(t *testing.T) {
+	tmp := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(tmp, "etc"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	testDirect(t, directFS{prefix: tmp})
+}
+
+type boundResolvConfFS struct {
+	directFS
+}
+
+func (fs boundResolvConfFS) Rename(old, new string) error {
+	if old == "/etc/resolv.conf" || new == "/etc/resolv.conf" {
+		return errors.New("cannot move to/from /etc/resolv.conf")
+	}
+	return fs.directFS.Rename(old, new)
+}
+
+func (fs boundResolvConfFS) Remove(name string) error {
+	if name == "/etc/resolv.conf" {
+		return errors.New("cannot remove /etc/resolv.conf")
+	}
+	return fs.directFS.Remove(name)
+}
+
+func TestDirectBrokenRename(t *testing.T) {
+	tmp := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(tmp, "etc"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	testDirect(t, boundResolvConfFS{directFS{prefix: tmp}})
+}
+
+func testDirect(t *testing.T, fs wholeFileFS) {
+	const orig = "nameserver 9.9.9.9 # orig"
+	resolvPath := "/etc/resolv.conf"
+	backupPath := "/etc/resolv.pre-ctrld-backup.conf"
+
+	if err := fs.WriteFile(resolvPath, []byte(orig), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	readFile := func(t *testing.T, path string) string {
+		t.Helper()
+		b, err := fs.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return string(b)
+	}
+	assertBaseState := func(t *testing.T) {
+		if got := readFile(t, resolvPath); got != orig {
+			t.Fatalf("resolv.conf:\n%s, want:\n%s", got, orig)
+		}
+		if _, err := fs.Stat(backupPath); !os.IsNotExist(err) {
+			t.Fatalf("resolv.conf backup: want it to be gone but: %v", err)
+		}
+	}
+
+	m := directManager{logf: t.Logf, fs: fs}
+	if err := m.SetDNS(OSConfig{
+		Nameservers:   []netip.Addr{netip.MustParseAddr("8.8.8.8"), netip.MustParseAddr("8.8.4.4")},
+		SearchDomains: []dnsname.FQDN{"controld.com."},
+		MatchDomains:  []dnsname.FQDN{"ignored."},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	want := `# resolv.conf(5) file generated by ctrld
+# DO NOT EDIT THIS FILE BY HAND -- CHANGES WILL BE OVERWRITTEN
+
+nameserver 8.8.8.8
+nameserver 8.8.4.4
+search controld.com
+`
+	if got := readFile(t, resolvPath); got != want {
+		t.Fatalf("resolv.conf:\n%s, want:\n%s", got, want)
+	}
+	if got := readFile(t, backupPath); got != orig {
+		t.Fatalf("resolv.conf backup:\n%s, want:\n%s", got, orig)
+	}
+
+	// Test that a nil OSConfig cleans up resolv.conf.
+	if err := m.SetDNS(OSConfig{}); err != nil {
+		t.Fatal(err)
+	}
+	assertBaseState(t)
+
+	// Test that Close cleans up resolv.conf.
+	if err := m.SetDNS(OSConfig{Nameservers: []netip.Addr{netip.MustParseAddr("8.8.8.8")}}); err != nil {
+		t.Fatal(err)
+	}
+	if err := m.Close(); err != nil {
+		t.Fatal(err)
+	}
+	assertBaseState(t)
+}
+
+type brokenRemoveFS struct {
+	directFS
+}
+
+func (b brokenRemoveFS) Rename(_, _ string) error {
+	return errors.New("nyaaah I'm a silly container!")
+}
+
+func (b brokenRemoveFS) Remove(name string) error {
+	if strings.Contains(name, "/etc/resolv.conf") {
+		return fmt.Errorf("Faking remove failure: %q", &fs.PathError{Err: syscall.EBUSY})
+	}
+	return b.directFS.Remove(name)
+}
+
+func TestDirectBrokenRemove(t *testing.T) {
+	tmp := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(tmp, "etc"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	testDirect(t, brokenRemoveFS{directFS{prefix: tmp}})
+}
+
+func TestReadResolve(t *testing.T) {
+	c := qt.New(t)
+	tests := []struct {
+		in      string
+		want    OSConfig
+		wantErr bool
+	}{
+		{in: `nameserver 192.168.0.100`,
+			want: OSConfig{
+				Nameservers: []netip.Addr{
+					netip.MustParseAddr("192.168.0.100"),
+				},
+			},
+		},
+		{in: `nameserver 192.168.0.100 # comment`,
+			want: OSConfig{
+				Nameservers: []netip.Addr{
+					netip.MustParseAddr("192.168.0.100"),
+				},
+			},
+		},
+		{in: `nameserver 192.168.0.100#`,
+			want: OSConfig{
+				Nameservers: []netip.Addr{
+					netip.MustParseAddr("192.168.0.100"),
+				},
+			},
+		},
+		{in: `nameserver #192.168.0.100`, wantErr: true},
+		{in: `nameserver`, wantErr: true},
+		{in: `# nameserver 192.168.0.100`, want: OSConfig{}},
+		{in: `nameserver192.168.0.100`, wantErr: true},
+
+		{in: `search controld.com`,
+			want: OSConfig{
+				SearchDomains: []dnsname.FQDN{"controld.com."},
+			},
+		},
+		{in: `search controld.com # typo`,
+			want: OSConfig{
+				SearchDomains: []dnsname.FQDN{"controld.com."},
+			},
+		},
+		{in: `searchcontrold.com`, wantErr: true},
+		{in: `search`, wantErr: true},
+	}
+
+	for _, test := range tests {
+		cfg, err := readResolv(strings.NewReader(test.in))
+		if test.wantErr {
+			c.Assert(err, qt.IsNotNil)
+		} else {
+			c.Assert(err, qt.IsNil)
+		}
+		c.Assert(cfg, qt.DeepEquals, test.want)
+	}
+}

internal/dns/manager_freebsd.go

@@ -0,0 +1,39 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"fmt"
+	"os"
+
+	"tailscale.com/types/logger"
+)
+
+func NewOSConfigurator(logf logger.Logf, _ string) (OSConfigurator, error) {
+	bs, err := os.ReadFile("/etc/resolv.conf")
+	if os.IsNotExist(err) {
+		return newDirectManager(logf), nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("reading /etc/resolv.conf: %w", err)
+	}
+
+	switch resolvOwner(bs) {
+	case "resolvconf":
+		switch resolvconfStyle() {
+		case "":
+			return newDirectManager(logf), nil
+		case "debian":
+			return newDebianResolvconfManager(logf)
+		case "openresolv":
+			return newOpenresolvManager()
+		default:
+			logf("[unexpected] got unknown flavor of resolvconf %q, falling back to direct manager", resolvconfStyle())
+			return newDirectManager(logf), nil
+		}
+	default:
+		return newDirectManager(logf), nil
+	}
+}

internal/dns/manager_linux.go

@@ -0,0 +1,391 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/godbus/dbus/v5"
+	"tailscale.com/health"
+	"tailscale.com/net/netaddr"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/util/cmpver"
+)
+
+var _ OSConfigurator = (*directManager)(nil)
+var _ OSConfigurator = (*resolvedManager)(nil)
+var _ OSConfigurator = (*nmManager)(nil)
+
+type kv struct {
+	k, v string
+}
+
+func (kv kv) String() string {
+	return fmt.Sprintf("%s=%s", kv.k, kv.v)
+}
+
+var publishOnce sync.Once
+
+func NewOSConfigurator(logf logger.Logf, interfaceName string) (ret OSConfigurator, err error) {
+	env := newOSConfigEnv{
+		fs:                directFS{},
+		dbusPing:          dbusPing,
+		dbusReadString:    dbusReadString,
+		nmIsUsingResolved: nmIsUsingResolved,
+		nmVersionBetween:  nmVersionBetween,
+		resolvconfStyle:   resolvconfStyle,
+	}
+	mode, err := dnsMode(logf, env)
+	if err != nil {
+		return nil, err
+	}
+	publishOnce.Do(func() {
+		sanitizedMode := strings.ReplaceAll(mode, "-", "_")
+		m := clientmetric.NewGauge(fmt.Sprintf("dns_manager_linux_mode_%s", sanitizedMode))
+		m.Set(1)
+	})
+	logf("dns: using %q mode", mode)
+	switch mode {
+	case "direct":
+		return newDirectManagerOnFS(logf, env.fs), nil
+	case "systemd-resolved":
+		return newResolvedManager(logf, interfaceName)
+	case "network-manager":
+		return newNMManager(interfaceName)
+	case "debian-resolvconf":
+		return newDebianResolvconfManager(logf)
+	case "openresolv":
+		return newOpenresolvManager()
+	default:
+		logf("[unexpected] detected unknown DNS mode %q, using direct manager as last resort", mode)
+		return newDirectManagerOnFS(logf, env.fs), nil
+	}
+}
+
+// newOSConfigEnv are the funcs newOSConfigurator needs, pulled out for testing.
+type newOSConfigEnv struct {
+	fs                wholeFileFS
+	dbusPing          func(string, string) error
+	dbusReadString    func(string, string, string, string) (string, error)
+	nmIsUsingResolved func() error
+	nmVersionBetween  func(v1, v2 string) (safe bool, err error)
+	resolvconfStyle   func() string
+}
+
+func dnsMode(logf logger.Logf, env newOSConfigEnv) (ret string, err error) {
+	var debug []kv
+	dbg := func(k, v string) {
+		debug = append(debug, kv{k, v})
+	}
+	defer func() {
+		if ret != "" {
+			dbg("ret", ret)
+		}
+		logf("dns: %v", debug)
+	}()
+
+	// In all cases that we detect systemd-resolved, try asking it what it
+	// thinks the current resolv.conf mode is so we can add it to our logs.
+	defer func() {
+		if ret != "systemd-resolved" {
+			return
+		}
+
+		// Try to ask systemd-resolved what it thinks the current
+		// status of resolv.conf is. This is documented at:
+		//    https://www.freedesktop.org/software/systemd/man/org.freedesktop.resolve1.html
+		mode, err := env.dbusReadString("org.freedesktop.resolve1", "/org/freedesktop/resolve1", "org.freedesktop.resolve1.Manager", "ResolvConfMode")
+		if err != nil {
+			logf("dns: ResolvConfMode error: %v", err)
+			dbg("resolv-conf-mode", "error")
+		} else {
+			dbg("resolv-conf-mode", mode)
+		}
+	}()
+
+	// Before we read /etc/resolv.conf (which might be in a broken
+	// or symlink-dangling state), try to ping the D-Bus service
+	// for systemd-resolved. If it's active on the machine, this
+	// will make it start up and write the /etc/resolv.conf file
+	// before it replies to the ping. (see how systemd's
+	// src/resolve/resolved.c calls manager_write_resolv_conf
+	// before the sd_event_loop starts)
+	resolvedUp := env.dbusPing("org.freedesktop.resolve1", "/org/freedesktop/resolve1") == nil
+	if resolvedUp {
+		dbg("resolved-ping", "yes")
+	}
+
+	bs, err := env.fs.ReadFile(resolvConf)
+	if os.IsNotExist(err) {
+		dbg("rc", "missing")
+		return "direct", nil
+	}
+	if err != nil {
+		return "", fmt.Errorf("reading /etc/resolv.conf: %w", err)
+	}
+
+	switch resolvOwner(bs) {
+	case "systemd-resolved":
+		dbg("rc", "resolved")
+
+		// Some systems, for reasons known only to them, have a
+		// resolv.conf that has the word "systemd-resolved" in its
+		// header, but doesn't actually point to resolved. We mustn't
+		// try to program resolved in that case.
+		// https://github.com/tailscale/tailscale/issues/2136
+		if err := resolvedIsActuallyResolver(bs); err != nil {
+			logf("dns: resolvedIsActuallyResolver error: %v", err)
+			dbg("resolved", "not-in-use")
+			return "direct", nil
+		}
+		if err := env.dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err != nil {
+			dbg("nm", "no")
+			return "systemd-resolved", nil
+		}
+		dbg("nm", "yes")
+		if err := env.nmIsUsingResolved(); err != nil {
+			dbg("nm-resolved", "no")
+			return "systemd-resolved", nil
+		}
+		dbg("nm-resolved", "yes")
+
+		// Version of NetworkManager before 1.26.6 programmed resolved
+		// incorrectly, such that NM's settings would always take
+		// precedence over other settings set by other resolved
+		// clients.
+		//
+		// If we're dealing with such a version, we have to set our
+		// DNS settings through NM to have them take.
+		//
+		// However, versions 1.26.6 later both fixed the resolved
+		// programming issue _and_ started ignoring DNS settings for
+		// "unmanaged" interfaces - meaning NM 1.26.6 and later
+		// actively ignore DNS configuration we give it. So, for those
+		// NM versions, we can and must use resolved directly.
+		//
+		// Even more fun, even-older versions of NM won't let us set
+		// DNS settings if the interface isn't managed by NM, with a
+		// hard failure on DBus requests. Empirically, NM 1.22 does
+		// this. Based on the versions popular distros shipped, we
+		// conservatively decree that only 1.26.0 through 1.26.5 are
+		// "safe" to use for our purposes. This roughly matches
+		// distros released in the latter half of 2020.
+		//
+		// In a perfect world, we'd avoid this by replacing
+		// configuration out from under NM entirely (e.g. using
+		// directManager to overwrite resolv.conf), but in a world
+		// where resolved runs, we need to get correct configuration
+		// into resolved regardless of what's in resolv.conf (because
+		// resolved can also be queried over dbus, or via an NSS
+		// module that bypasses /etc/resolv.conf). Given that we must
+		// get correct configuration into resolved, we have no choice
+		// but to use NM, and accept the loss of IPv6 configuration
+		// that comes with it (see
+		// https://github.com/tailscale/tailscale/issues/1699,
+		// https://github.com/tailscale/tailscale/pull/1945)
+		safe, err := env.nmVersionBetween("1.26.0", "1.26.5")
+		if err != nil {
+			// Failed to figure out NM's version, can't make a correct
+			// decision.
+			return "", fmt.Errorf("checking NetworkManager version: %v", err)
+		}
+		if safe {
+			dbg("nm-safe", "yes")
+			return "network-manager", nil
+		}
+		dbg("nm-safe", "no")
+		return "systemd-resolved", nil
+	case "resolvconf":
+		dbg("rc", "resolvconf")
+		style := env.resolvconfStyle()
+		switch style {
+		case "":
+			dbg("resolvconf", "no")
+			return "direct", nil
+		case "debian":
+			dbg("resolvconf", "debian")
+			return "debian-resolvconf", nil
+		case "openresolv":
+			dbg("resolvconf", "openresolv")
+			return "openresolv", nil
+		default:
+			// Shouldn't happen, that means we updated flavors of
+			// resolvconf without updating here.
+			dbg("resolvconf", style)
+			logf("[unexpected] got unknown flavor of resolvconf %q, falling back to direct manager", env.resolvconfStyle())
+			return "direct", nil
+		}
+	case "NetworkManager":
+		dbg("rc", "nm")
+		// Sometimes, NetworkManager owns the configuration but points
+		// it at systemd-resolved.
+		if err := resolvedIsActuallyResolver(bs); err != nil {
+			logf("dns: resolvedIsActuallyResolver error: %v", err)
+			dbg("resolved", "not-in-use")
+			// You'd think we would use newNMManager here. However, as
+			// explained in
+			// https://github.com/tailscale/tailscale/issues/1699 ,
+			// using NetworkManager for DNS configuration carries with
+			// it the cost of losing IPv6 configuration on the
+			// Tailscale network interface. So, when we can avoid it,
+			// we bypass NetworkManager by replacing resolv.conf
+			// directly.
+			//
+			// If you ever try to put NMManager back here, keep in mind
+			// that versions >=1.26.6 will ignore DNS configuration
+			// anyway, so you still need a fallback path that uses
+			// directManager.
+			return "direct", nil
+		}
+		dbg("nm-resolved", "yes")
+
+		// See large comment above for reasons we'd use NM rather than
+		// resolved. systemd-resolved is actually in charge of DNS
+		// configuration, but in some cases we might need to configure
+		// it via NetworkManager. All the logic below is probing for
+		// that case: is NetworkManager running? If so, is it one of
+		// the versions that requires direct interaction with it?
+		if err := env.dbusPing("org.freedesktop.NetworkManager", "/org/freedesktop/NetworkManager/DnsManager"); err != nil {
+			dbg("nm", "no")
+			return "systemd-resolved", nil
+		}
+		safe, err := env.nmVersionBetween("1.26.0", "1.26.5")
+		if err != nil {
+			// Failed to figure out NM's version, can't make a correct
+			// decision.
+			return "", fmt.Errorf("checking NetworkManager version: %v", err)
+		}
+		if safe {
+			dbg("nm-safe", "yes")
+			return "network-manager", nil
+		}
+		health.SetDNSManagerHealth(errors.New("systemd-resolved and NetworkManager are wired together incorrectly; MagicDNS will probably not work. For more info, see https://tailscale.com/s/resolved-nm"))
+		dbg("nm-safe", "no")
+		return "systemd-resolved", nil
+	default:
+		dbg("rc", "unknown")
+		return "direct", nil
+	}
+}
+
+func nmVersionBetween(first, last string) (bool, error) {
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		// DBus probably not running.
+		return false, err
+	}
+
+	nm := conn.Object("org.freedesktop.NetworkManager", dbus.ObjectPath("/org/freedesktop/NetworkManager"))
+	v, err := nm.GetProperty("org.freedesktop.NetworkManager.Version")
+	if err != nil {
+		return false, err
+	}
+
+	version, ok := v.Value().(string)
+	if !ok {
+		return false, fmt.Errorf("unexpected type %T for NM version", v.Value())
+	}
+
+	outside := cmpver.Compare(version, first) < 0 || cmpver.Compare(version, last) > 0
+	return !outside, nil
+}
+
+func nmIsUsingResolved() error {
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		// DBus probably not running.
+		return err
+	}
+
+	nm := conn.Object("org.freedesktop.NetworkManager", dbus.ObjectPath("/org/freedesktop/NetworkManager/DnsManager"))
+	v, err := nm.GetProperty("org.freedesktop.NetworkManager.DnsManager.Mode")
+	if err != nil {
+		return fmt.Errorf("getting NM mode: %w", err)
+	}
+	mode, ok := v.Value().(string)
+	if !ok {
+		return fmt.Errorf("unexpected type %T for NM DNS mode", v.Value())
+	}
+	if mode != "systemd-resolved" {
+		return errors.New("NetworkManager is not using systemd-resolved for DNS")
+	}
+	return nil
+}
+
+// resolvedIsActuallyResolver reports whether the given resolv.conf
+// bytes describe a configuration where systemd-resolved (127.0.0.53)
+// is the only configured nameserver.
+//
+// Returns an error if the configuration is something other than
+// exclusively systemd-resolved, or nil if the config is only
+// systemd-resolved.
+func resolvedIsActuallyResolver(bs []byte) error {
+	cfg, err := readResolv(bytes.NewBuffer(bs))
+	if err != nil {
+		return err
+	}
+	// We've encountered at least one system where the line
+	// "nameserver 127.0.0.53" appears twice, so we look exhaustively
+	// through all of them and allow any number of repeated mentions
+	// of the systemd-resolved stub IP.
+	if len(cfg.Nameservers) == 0 {
+		return errors.New("resolv.conf has no nameservers")
+	}
+	for _, ns := range cfg.Nameservers {
+		if ns != netaddr.IPv4(127, 0, 0, 53) {
+			return fmt.Errorf("resolv.conf doesn't point to systemd-resolved; points to %v", cfg.Nameservers)
+		}
+	}
+	return nil
+}
+
+func dbusPing(name, objectPath string) error {
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		// DBus probably not running.
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	obj := conn.Object(name, dbus.ObjectPath(objectPath))
+	call := obj.CallWithContext(ctx, "org.freedesktop.DBus.Peer.Ping", 0)
+	return call.Err
+}
+
+// dbusReadString reads a string property from the provided name and object
+// path. property must be in "interface.member" notation.
+func dbusReadString(name, objectPath, iface, member string) (string, error) {
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		// DBus probably not running.
+		return "", err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	obj := conn.Object(name, dbus.ObjectPath(objectPath))
+
+	var result dbus.Variant
+	err = obj.CallWithContext(ctx, "org.freedesktop.DBus.Properties.Get", 0, iface, member).Store(&result)
+	if err != nil {
+		return "", err
+	}
+
+	if s, ok := result.Value().(string); ok {
+		return s, nil
+	}
+	return result.String(), nil
+}

internal/dns/manager_linux_test.go

@@ -0,0 +1,439 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+	"strings"
+	"testing"
+
+	"tailscale.com/tstest"
+	"tailscale.com/util/cmpver"
+)
+
+func TestLinuxDNSMode(t *testing.T) {
+	tests := []struct {
+		name    string
+		env     newOSConfigEnv
+		wantLog string
+		want    string
+	}{
+		{
+			name:    "no_obvious_resolv.conf_owner",
+			env:     env(resolvDotConf("nameserver 10.0.0.1")),
+			wantLog: "dns: [rc=unknown ret=direct]",
+			want:    "direct",
+		},
+		{
+			name: "network_manager",
+			env: env(
+				resolvDotConf(
+					"# Managed by NetworkManager",
+					"nameserver 10.0.0.1")),
+			wantLog: "dns: resolvedIsActuallyResolver error: resolv.conf doesn't point to systemd-resolved; points to [10.0.0.1]\n" +
+				"dns: [rc=nm resolved=not-in-use ret=direct]",
+			want: "direct",
+		},
+		{
+			name:    "resolvconf_but_no_resolvconf_binary",
+			env:     env(resolvDotConf("# Managed by resolvconf", "nameserver 10.0.0.1")),
+			wantLog: "dns: [rc=resolvconf resolvconf=no ret=direct]",
+			want:    "direct",
+		},
+		{
+			name: "debian_resolvconf",
+			env: env(
+				resolvDotConf("# Managed by resolvconf", "nameserver 10.0.0.1"),
+				resolvconf("debian")),
+			wantLog: "dns: [rc=resolvconf resolvconf=debian ret=debian-resolvconf]",
+			want:    "debian-resolvconf",
+		},
+		{
+			name: "openresolv",
+			env: env(
+				resolvDotConf("# Managed by resolvconf", "nameserver 10.0.0.1"),
+				resolvconf("openresolv")),
+			wantLog: "dns: [rc=resolvconf resolvconf=openresolv ret=openresolv]",
+			want:    "openresolv",
+		},
+		{
+			name: "unknown_resolvconf_flavor",
+			env: env(
+				resolvDotConf("# Managed by resolvconf", "nameserver 10.0.0.1"),
+				resolvconf("daves-discount-resolvconf")),
+			wantLog: "[unexpected] got unknown flavor of resolvconf \"daves-discount-resolvconf\", falling back to direct manager\ndns: [rc=resolvconf resolvconf=daves-discount-resolvconf ret=direct]",
+			want:    "direct",
+		},
+		{
+			name:    "resolved_alone_without_ping",
+			env:     env(resolvDotConf("# Managed by systemd-resolved", "nameserver 127.0.0.53")),
+			wantLog: "dns: ResolvConfMode error: dbus property not found\ndns: [rc=resolved nm=no resolv-conf-mode=error ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			name: "resolved_alone_with_ping",
+			env: env(
+				resolvDotConf("# Managed by systemd-resolved", "nameserver 127.0.0.53"),
+				resolvedRunning()),
+			wantLog: "dns: [resolved-ping=yes rc=resolved nm=no resolv-conf-mode=fortests ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			name: "resolved_and_networkmanager_not_using_resolved",
+			env: env(
+				resolvDotConf("# Managed by systemd-resolved", "nameserver 127.0.0.53"),
+				resolvedRunning(),
+				nmRunning("1.2.3", false)),
+			wantLog: "dns: [resolved-ping=yes rc=resolved nm=yes nm-resolved=no resolv-conf-mode=fortests ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			name: "resolved_and_mid_2020_networkmanager",
+			env: env(
+				resolvDotConf("# Managed by systemd-resolved", "nameserver 127.0.0.53"),
+				resolvedRunning(),
+				nmRunning("1.26.2", true)),
+			wantLog: "dns: [resolved-ping=yes rc=resolved nm=yes nm-resolved=yes nm-safe=yes ret=network-manager]",
+			want:    "network-manager",
+		},
+		{
+			name: "resolved_and_2021_networkmanager",
+			env: env(
+				resolvDotConf("# Managed by systemd-resolved", "nameserver 127.0.0.53"),
+				resolvedRunning(),
+				nmRunning("1.27.0", true)),
+			wantLog: "dns: [resolved-ping=yes rc=resolved nm=yes nm-resolved=yes nm-safe=no resolv-conf-mode=fortests ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			name: "resolved_and_ancient_networkmanager",
+			env: env(
+				resolvDotConf("# Managed by systemd-resolved", "nameserver 127.0.0.53"),
+				resolvedRunning(),
+				nmRunning("1.22.0", true)),
+			wantLog: "dns: [resolved-ping=yes rc=resolved nm=yes nm-resolved=yes nm-safe=no resolv-conf-mode=fortests ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		// Regression tests for extreme corner cases below.
+		{
+			// One user reported a configuration whose comment string
+			// alleged that it was managed by systemd-resolved, but it
+			// was actually a completely static config file pointing
+			// elsewhere.
+			name: "allegedly_resolved_but_not_in_resolv.conf",
+			env:  env(resolvDotConf("# Managed by systemd-resolved", "nameserver 10.0.0.1")),
+			wantLog: "dns: resolvedIsActuallyResolver error: resolv.conf doesn't point to systemd-resolved; points to [10.0.0.1]\n" +
+				"dns: [rc=resolved resolved=not-in-use ret=direct]",
+			want: "direct",
+		},
+		{
+			// We used to incorrectly decide that resolved wasn't in
+			// charge when handed this (admittedly weird and bugged)
+			// resolv.conf.
+			name: "resolved_with_duplicates_in_resolv.conf",
+			env: env(
+				resolvDotConf(
+					"# Managed by systemd-resolved",
+					"nameserver 127.0.0.53",
+					"nameserver 127.0.0.53"),
+				resolvedRunning()),
+			wantLog: "dns: [resolved-ping=yes rc=resolved nm=no resolv-conf-mode=fortests ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			// More than one user has had resolvconf write a config that points to
+			// systemd-resolved. We're better off using systemd-resolved.
+			// regression test for https://github.com/tailscale/tailscale/issues/3026
+			name: "allegedly_resolvconf_but_actually_systemd-resolved",
+			env: env(resolvDotConf(
+				"# Dynamic resolv.conf(5) file for glibc resolver(3) generated by resolvconf(8)",
+				"#     DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN",
+				"# 127.0.0.53 is the systemd-resolved stub resolver.",
+				"# run \"systemd-resolve --status\" to see details about the actual nameservers.",
+				"nameserver 127.0.0.53"),
+				resolvedRunning()),
+			wantLog: "dns: [resolved-ping=yes rc=resolved nm=no resolv-conf-mode=fortests ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			// More than one user has had resolvconf write a config that points to
+			// systemd-resolved. We're better off using systemd-resolved.
+			// and assuming that even if the ping doesn't show that env is correct
+			// regression test for https://github.com/tailscale/tailscale/issues/3026
+			name: "allegedly_resolvconf_but_actually_systemd-resolved_but_no_ping",
+			env: env(resolvDotConf(
+				"# Dynamic resolv.conf(5) file for glibc resolver(3) generated by resolvconf(8)",
+				"#     DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN",
+				"# 127.0.0.53 is the systemd-resolved stub resolver.",
+				"# run \"systemd-resolve --status\" to see details about the actual nameservers.",
+				"nameserver 127.0.0.53")),
+			wantLog: "dns: ResolvConfMode error: dbus property not found\ndns: [rc=resolved nm=no resolv-conf-mode=error ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				resolvedRunning(),
+				nmRunning("1.32.12", true)),
+			wantLog: "dns: [resolved-ping=yes rc=nm nm-resolved=yes nm-safe=no resolv-conf-mode=fortests ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved_but_no_resolved_ping",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				nmRunning("1.32.12", true)),
+			wantLog: "dns: ResolvConfMode error: dbus property not found\ndns: [rc=nm nm-resolved=yes nm-safe=no resolv-conf-mode=error ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved_and_safe_nm",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				resolvedRunning(),
+				nmRunning("1.26.3", true)),
+			wantLog: "dns: [resolved-ping=yes rc=nm nm-resolved=yes nm-safe=yes ret=network-manager]",
+			want:    "network-manager",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3304
+			name: "networkmanager_but_pointing_at_systemd-resolved_and_no_networkmanager",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"nameserver 127.0.0.53",
+				"options edns0 trust-ad"),
+				resolvedRunning()),
+			wantLog: "dns: [resolved-ping=yes rc=nm nm-resolved=yes nm=no resolv-conf-mode=fortests ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			// regression test for https://github.com/tailscale/tailscale/issues/3531
+			name: "networkmanager_but_systemd-resolved_with_search_domain",
+			env: env(resolvDotConf(
+				"# Generated by NetworkManager",
+				"search lan",
+				"nameserver 127.0.0.53"),
+				resolvedRunning()),
+			wantLog: "dns: [resolved-ping=yes rc=nm nm-resolved=yes nm=no resolv-conf-mode=fortests ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+		{
+			// Make sure that we ping systemd-resolved to let it start up and write its resolv.conf
+			// before we read its file.
+			env: env(resolvedStartOnPingAndThen(
+				resolvDotConf("# Managed by systemd-resolved", "nameserver 127.0.0.53"),
+				resolvedDbusProperty(),
+			)),
+			wantLog: "dns: [resolved-ping=yes rc=resolved nm=no resolv-conf-mode=fortests ret=systemd-resolved]",
+			want:    "systemd-resolved",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var logBuf tstest.MemLogger
+			got, err := dnsMode(logBuf.Logf, tt.env)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got != tt.want {
+				t.Errorf("got %s; want %s", got, tt.want)
+			}
+			if got := strings.TrimSpace(logBuf.String()); got != tt.wantLog {
+				t.Errorf("log output mismatch:\n got: %q\nwant: %q\n", got, tt.wantLog)
+			}
+		})
+	}
+}
+
+type memFS map[string]any // full path => string for regular files
+
+func (m memFS) Stat(name string) (isRegular bool, err error) {
+	v, ok := m[name]
+	if !ok {
+		return false, fs.ErrNotExist
+	}
+	if _, ok := v.(string); ok {
+		return true, nil
+	}
+	return false, nil
+}
+
+func (m memFS) Rename(_, _ string) error { panic("TODO") }
+func (m memFS) Remove(_ string) error    { panic("TODO") }
+func (m memFS) ReadFile(name string) ([]byte, error) {
+	v, ok := m[name]
+	if !ok {
+		return nil, fs.ErrNotExist
+	}
+	if s, ok := v.(string); ok {
+		return []byte(s), nil
+	}
+	panic("TODO")
+}
+
+func (m memFS) Truncate(name string) error {
+	v, ok := m[name]
+	if !ok {
+		return fs.ErrNotExist
+	}
+	if s, ok := v.(string); ok {
+		m[name] = s[:0]
+	}
+
+	return nil
+}
+
+func (m memFS) WriteFile(name string, contents []byte, _ os.FileMode) error {
+	m[name] = string(contents)
+	return nil
+}
+
+type dbusService struct {
+	name, path string
+	hook       func() // if non-nil, run on ping
+}
+
+type dbusProperty struct {
+	name, path    string
+	iface, member string
+	hook          func() (string, error) // what to return
+}
+
+type envBuilder struct {
+	fs              memFS
+	dbus            []dbusService
+	dbusProperties  []dbusProperty
+	nmUsingResolved bool
+	nmVersion       string
+	resolvconfStyle string
+}
+
+type envOption interface {
+	apply(*envBuilder)
+}
+
+type envOpt func(*envBuilder)
+
+func (e envOpt) apply(b *envBuilder) {
+	e(b)
+}
+
+func env(opts ...envOption) newOSConfigEnv {
+	b := &envBuilder{
+		fs: memFS{},
+	}
+	for _, opt := range opts {
+		opt.apply(b)
+	}
+
+	return newOSConfigEnv{
+		fs: b.fs,
+		dbusPing: func(name, path string) error {
+			for _, svc := range b.dbus {
+				if svc.name == name && svc.path == path {
+					if svc.hook != nil {
+						svc.hook()
+					}
+					return nil
+				}
+			}
+			return errors.New("dbus service not found")
+		},
+		dbusReadString: func(name, path, iface, member string) (string, error) {
+			for _, svc := range b.dbusProperties {
+				if svc.name == name && svc.path == path && svc.iface == iface && svc.member == member {
+					return svc.hook()
+				}
+			}
+			return "", errors.New("dbus property not found")
+		},
+		nmIsUsingResolved: func() error {
+			if !b.nmUsingResolved {
+				return errors.New("networkmanager not using resolved")
+			}
+			return nil
+		},
+		nmVersionBetween: func(first, last string) (bool, error) {
+			outside := cmpver.Compare(b.nmVersion, first) < 0 || cmpver.Compare(b.nmVersion, last) > 0
+			return !outside, nil
+		},
+		resolvconfStyle: func() string { return b.resolvconfStyle },
+	}
+}
+
+func resolvDotConf(ss ...string) envOption {
+	return envOpt(func(b *envBuilder) {
+		b.fs["/etc/resolv.conf"] = strings.Join(ss, "\n")
+	})
+}
+
+// resolvedRunning returns an option that makes resolved reply to a dbusPing
+// and the ResolvConfMode property.
+func resolvedRunning() envOption {
+	return resolvedStartOnPingAndThen(resolvedDbusProperty())
+}
+
+// resolvedDbusProperty returns an option that responds to the ResolvConfMode
+// property that resolved exposes.
+func resolvedDbusProperty() envOption {
+	return setDbusProperty("org.freedesktop.resolve1", "/org/freedesktop/resolve1", "org.freedesktop.resolve1.Manager", "ResolvConfMode", "fortests")
+}
+
+// resolvedStartOnPingAndThen returns an option that makes resolved be
+// active but not yet running.  On a dbus ping, it then applies the
+// provided options.
+func resolvedStartOnPingAndThen(opts ...envOption) envOption {
+	return envOpt(func(b *envBuilder) {
+		b.dbus = append(b.dbus, dbusService{
+			name: "org.freedesktop.resolve1",
+			path: "/org/freedesktop/resolve1",
+			hook: func() {
+				for _, opt := range opts {
+					opt.apply(b)
+				}
+			},
+		})
+	})
+}
+
+func nmRunning(version string, usingResolved bool) envOption {
+	return envOpt(func(b *envBuilder) {
+		b.nmUsingResolved = usingResolved
+		b.nmVersion = version
+		b.dbus = append(b.dbus, dbusService{name: "org.freedesktop.NetworkManager", path: "/org/freedesktop/NetworkManager/DnsManager"})
+	})
+}
+
+func resolvconf(s string) envOption {
+	return envOpt(func(b *envBuilder) {
+		b.resolvconfStyle = s
+	})
+}
+
+func setDbusProperty(name, path, iface, member, value string) envOption {
+	return envOpt(func(b *envBuilder) {
+		b.dbusProperties = append(b.dbusProperties, dbusProperty{
+			name:   name,
+			path:   path,
+			iface:  iface,
+			member: member,
+			hook: func() (string, error) {
+				return value, nil
+			},
+		})
+	})
+}

internal/dns/nm.go

@@ -0,0 +1,271 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux
+
+package dns
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"time"
+
+	"github.com/godbus/dbus/v5"
+	"tailscale.com/util/dnsname"
+	"tailscale.com/util/endian"
+)
+
+const (
+	highestPriority = int32(-1 << 31)
+	mediumPriority  = int32(1)   // Highest priority that doesn't hard-override
+	lowerPriority   = int32(200) // lower than all builtin auto priorities
+)
+
+// nmManager uses the NetworkManager DBus API.
+type nmManager struct {
+	interfaceName string
+	manager       dbus.BusObject
+	dnsManager    dbus.BusObject
+}
+
+var _ OSConfigurator = (*nmManager)(nil)
+
+func newNMManager(interfaceName string) (*nmManager, error) {
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		return nil, err
+	}
+
+	return &nmManager{
+		interfaceName: interfaceName,
+		manager:       conn.Object("org.freedesktop.NetworkManager", dbus.ObjectPath("/org/freedesktop/NetworkManager")),
+		dnsManager:    conn.Object("org.freedesktop.NetworkManager", dbus.ObjectPath("/org/freedesktop/NetworkManager/DnsManager")),
+	}, nil
+}
+
+type nmConnectionSettings map[string]map[string]dbus.Variant
+
+func (m *nmManager) SetDNS(config OSConfig) error {
+	ctx, cancel := context.WithTimeout(context.Background(), reconfigTimeout)
+	defer cancel()
+
+	// NetworkManager only lets you set DNS settings on "active"
+	// connections, which requires an assigned IP address. This got
+	// configured before the DNS manager was invoked, but it might
+	// take a little time for the netlink notifications to propagate
+	// up. So, keep retrying for the duration of the reconfigTimeout.
+	var err error
+	for ctx.Err() == nil {
+		err = m.trySet(ctx, config)
+		if err == nil {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	return err
+}
+
+func (m *nmManager) trySet(ctx context.Context, config OSConfig) error {
+	conn, err := dbus.SystemBus()
+	if err != nil {
+		return fmt.Errorf("connecting to system bus: %w", err)
+	}
+
+	// This is how we get at the DNS settings:
+	//
+	//               org.freedesktop.NetworkManager
+	//                              |
+	//                    [GetDeviceByIpIface]
+	//                              |
+	//                              v
+	//           org.freedesktop.NetworkManager.Device <--------\
+	//              (describes a network interface)             |
+	//                              |                           |
+	//                   [GetAppliedConnection]             [Reapply]
+	//                              |                           |
+	//                              v                           |
+	//          org.freedesktop.NetworkManager.Connection       |
+	//                   (connection settings)            ------/
+	//          contains {dns, dns-priority, dns-search}
+	//
+	// Ref: https://developer.gnome.org/NetworkManager/stable/settings-ipv4.html.
+
+	nm := conn.Object(
+		"org.freedesktop.NetworkManager",
+		dbus.ObjectPath("/org/freedesktop/NetworkManager"),
+	)
+
+	var devicePath dbus.ObjectPath
+	err = nm.CallWithContext(
+		ctx, "org.freedesktop.NetworkManager.GetDeviceByIpIface", 0,
+		m.interfaceName,
+	).Store(&devicePath)
+	if err != nil {
+		return fmt.Errorf("getDeviceByIpIface: %w", err)
+	}
+	device := conn.Object("org.freedesktop.NetworkManager", devicePath)
+
+	var (
+		settings nmConnectionSettings
+		version  uint64
+	)
+	err = device.CallWithContext(
+		ctx, "org.freedesktop.NetworkManager.Device.GetAppliedConnection", 0,
+		uint32(0),
+	).Store(&settings, &version)
+	if err != nil {
+		return fmt.Errorf("getAppliedConnection: %w", err)
+	}
+
+	// Frustratingly, NetworkManager represents IPv4 addresses as uint32s,
+	// although IPv6 addresses are represented as byte arrays.
+	// Perform the conversion here.
+	var (
+		dnsv4 []uint32
+		dnsv6 [][]byte
+	)
+	for _, ip := range config.Nameservers {
+		b := ip.As16()
+		if ip.Is4() {
+			dnsv4 = append(dnsv4, endian.Native.Uint32(b[12:]))
+		} else {
+			dnsv6 = append(dnsv6, b[:])
+		}
+	}
+
+	// NetworkManager wipes out IPv6 address configuration unless we
+	// tell it explicitly to keep it. Read out the current interface
+	// settings and mirror them out to NetworkManager.
+	var addrs6 []map[string]any
+	if netIface, err := net.InterfaceByName(m.interfaceName); err == nil {
+		if addrs, err := netIface.Addrs(); err == nil {
+			for _, a := range addrs {
+				if ipnet, ok := a.(*net.IPNet); ok {
+					nip, ok := netip.AddrFromSlice(ipnet.IP)
+					nip = nip.Unmap()
+					if ok && nip.Is6() {
+						addrs6 = append(addrs6, map[string]any{
+							"address": nip.String(),
+							"prefix":  uint32(128),
+						})
+					}
+				}
+			}
+		}
+	}
+
+	seen := map[dnsname.FQDN]bool{}
+	var search []string
+	for _, dom := range config.SearchDomains {
+		if seen[dom] {
+			continue
+		}
+		seen[dom] = true
+		search = append(search, dom.WithTrailingDot())
+	}
+	for _, dom := range config.MatchDomains {
+		if seen[dom] {
+			continue
+		}
+		seen[dom] = true
+		search = append(search, "~"+dom.WithTrailingDot())
+	}
+	if len(config.MatchDomains) == 0 {
+		// Non-split routing requested, add an all-domains match.
+		search = append(search, "~.")
+	}
+
+	// Ideally we would like to disable LLMNR and mdns on the
+	// interface here, but older NetworkManagers don't understand
+	// those settings and choke on them, so we don't. Both LLMNR and
+	// mdns will fail since tailscale0 doesn't do multicast, so it's
+	// effectively fine. We used to try and enforce LLMNR and mdns
+	// settings here, but that led to #1870.
+
+	ipv4Map := settings["ipv4"]
+	ipv4Map["dns"] = dbus.MakeVariant(dnsv4)
+	ipv4Map["dns-search"] = dbus.MakeVariant(search)
+	// We should only request priority if we have nameservers to set.
+	if len(dnsv4) == 0 {
+		ipv4Map["dns-priority"] = dbus.MakeVariant(lowerPriority)
+	} else if len(config.MatchDomains) > 0 {
+		// Set a fairly high priority, but don't override all other
+		// configs when in split-DNS mode.
+		ipv4Map["dns-priority"] = dbus.MakeVariant(mediumPriority)
+	} else {
+		// Negative priority means only the settings from the most
+		// negative connection get used. The way this mixes with
+		// per-domain routing is unclear, but it _seems_ that the
+		// priority applies after routing has found possible
+		// candidates for a resolution.
+		ipv4Map["dns-priority"] = dbus.MakeVariant(highestPriority)
+	}
+
+	ipv6Map := settings["ipv6"]
+	// In IPv6 settings, you're only allowed to provide additional
+	// static DNS settings in "auto" (SLAAC) or "manual" mode. In
+	// "manual" mode you also have to specify IP addresses, so we use
+	// "auto".
+	//
+	// NM actually documents that to set just DNS servers, you should
+	// use "auto" mode and then set ignore auto routes and DNS, which
+	// basically means "autoconfigure but ignore any autoconfiguration
+	// results you might get". As a safety, we also say that
+	// NetworkManager should never try to make us the default route
+	// (none of its business anyway, we handle our own default
+	// routing).
+	ipv6Map["method"] = dbus.MakeVariant("auto")
+	if len(addrs6) > 0 {
+		ipv6Map["address-data"] = dbus.MakeVariant(addrs6)
+	}
+	ipv6Map["ignore-auto-routes"] = dbus.MakeVariant(true)
+	ipv6Map["ignore-auto-dns"] = dbus.MakeVariant(true)
+	ipv6Map["never-default"] = dbus.MakeVariant(true)
+
+	ipv6Map["dns"] = dbus.MakeVariant(dnsv6)
+	ipv6Map["dns-search"] = dbus.MakeVariant(search)
+	if len(dnsv6) == 0 {
+		ipv6Map["dns-priority"] = dbus.MakeVariant(lowerPriority)
+	} else if len(config.MatchDomains) > 0 {
+		// Set a fairly high priority, but don't override all other
+		// configs when in split-DNS mode.
+		ipv6Map["dns-priority"] = dbus.MakeVariant(mediumPriority)
+	} else {
+		ipv6Map["dns-priority"] = dbus.MakeVariant(highestPriority)
+	}
+
+	// deprecatedProperties are the properties in interface settings
+	// that are deprecated by NetworkManager.
+	//
+	// In practice, this means that they are returned for reading,
+	// but submitting a settings object with them present fails
+	// with hard-to-diagnose errors. They must be removed.
+	deprecatedProperties := []string{
+		"addresses", "routes",
+	}
+
+	for _, property := range deprecatedProperties {
+		delete(ipv4Map, property)
+		delete(ipv6Map, property)
+	}
+
+	if call := device.CallWithContext(ctx, "org.freedesktop.NetworkManager.Device.Reapply", 0, settings, version, uint32(0)); call.Err != nil {
+		return fmt.Errorf("reapply: %w", call.Err)
+	}
+
+	return nil
+}
+
+func (m *nmManager) Close() error {
+	// No need to do anything on close, NetworkManager will delete our
+	// settings when the tailscale interface goes away.
+	return nil
+}
+
+func (m *nmManager) Mode() string {
+	return "network-maanger"
+}

internal/dns/openresolv.go

@@ -0,0 +1,57 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux || freebsd || openbsd
+
+package dns
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+)
+
+// openresolvManager manages DNS configuration using the openresolv
+// implementation of the `resolvconf` program.
+type openresolvManager struct{}
+
+var _ OSConfigurator = (*openresolvManager)(nil)
+
+func newOpenresolvManager() (openresolvManager, error) {
+	return openresolvManager{}, nil
+}
+
+func (m openresolvManager) deleteTailscaleConfig() error {
+	cmd := exec.Command("resolvconf", "-f", "-d", "ctrld")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("running %s: %s", cmd, out)
+	}
+	return nil
+}
+
+func (m openresolvManager) SetDNS(config OSConfig) error {
+	if config.IsZero() {
+		return m.deleteTailscaleConfig()
+	}
+
+	var stdin bytes.Buffer
+	writeResolvConf(&stdin, config.Nameservers, config.SearchDomains)
+
+	cmd := exec.Command("resolvconf", "-m", "0", "-x", "-a", "ctrld")
+	cmd.Stdin = &stdin
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("running %s: %s", cmd, out)
+	}
+	return nil
+}
+
+func (m openresolvManager) Close() error {
+	return m.deleteTailscaleConfig()
+}
+
+func (m openresolvManager) Mode() string {
+	return "resolvconf"
+}

internal/dns/osconfig.go

@@ -0,0 +1,126 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"bufio"
+	"fmt"
+	"net/netip"
+
+	"tailscale.com/types/logger"
+	"tailscale.com/util/dnsname"
+)
+
+var _ OSConfigurator = (*directManager)(nil)
+
+// An OSConfigurator applies DNS settings to the operating system.
+type OSConfigurator interface {
+	// SetDNS updates the OS's DNS configuration to match cfg.
+	// If cfg is the zero value, all ctrld-related DNS
+	// configuration is removed.
+	// SetDNS must not be called after Close.
+	// SetDNS takes ownership of cfg.
+	SetDNS(cfg OSConfig) error
+
+	// Close removes ctrld-related DNS configuration from the OS.
+	Close() error
+
+	Mode() string
+}
+
+// HostEntry represents a single line in the OS's hosts file.
+type HostEntry struct {
+	Addr  netip.Addr
+	Hosts []string
+}
+
+// OSConfig is an OS DNS configuration.
+type OSConfig struct {
+	// Hosts is a map of DNS FQDNs to their IPs, which should be added to the
+	// OS's hosts file. Currently, (2022-08-12) it is only populated for Windows
+	// in SplitDNS mode and with Smart Name Resolution turned on.
+	Hosts []*HostEntry
+	// Nameservers are the IP addresses of the nameservers to use.
+	Nameservers []netip.Addr
+	// SearchDomains are the domain suffixes to use when expanding
+	// single-label name queries. SearchDomains is additive to
+	// whatever non-Tailscale search domains the OS has.
+	SearchDomains []dnsname.FQDN
+	// MatchDomains are the DNS suffixes for which Nameservers should
+	// be used. If empty, Nameservers is installed as the "primary" resolver.
+	MatchDomains []dnsname.FQDN
+}
+
+func (o OSConfig) IsZero() bool {
+	return len(o.Nameservers) == 0 && len(o.SearchDomains) == 0 && len(o.MatchDomains) == 0
+}
+
+func (a OSConfig) Equal(b OSConfig) bool {
+	if len(a.Nameservers) != len(b.Nameservers) {
+		return false
+	}
+	if len(a.SearchDomains) != len(b.SearchDomains) {
+		return false
+	}
+	if len(a.MatchDomains) != len(b.MatchDomains) {
+		return false
+	}
+
+	for i := range a.Nameservers {
+		if a.Nameservers[i] != b.Nameservers[i] {
+			return false
+		}
+	}
+	for i := range a.SearchDomains {
+		if a.SearchDomains[i] != b.SearchDomains[i] {
+			return false
+		}
+	}
+	for i := range a.MatchDomains {
+		if a.MatchDomains[i] != b.MatchDomains[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+// Format implements the fmt.Formatter interface to ensure that Hosts is
+// printed correctly (i.e. not as a bunch of pointers).
+//
+// Fixes https://github.com/tailscale/tailscale/issues/5669
+func (a OSConfig) Format(f fmt.State, verb rune) {
+	logger.ArgWriter(func(w *bufio.Writer) {
+		_, _ = w.WriteString(`{Nameservers:[`)
+		for i, ns := range a.Nameservers {
+			if i != 0 {
+				_, _ = w.WriteString(" ")
+			}
+			_, _ = fmt.Fprintf(w, "%+v", ns)
+		}
+		_, _ = w.WriteString(`] SearchDomains:[`)
+		for i, domain := range a.SearchDomains {
+			if i != 0 {
+				_, _ = w.WriteString(" ")
+			}
+			_, _ = fmt.Fprintf(w, "%+v", domain)
+		}
+		_, _ = w.WriteString(`] MatchDomains:[`)
+		for i, domain := range a.MatchDomains {
+			if i != 0 {
+				_, _ = w.WriteString(" ")
+			}
+			_, _ = fmt.Fprintf(w, "%+v", domain)
+		}
+		_, _ = w.WriteString(`] Hosts:[`)
+		for i, host := range a.Hosts {
+			if i != 0 {
+				_, _ = w.WriteString(" ")
+			}
+			_, _ = fmt.Fprintf(w, "%+v", host)
+		}
+		_, _ = w.WriteString(`]}`)
+	}).Format(f, verb)
+}

internal/dns/osconfig_test.go

@@ -0,0 +1,44 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dns
+
+import (
+	"fmt"
+	"net/netip"
+	"testing"
+
+	"tailscale.com/util/dnsname"
+)
+
+func TestOSConfigPrintable(t *testing.T) {
+	ocfg := OSConfig{
+		Hosts: []*HostEntry{
+			{
+				Addr:  netip.AddrFrom4([4]byte{100, 1, 2, 3}),
+				Hosts: []string{"server", "client"},
+			},
+			{
+				Addr:  netip.AddrFrom4([4]byte{100, 1, 2, 4}),
+				Hosts: []string{"otherhost"},
+			},
+		},
+		Nameservers: []netip.Addr{
+			netip.AddrFrom4([4]byte{8, 8, 8, 8}),
+		},
+		SearchDomains: []dnsname.FQDN{
+			dnsname.FQDN("foo.beta.controld.com."),
+			dnsname.FQDN("bar.beta.controld.com."),
+		},
+		MatchDomains: []dnsname.FQDN{
+			dnsname.FQDN("controld.com."),
+		},
+	}
+	s := fmt.Sprintf("%+v", ocfg)
+
+	const expected = `{Nameservers:[8.8.8.8] SearchDomains:[foo.beta.controld.com. bar.beta.controld.com.] MatchDomains:[controld.com.] Hosts:[&{Addr:100.1.2.3 Hosts:[server client]} &{Addr:100.1.2.4 Hosts:[otherhost]}]}`
+	if s != expected {
+		t.Errorf("format mismatch:\n   got: %s\n  want: %s", s, expected)
+	}
+}

internal/dns/resolvconf-workaround.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+# Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+#
+# This script is a workaround for a vpn-unfriendly behavior of the
+# original resolvconf by Thomas Hood. Unlike the `openresolv`
+# implementation (whose binary is also called resolvconf,
+# confusingly), the original resolvconf lacks a way to specify
+# "exclusive mode" for a provider configuration. In practice, this
+# means that if Ctrld wants to install a DNS configuration, that
+# config will get "blended" with the configs from other sources,
+# rather than override those other sources.
+#
+# This script gets installed at /etc/resolvconf/update-libc.d, which
+# is a directory of hook scripts that get run after resolvconf's libc
+# helper has finished rewriting /etc/resolv.conf. It's meant to notify
+# consumers of resolv.conf of a new configuration.
+#
+# Instead, we use that hook mechanism to reach into resolvconf's
+# stuff, and rewrite the libc-generated resolv.conf to exclusively
+# contain Ctrld's configuration - effectively implementing
+# exclusive mode ourselves in post-production.
+
+set -e
+
+if [ -n "$CTRLD_RESOLVCONF_HOOK_LOOP" ]; then
+	# Hook script being invoked by itself, skip.
+	exit 0
+fi
+
+if [ ! -f ctrld.inet ]; then
+	# Ctrld isn't trying to manage DNS, do nothing.
+	exit 0
+fi
+
+if ! grep resolvconf /etc/resolv.conf >/dev/null; then
+	# resolvconf isn't managing /etc/resolv.conf, do nothing.
+	exit 0
+fi
+
+# Write out a modified /etc/resolv.conf containing just our config.
+(
+	if [ -f /etc/resolvconf/resolv.conf.d/head ]; then
+		cat /etc/resolvconf/resolv.conf.d/head
+	fi
+	echo "# Ctrld workaround applied to set exclusive DNS configuration."
+	cat tun-tailscale.inet
+	if [ -f /etc/resolvconf/resolv.conf.d/base ]; then
+		# Keep options and sortlist, discard other base things since
+		# they're the things we're trying to override.
+		grep -e 'sortlist ' -e 'options ' /etc/resolvconf/resolv.conf.d/base || true
+	fi
+	if [ -f /etc/resolvconf/resolv.conf.d/tail ]; then
+		cat /etc/resolvconf/resolv.conf.d/tail
+	fi
+) >/etc/resolv.conf
+
+if [ -d /etc/resolvconf/update-libc.d ] ; then
+	# Re-notify libc watchers that we've changed resolv.conf again.
+	export CTRLD_RESOLVCONF_HOOK_LOOP=1
+	exec run-parts /etc/resolvconf/update-libc.d
+fi

internal/dns/resolvconf.go

@@ -0,0 +1,26 @@
+// Copyright (c) 2021 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux || freebsd || openbsd
+
+package dns
+
+import (
+	"os/exec"
+)
+
+func resolvconfStyle() string {
+	if _, err := exec.LookPath("resolvconf"); err != nil {
+		return ""
+	}
+	if _, err := exec.Command("resolvconf", "--version").CombinedOutput(); err != nil {
+		// Debian resolvconf doesn't understand --version, and
+		// exits with a specific error code.
+		if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 99 {
+			return "debian"
+		}
+	}
+	// Treat everything else as openresolv, by far the more popular implementation.
+	return "openresolv"
+}

internal/dns/resolvconffile/resolvconffile.go

@@ -0,0 +1,119 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package resolvconffile parses & serializes /etc/resolv.conf-style files.
+package resolvconffile
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"net/netip"
+	"os"
+	"strings"
+
+	"tailscale.com/util/dnsname"
+	"tailscale.com/util/strs"
+)
+
+// Path is the canonical location of resolv.conf.
+const Path = "/etc/resolv.conf"
+
+// Config represents a resolv.conf(5) file.
+type Config struct {
+	// Nameservers are the IP addresses of the nameservers to use.
+	Nameservers []netip.Addr
+
+	// SearchDomains are the domain suffixes to use when expanding
+	// single-label name queries. SearchDomains is additive to
+	// whatever non-Tailscale search domains the OS has.
+	SearchDomains []dnsname.FQDN
+}
+
+// Write writes c to w. It does so in one Write call.
+func (c *Config) Write(w io.Writer) error {
+	buf := new(bytes.Buffer)
+	io.WriteString(buf, "# resolv.conf(5) file generated by ctrld\n")
+	io.WriteString(buf, "# DO NOT EDIT THIS FILE BY HAND -- CHANGES WILL BE OVERWRITTEN\n\n")
+	for _, ns := range c.Nameservers {
+		io.WriteString(buf, "nameserver ")
+		io.WriteString(buf, ns.String())
+		io.WriteString(buf, "\n")
+	}
+	if len(c.SearchDomains) > 0 {
+		io.WriteString(buf, "search")
+		for _, domain := range c.SearchDomains {
+			io.WriteString(buf, " ")
+			io.WriteString(buf, domain.WithoutTrailingDot())
+		}
+		io.WriteString(buf, "\n")
+	}
+	_, err := w.Write(buf.Bytes())
+	return err
+}
+
+// Parse parses a resolv.conf file from r.
+func Parse(r io.Reader) (*Config, error) {
+	config := new(Config)
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		line := scanner.Text()
+		line, _, _ = strings.Cut(line, "#") // remove any comments
+		line = strings.TrimSpace(line)
+
+		if s, ok := strs.CutPrefix(line, "nameserver"); ok {
+			nameserver := strings.TrimSpace(s)
+			if len(nameserver) == len(s) {
+				return nil, fmt.Errorf("missing space after \"nameserver\" in %q", line)
+			}
+			ip, err := netip.ParseAddr(nameserver)
+			if err != nil {
+				return nil, err
+			}
+			config.Nameservers = append(config.Nameservers, ip)
+			continue
+		}
+
+		if s, ok := strs.CutPrefix(line, "search"); ok {
+			domains := strings.TrimSpace(s)
+			if len(domains) == len(s) {
+				// No leading space?!
+				return nil, fmt.Errorf("missing space after \"search\" in %q", line)
+			}
+			for len(domains) > 0 {
+				domain := domains
+				i := strings.IndexAny(domain, " \t")
+				if i != -1 {
+					domain = domain[:i]
+					domains = strings.TrimSpace(domains[i+1:])
+				} else {
+					domains = ""
+				}
+				fqdn, err := dnsname.ToFQDN(domain)
+				if err != nil {
+					return nil, fmt.Errorf("parsing search domain %q in %q: %w", domain, line, err)
+				}
+				config.SearchDomains = append(config.SearchDomains, fqdn)
+			}
+		}
+	}
+	return config, nil
+}
+
+// ParseFile parses the named resolv.conf file.
+func ParseFile(name string) (*Config, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return nil, err
+	}
+	if n := fi.Size(); n > 10<<10 {
+		return nil, fmt.Errorf("unexpectedly large %q file: %d bytes", name, n)
+	}
+	all, err := os.ReadFile(name)
+	if err != nil {
+		return nil, err
+	}
+	return Parse(bytes.NewReader(all))
+}

internal/dns/resolvconffile/resolvconffile_test.go

@@ -0,0 +1,104 @@
+// Copyright (c) 2022 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package resolvconffile
+
+import (
+	"net/netip"
+	"reflect"
+	"strings"
+	"testing"
+
+	"tailscale.com/util/dnsname"
+)
+
+func TestParse(t *testing.T) {
+	tests := []struct {
+		in      string
+		want    *Config
+		wantErr bool
+	}{
+		{in: `nameserver 192.168.0.100`,
+			want: &Config{
+				Nameservers: []netip.Addr{
+					netip.MustParseAddr("192.168.0.100"),
+				},
+			},
+		},
+		{in: `nameserver 192.168.0.100 # comment`,
+			want: &Config{
+				Nameservers: []netip.Addr{
+					netip.MustParseAddr("192.168.0.100"),
+				},
+			},
+		},
+		{in: `nameserver 192.168.0.100#`,
+			want: &Config{
+				Nameservers: []netip.Addr{
+					netip.MustParseAddr("192.168.0.100"),
+				},
+			},
+		},
+		{in: `nameserver #192.168.0.100`, wantErr: true},
+		{in: `nameserver`, wantErr: true},
+		{in: `# nameserver 192.168.0.100`, want: &Config{}},
+		{in: `nameserver192.168.0.100`, wantErr: true},
+
+		{in: `search tailsacle.com`,
+			want: &Config{
+				SearchDomains: []dnsname.FQDN{"tailsacle.com."},
+			},
+		},
+		{in: `search tailsacle.com # typo`,
+			want: &Config{
+				SearchDomains: []dnsname.FQDN{"tailsacle.com."},
+			},
+		},
+		{in: `searchtailsacle.com`, wantErr: true},
+		{in: `search`, wantErr: true},
+
+		// Issue 6875: there can be multiple search domains, and even if they're
+		// over 253 bytes long total.
+		{
+			in: "search search-01.example search-02.example search-03.example search-04.example search-05.example search-06.example search-07.example search-08.example search-09.example search-10.example search-11.example search-12.example search-13.example search-14.example search-15.example\n",
+			want: &Config{
+				SearchDomains: []dnsname.FQDN{
+					"search-01.example.",
+					"search-02.example.",
+					"search-03.example.",
+					"search-04.example.",
+					"search-05.example.",
+					"search-06.example.",
+					"search-07.example.",
+					"search-08.example.",
+					"search-09.example.",
+					"search-10.example.",
+					"search-11.example.",
+					"search-12.example.",
+					"search-13.example.",
+					"search-14.example.",
+					"search-15.example.",
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		cfg, err := Parse(strings.NewReader(tt.in))
+		if tt.wantErr {
+			if err != nil {
+				continue
+			}
+			t.Errorf("missing error for %q", tt.in)
+			continue
+		}
+		if err != nil {
+			t.Errorf("unexpected error for %q: %v", tt.in, err)
+			continue
+		}
+		if !reflect.DeepEqual(cfg, tt.want) {
+			t.Errorf("got: %v\nwant: %v\n", cfg, tt.want)
+		}
+	}
+}

internal/dns/resolved.go

@@ -0,0 +1,459 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux
+
+package dns
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/godbus/dbus/v5"
+	"golang.org/x/sys/unix"
+	"tailscale.com/health"
+	"tailscale.com/logtail/backoff"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/dnsname"
+)
+
+const reconfigTimeout = time.Second
+
+// DBus entities we talk to.
+//
+// DBus is an RPC bus. In particular, the bus we're talking to is the
+// system-wide bus (there is also a per-user session bus for
+// user-specific applications).
+//
+// Daemons connect to the bus, and advertise themselves under a
+// well-known object name. That object exposes paths, and each path
+// implements one or more interfaces that contain methods, properties,
+// and signals.
+//
+// Clients connect to the bus and walk that same hierarchy to invoke
+// RPCs, get/set properties, or listen for signals.
+const (
+	dbusResolvedObject                        = "org.freedesktop.resolve1"
+	dbusNetworkdObject                        = "org.freedesktop.network1"
+	dbusResolvedPath          dbus.ObjectPath = "/org/freedesktop/resolve1"
+	dbusNetworkdPath          dbus.ObjectPath = "/org/freedesktop/network1"
+	dbusResolvedInterface                     = "org.freedesktop.resolve1.Manager"
+	dbusNetworkdInterface                     = "org.freedesktop.network1.Manager"
+	dbusPath                  dbus.ObjectPath = "/org/freedesktop/DBus"
+	dbusInterface                             = "org.freedesktop.DBus"
+	dbusOwnerSignal                           = "NameOwnerChanged" // broadcast when a well-known name's owning process changes.
+	dbusResolvedErrorLinkBusy                 = "org.freedesktop.resolve1.LinkBusy"
+)
+
+var (
+	dbusSetLinkDNS          string
+	dbusSetLinkDomains      string
+	dbusSetLinkDefaultRoute string
+	dbusSetLinkLLMNR        string
+	dbusSetLinkMulticastDNS string
+	dbusSetLinkDNSSEC       string
+	dbusSetLinkDNSOverTLS   string
+	dbusFlushCaches         string
+	dbusRevertLink          string
+)
+
+func setDbusMethods(dbusInterface string) {
+	dbusSetLinkDNS = dbusInterface + ".SetLinkDNS"
+	dbusSetLinkDomains = dbusInterface + ".SetLinkDomains"
+	dbusSetLinkDefaultRoute = dbusInterface + ".SetLinkDefaultRoute"
+	dbusSetLinkLLMNR = dbusInterface + ".SetLinkLLMNR"
+	dbusSetLinkMulticastDNS = dbusInterface + ".SetLinkMulticastDNS"
+	dbusSetLinkDNSSEC = dbusInterface + ".SetLinkDNSSEC"
+	dbusSetLinkDNSOverTLS = dbusInterface + ".SetLinkDNSOverTLS"
+	dbusFlushCaches = dbusInterface + ".FlushCaches"
+	dbusRevertLink = dbusInterface + ".RevertLink"
+	if dbusInterface == dbusNetworkdInterface {
+		dbusRevertLink = dbusInterface + ".RevertLinkDNS"
+	}
+}
+
+type resolvedLinkNameserver struct {
+	Family  int32
+	Address []byte
+}
+
+type resolvedLinkDomain struct {
+	Domain      string
+	RoutingOnly bool
+}
+
+// changeRequest tracks latest OSConfig and related error responses to update.
+type changeRequest struct {
+	config OSConfig     // configs OSConfigs, one per each SetDNS call
+	res    chan<- error // response channel
+}
+
+// resolvedManager is an OSConfigurator which uses the systemd-resolved DBus API.
+type resolvedManager struct {
+	ctx    context.Context
+	cancel func() // terminate the context, for close
+
+	logf  logger.Logf
+	ifidx int
+
+	configCR chan changeRequest // tracks OSConfigs changes and error responses
+	revertCh chan struct{}
+}
+
+var _ OSConfigurator = (*resolvedManager)(nil)
+
+func newResolvedManager(logf logger.Logf, interfaceName string) (*resolvedManager, error) {
+	iface, err := net.InterfaceByName(interfaceName)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	logf = logger.WithPrefix(logf, "dns: ")
+
+	mgr := &resolvedManager{
+		ctx:    ctx,
+		cancel: cancel,
+
+		logf:  logf,
+		ifidx: iface.Index,
+
+		configCR: make(chan changeRequest),
+		revertCh: make(chan struct{}),
+	}
+
+	go mgr.run(ctx)
+
+	return mgr, nil
+}
+
+func (m *resolvedManager) SetDNS(config OSConfig) error {
+	errc := make(chan error, 1)
+	defer close(errc)
+
+	select {
+	case <-m.ctx.Done():
+		return m.ctx.Err()
+	case m.configCR <- changeRequest{config, errc}:
+	}
+
+	select {
+	case <-m.ctx.Done():
+		return m.ctx.Err()
+	case err := <-errc:
+		if err != nil {
+			m.logf("failed to configure resolved: %v", err)
+		}
+		return err
+	}
+}
+
+func newResolvedObject(conn *dbus.Conn) dbus.BusObject {
+	setDbusMethods(dbusResolvedInterface)
+	return conn.Object(dbusResolvedObject, dbusResolvedPath)
+}
+
+func newNetworkdObject(conn *dbus.Conn) dbus.BusObject {
+	setDbusMethods(dbusNetworkdInterface)
+	return conn.Object(dbusNetworkdObject, dbusNetworkdPath)
+}
+
+func (m *resolvedManager) run(ctx context.Context) {
+	var (
+		conn     *dbus.Conn
+		signals  chan *dbus.Signal
+		rManager dbus.BusObject // rManager is the Resolved DBus connection
+	)
+	bo := backoff.NewBackoff("resolved-dbus", m.logf, 30*time.Second)
+	needsReconnect := make(chan bool, 1)
+	defer func() {
+		if conn != nil {
+			_ = conn.Close()
+		}
+	}()
+
+	newManager := newResolvedObject
+	func() {
+		conn, err := dbus.SystemBus()
+		if err != nil {
+			m.logf("dbus connection error: %v", err)
+			return
+		}
+		rManager = newManager(conn)
+		if call := rManager.CallWithContext(ctx, dbusRevertLink, 0, m.ifidx); call.Err != nil {
+			if dbusErr, ok := call.Err.(dbus.Error); ok && dbusErr.Name == dbusResolvedErrorLinkBusy {
+				m.logf("[v1] Using %s as manager", dbusNetworkdObject)
+				newManager = newNetworkdObject
+			}
+		}
+	}()
+
+	// Reconnect the systemBus if disconnected.
+	reconnect := func() error {
+		var err error
+		signals = make(chan *dbus.Signal, 16)
+		conn, err = dbus.SystemBus()
+		if err != nil {
+			m.logf("dbus connection error: %v", err)
+		} else {
+			m.logf("[v1] dbus connected")
+		}
+
+		if err != nil {
+			// Backoff increases time between reconnect attempts.
+			go func() {
+				bo.BackOff(ctx, err)
+				needsReconnect <- true
+			}()
+			return err
+		}
+
+		rManager = newManager(conn)
+
+		// Only receive the DBus signals we need to resync our config on
+		// resolved restart. Failure to set filters isn't a fatal error,
+		// we'll just receive all broadcast signals and have to ignore
+		// them on our end.
+		if err = conn.AddMatchSignal(dbus.WithMatchObjectPath(dbusPath), dbus.WithMatchInterface(dbusInterface), dbus.WithMatchMember(dbusOwnerSignal), dbus.WithMatchArg(0, dbusResolvedObject)); err != nil {
+			m.logf("[v1] Setting DBus signal filter failed: %v", err)
+		}
+		if err = conn.AddMatchSignal(dbus.WithMatchObjectPath(dbusPath), dbus.WithMatchInterface(dbusInterface), dbus.WithMatchMember(dbusOwnerSignal), dbus.WithMatchArg(0, dbusNetworkdObject)); err != nil {
+			m.logf("[v1] Setting DBus signal filter failed: %v", err)
+		}
+		conn.Signal(signals)
+
+		// Reset backoff and SetNSOSHealth after successful on reconnect.
+		bo.BackOff(ctx, nil)
+		health.SetDNSOSHealth(nil)
+		return nil
+	}
+
+	// Create initial systemBus connection.
+	_ = reconnect()
+
+	lastConfig := OSConfig{}
+
+	for {
+		select {
+		case <-ctx.Done():
+			if rManager == nil {
+				return
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			// RevertLink resets all per-interface settings on systemd-resolved to defaults.
+			// When ctx goes away systemd-resolved auto reverts.
+			// Keeping for potential use in future refactor.
+			if call := rManager.CallWithContext(ctx, dbusRevertLink, 0, m.ifidx); call.Err != nil {
+				m.logf("[v1] RevertLink: %v", call.Err)
+			}
+			cancel()
+			close(m.revertCh)
+			return
+		case configCR := <-m.configCR:
+			// Track and update sync with latest config change.
+			lastConfig = configCR.config
+
+			if rManager == nil {
+				configCR.res <- fmt.Errorf("resolved DBus does not have a connection")
+				continue
+			}
+			err := m.setConfigOverDBus(ctx, rManager, configCR.config)
+			configCR.res <- err
+		case <-needsReconnect:
+			if err := reconnect(); err != nil {
+				m.logf("[v1] SystemBus reconnect error %T", err)
+			}
+			continue
+		case signal, ok := <-signals:
+			// If signal ends and is nil then program tries to reconnect.
+			if !ok {
+				if err := reconnect(); err != nil {
+					m.logf("[v1] SystemBus reconnect error %T", err)
+				}
+				continue
+			}
+			// In theory the signal was filtered by DBus, but if
+			// AddMatchSignal in the constructor failed, we may be
+			// getting other spam.
+			if signal.Path != dbusPath || signal.Name != dbusInterface+"."+dbusOwnerSignal {
+				continue
+			}
+			if lastConfig.IsZero() {
+				continue
+			}
+			// signal.Body is a []any of 3 strings: bus name, previous owner, new owner.
+			if len(signal.Body) != 3 {
+				m.logf("[unexpected] DBus NameOwnerChanged len(Body) = %d, want 3")
+			}
+			if name, ok := signal.Body[0].(string); !ok || (name != dbusResolvedObject && name != dbusNetworkdObject) {
+				continue
+			}
+			newOwner, ok := signal.Body[2].(string)
+			if !ok {
+				m.logf("[unexpected] DBus NameOwnerChanged.new_owner is a %T, not a string", signal.Body[2])
+			}
+			if newOwner == "" {
+				// systemd-resolved left the bus, no current owner,
+				// nothing to do.
+				continue
+			}
+			// The resolved bus name has a new owner, meaning resolved
+			// restarted. Reprogram current config.
+			m.logf("systemd-resolved restarted, syncing DNS config")
+			err := m.setConfigOverDBus(ctx, rManager, lastConfig)
+			// Set health while holding the lock, because this will
+			// graciously serialize the resync's health outcome with a
+			// concurrent SetDNS call.
+			health.SetDNSOSHealth(err)
+			if err != nil {
+				m.logf("failed to configure systemd-resolved: %v", err)
+			}
+		}
+	}
+}
+
+// setConfigOverDBus updates resolved DBus config and is only called from the run goroutine.
+func (m *resolvedManager) setConfigOverDBus(ctx context.Context, rManager dbus.BusObject, config OSConfig) error {
+	ctx, cancel := context.WithTimeout(ctx, reconfigTimeout)
+	defer cancel()
+
+	var linkNameservers = make([]resolvedLinkNameserver, len(config.Nameservers))
+	for i, server := range config.Nameservers {
+		ip := server.As16()
+		if server.Is4() {
+			linkNameservers[i] = resolvedLinkNameserver{
+				Family:  unix.AF_INET,
+				Address: ip[12:],
+			}
+		} else {
+			linkNameservers[i] = resolvedLinkNameserver{
+				Family:  unix.AF_INET6,
+				Address: ip[:],
+			}
+		}
+	}
+	err := rManager.CallWithContext(
+		ctx, dbusSetLinkDNS, 0,
+		m.ifidx, linkNameservers,
+	).Store()
+	if err != nil {
+		return fmt.Errorf("setLinkDNS: %w", err)
+	}
+	linkDomains := make([]resolvedLinkDomain, 0, len(config.SearchDomains)+len(config.MatchDomains))
+	seenDomains := map[dnsname.FQDN]bool{}
+	for _, domain := range config.SearchDomains {
+		if seenDomains[domain] {
+			continue
+		}
+		seenDomains[domain] = true
+		linkDomains = append(linkDomains, resolvedLinkDomain{
+			Domain:      domain.WithTrailingDot(),
+			RoutingOnly: false,
+		})
+	}
+	for _, domain := range config.MatchDomains {
+		if seenDomains[domain] {
+			// Search domains act as both search and match in
+			// resolved, so it's correct to skip.
+			continue
+		}
+		seenDomains[domain] = true
+		linkDomains = append(linkDomains, resolvedLinkDomain{
+			Domain:      domain.WithTrailingDot(),
+			RoutingOnly: true,
+		})
+	}
+	if len(config.MatchDomains) == 0 && len(config.Nameservers) > 0 {
+		// Caller requested full DNS interception, install a
+		// routing-only root domain.
+		linkDomains = append(linkDomains, resolvedLinkDomain{
+			Domain:      ".",
+			RoutingOnly: true,
+		})
+	}
+
+	err = rManager.CallWithContext(
+		ctx, dbusSetLinkDomains, 0,
+		m.ifidx, linkDomains,
+	).Store()
+	if err != nil && err.Error() == "Argument list too long" { // TODO: better error match
+		// Issue 3188: older systemd-resolved had argument length limits.
+		// Trim out the *.arpa. entries and try again.
+		err = rManager.CallWithContext(
+			ctx, dbusSetLinkDomains, 0,
+			m.ifidx, linkDomainsWithoutReverseDNS(linkDomains),
+		).Store()
+	}
+	if err != nil {
+		return fmt.Errorf("setLinkDomains: %w", err)
+	}
+
+	if call := rManager.CallWithContext(ctx, dbusSetLinkDefaultRoute, 0, m.ifidx, len(config.MatchDomains) == 0); call.Err != nil {
+		if dbusErr, ok := call.Err.(dbus.Error); ok && dbusErr.Name == dbus.ErrMsgUnknownMethod.Name {
+			// on some older systems like Kubuntu 18.04.6 with systemd 237 method SetLinkDefaultRoute is absent,
+			// but otherwise it's working good
+			m.logf("[v1] failed to set SetLinkDefaultRoute: %v", call.Err)
+		} else {
+			return fmt.Errorf("setLinkDefaultRoute: %w", call.Err)
+		}
+	}
+
+	// Some best-effort setting of things, but resolved should do the
+	// right thing if these fail (e.g. a really old resolved version
+	// or something).
+
+	// Disable LLMNR, we don't do multicast.
+	if call := rManager.CallWithContext(ctx, dbusSetLinkLLMNR, 0, m.ifidx, "no"); call.Err != nil {
+		m.logf("[v1] failed to disable LLMNR: %v", call.Err)
+	}
+
+	// Disable mdns.
+	if call := rManager.CallWithContext(ctx, dbusSetLinkMulticastDNS, 0, m.ifidx, "no"); call.Err != nil {
+		m.logf("[v1] failed to disable mdns: %v", call.Err)
+	}
+
+	// We don't support dnssec consistently right now, force it off to
+	// avoid partial failures when we split DNS internally.
+	if call := rManager.CallWithContext(ctx, dbusSetLinkDNSSEC, 0, m.ifidx, "no"); call.Err != nil {
+		m.logf("[v1] failed to disable DNSSEC: %v", call.Err)
+	}
+
+	if call := rManager.CallWithContext(ctx, dbusSetLinkDNSOverTLS, 0, m.ifidx, "no"); call.Err != nil {
+		m.logf("[v1] failed to disable DoT: %v", call.Err)
+	}
+
+	if rManager.Path() == dbusResolvedPath {
+		if call := rManager.CallWithContext(ctx, dbusFlushCaches, 0); call.Err != nil {
+			m.logf("failed to flush resolved DNS cache: %v", call.Err)
+		}
+	}
+
+	return nil
+}
+
+func (m *resolvedManager) Close() error {
+	m.cancel() // stops the 'run' method goroutine
+	<-m.revertCh
+	return nil
+}
+
+func (m *resolvedManager) Mode() string {
+	return "systemd-resolved"
+}
+
+// linkDomainsWithoutReverseDNS returns a copy of v without
+// *.arpa. entries.
+func linkDomainsWithoutReverseDNS(v []resolvedLinkDomain) (ret []resolvedLinkDomain) {
+	for _, d := range v {
+		if strings.HasSuffix(d.Domain, ".arpa.") {
+			// Oh well. At least the rest will work.
+			continue
+		}
+		ret = append(ret, d)
+	}
+	return ret
+}

internal/dnscache/cache.go

@@ -0,0 +1,77 @@
+package dnscache
+
+import (
+	"strings"
+	"time"
+
+	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/miekg/dns"
+)
+
+// Cacher is the interface for caching DNS response.
+type Cacher interface {
+	Get(Key) *Value
+	Add(Key, *Value)
+}
+
+// Key is the caching key for DNS message.
+type Key struct {
+	Qtype    uint16
+	Qclass   uint16
+	Name     string
+	Upstream string
+}
+
+type Value struct {
+	Expire time.Time
+	Msg    *dns.Msg
+}
+
+var _ Cacher = (*LRUCache)(nil)
+
+// LRUCache implements Cacher interface.
+type LRUCache struct {
+	cacher *lru.ARCCache[Key, *Value]
+}
+
+func (l *LRUCache) Get(key Key) *Value {
+	v, _ := l.cacher.Get(key)
+	return v
+}
+
+func (l *LRUCache) Add(key Key, value *Value) {
+	l.cacher.Add(key, value)
+}
+
+// NewLRUCache creates a new LRUCache instance with given size.
+func NewLRUCache(size int) (*LRUCache, error) {
+	cacher, err := lru.NewARC[Key, *Value](size)
+	return &LRUCache{cacher: cacher}, err
+}
+
+// NewKey creates a new cache key for given DNS message.
+func NewKey(msg *dns.Msg, upstream string) Key {
+	q := msg.Question[0]
+	return Key{Qtype: q.Qtype, Qclass: q.Qclass, Name: normalizeQname(q.Name), Upstream: upstream}
+}
+
+// NewValue creates a new cache value for given DNS message.
+func NewValue(msg *dns.Msg, expire time.Time) *Value {
+	return &Value{
+		Expire: expire,
+		Msg:    msg,
+	}
+}
+
+func normalizeQname(name string) string {
+	var b strings.Builder
+	b.Grow(len(name))
+	for i := 0; i < len(name); i++ {
+		c := name[i]
+		if 'A' <= c && c <= 'Z' {
+			c += 'a' - 'A'
+		}
+		b.WriteByte(c)
+	}
+	return b.String()
+}

internal/dnsrcode/rcode.go

@@ -0,0 +1,39 @@
+package dnsrcode
+
+import "strings"
+
+// https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-6
+var dnsRcode = map[string]int{
+	"NOERROR":   0,  // NoError   - No Error
+	"FORMERR":   1,  // FormErr   - Format Error
+	"SERVFAIL":  2,  // ServFail  - Server Failure
+	"NXDOMAIN":  3,  // NXDomain  - Non-Existent Domain
+	"NOTIMP":    4,  // NotImp    - Not Implemented
+	"REFUSED":   5,  // Refused   - Query Refused
+	"YXDOMAIN":  6,  // YXDomain  - Name Exists when it should not
+	"YXRRSET":   7,  // YXRRSet   - RR Set Exists when it should not
+	"NXRRSET":   8,  // NXRRSet   - RR Set that should exist does not
+	"NOTAUTH":   9,  // NotAuth   - Server Not Authoritative for zone
+	"NOTZONE":   10, // NotZone   - Name not contained in zone
+	"BADSIG":    16, // BADSIG    - TSIG Signature Failure
+	"BADVERS":   16, // BADVERS   - Bad OPT Version
+	"BADKEY":    17, // BADKEY    - Key not recognized
+	"BADTIME":   18, // BADTIME   - Signature out of time window
+	"BADMODE":   19, // BADMODE   - Bad TKEY Mode
+	"BADNAME":   20, // BADNAME   - Duplicate key name
+	"BADALG":    21, // BADALG    - Algorithm not supported
+	"BADTRUNC":  22, // BADTRUNC  - Bad Truncation
+	"BADCOOKIE": 23, // BADCOOKIE - Bad/missing Server Cookie
+}
+
+// FromString returns the DNS Rcode number from given DNS Rcode string.
+// The string value is treated as case-insensitive. If the input string
+// is an invalid DNS Rcode, -1 is returned.
+func FromString(rcode string) int {
+	rcode = strings.ToUpper(rcode)
+	val, ok := dnsRcode[rcode]
+	if !ok {
+		return -1
+	}
+	return val
+}

internal/dnsrcode/rcode_test.go

@@ -0,0 +1,29 @@
+package dnsrcode
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFromString(t *testing.T) {
+	tests := []struct {
+		name          string
+		rcode         string
+		expectedRcode int
+	}{
+		{"valid", "NoError", 0},
+		{"upper", "NOERROR", 0},
+		{"lower", "noerror", 0},
+		{"mix", "nOeRrOr", 0},
+		{"invalid", "foo", -1},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			assert.Equal(t, tc.expectedRcode, FromString(tc.rcode))
+		})
+	}
+}

internal/net/net.go

@@ -0,0 +1,107 @@
+package net
+
+import (
+	"context"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"tailscale.com/logtail/backoff"
+)
+
+const (
+	controldIPv6Test = "ipv6.controld.io"
+	controldIPv4Test = "ipv4.controld.io"
+	bootstrapDNS     = "76.76.2.0:53"
+)
+
+var Dialer = &net.Dialer{
+	Resolver: &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+			d := net.Dialer{
+				Timeout: 10 * time.Second,
+			}
+			return d.DialContext(ctx, "udp", bootstrapDNS)
+		},
+	},
+}
+
+var (
+	stackOnce          atomic.Pointer[sync.Once]
+	ipv4Enabled        bool
+	ipv6Enabled        bool
+	canListenIPv6Local bool
+	hasNetworkUp       bool
+)
+
+func init() {
+	stackOnce.Store(new(sync.Once))
+}
+
+func supportIPv4() bool {
+	_, err := Dialer.Dial("tcp4", net.JoinHostPort(controldIPv4Test, "80"))
+	return err == nil
+}
+
+func supportIPv6(ctx context.Context) bool {
+	_, err := Dialer.DialContext(ctx, "tcp6", net.JoinHostPort(controldIPv6Test, "80"))
+	return err == nil
+}
+
+func supportListenIPv6Local() bool {
+	if ln, err := net.Listen("tcp6", "[::1]:0"); err == nil {
+		ln.Close()
+		return true
+	}
+	return false
+}
+
+func probeStack() {
+	b := backoff.NewBackoff("probeStack", func(format string, args ...any) {}, time.Minute)
+	for {
+		if _, err := Dialer.Dial("udp", bootstrapDNS); err == nil {
+			hasNetworkUp = true
+			break
+		} else {
+			b.BackOff(context.Background(), err)
+		}
+	}
+	ipv4Enabled = supportIPv4()
+	ipv6Enabled = supportIPv6(context.Background())
+	canListenIPv6Local = supportListenIPv6Local()
+}
+
+func Up() bool {
+	stackOnce.Load().Do(probeStack)
+	return hasNetworkUp
+}
+
+func SupportsIPv4() bool {
+	stackOnce.Load().Do(probeStack)
+	return ipv4Enabled
+}
+
+func SupportsIPv6() bool {
+	stackOnce.Load().Do(probeStack)
+	return ipv6Enabled
+}
+
+func SupportsIPv6ListenLocal() bool {
+	stackOnce.Load().Do(probeStack)
+	return canListenIPv6Local
+}
+
+// IPv6Available is like SupportsIPv6, but always do the check without caching.
+func IPv6Available(ctx context.Context) bool {
+	return supportIPv6(ctx)
+}
+
+// IsIPv6 checks if the provided IP is v6.
+//
+//lint:ignore U1000 use in os_windows.go
+func IsIPv6(ip string) bool {
+	parsedIP := net.ParseIP(ip)
+	return parsedIP != nil && parsedIP.To4() == nil && parsedIP.To16() != nil
+}

internal/resolvconffile/dns.go

@@ -0,0 +1,35 @@
+//go:build !js && !windows
+
+package resolvconffile
+
+import (
+	"net"
+
+	"tailscale.com/net/dns/resolvconffile"
+)
+
+const resolvconfPath = "/etc/resolv.conf"
+
+func NameServersWithPort() []string {
+	c, err := resolvconffile.ParseFile(resolvconfPath)
+	if err != nil {
+		return nil
+	}
+	ns := make([]string, 0, len(c.Nameservers))
+	for _, nameserver := range c.Nameservers {
+		ns = append(ns, net.JoinHostPort(nameserver.String(), "53"))
+	}
+	return ns
+}
+
+func NameServers(_ string) []string {
+	c, err := resolvconffile.ParseFile(resolvconfPath)
+	if err != nil {
+		return nil
+	}
+	ns := make([]string, 0, len(c.Nameservers))
+	for _, nameserver := range c.Nameservers {
+		ns = append(ns, nameserver.String())
+	}
+	return ns
+}

internal/resolvconffile/dns_test.go

@@ -0,0 +1,15 @@
+//go:build !js && !windows
+
+package resolvconffile
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNameServers(t *testing.T) {
+	ns := NameServers("")
+	require.NotNil(t, ns)
+	t.Log(ns)
+}

nameservers_unix.go

@@ -0,0 +1,11 @@
+//go:build !js && !windows
+
+package ctrld
+
+import (
+	"github.com/Control-D-Inc/ctrld/internal/resolvconffile"
+)
+
+func nameservers() []string {
+	return resolvconffile.NameServersWithPort()
+}

nameservers_windows.go

@@ -0,0 +1,71 @@
+package ctrld
+
+import (
+	"net"
+	"os"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+func nameservers() []string {
+	aas, err := adapterAddresses()
+	if err != nil {
+		return nil
+	}
+	ns := make([]string, 0, len(aas))
+	for _, aa := range aas {
+		for dns := aa.FirstDnsServerAddress; dns != nil; dns = dns.Next {
+			sa, err := dns.Address.Sockaddr.Sockaddr()
+			if err != nil {
+				continue
+			}
+			var ip net.IP
+			switch sa := sa.(type) {
+			case *syscall.SockaddrInet4:
+				ip = net.IPv4(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3])
+			case *syscall.SockaddrInet6:
+				ip = make(net.IP, net.IPv6len)
+				copy(ip, sa.Addr[:])
+				if ip[0] == 0xfe && ip[1] == 0xc0 {
+					// Ignore these fec0/10 ones. Windows seems to
+					// populate them as defaults on its misc rando
+					// interfaces.
+					continue
+				}
+			default:
+				// Unexpected type.
+				continue
+			}
+			ns = append(ns, net.JoinHostPort(ip.String(), "53"))
+		}
+	}
+	return ns
+}
+
+func adapterAddresses() ([]*windows.IpAdapterAddresses, error) {
+	var b []byte
+	l := uint32(15000) // recommended initial size
+	for {
+		b = make([]byte, l)
+		err := windows.GetAdaptersAddresses(syscall.AF_UNSPEC, windows.GAA_FLAG_INCLUDE_PREFIX, 0, (*windows.IpAdapterAddresses)(unsafe.Pointer(&b[0])), &l)
+		if err == nil {
+			if l == 0 {
+				return nil, nil
+			}
+			break
+		}
+		if err.(syscall.Errno) != syscall.ERROR_BUFFER_OVERFLOW {
+			return nil, os.NewSyscallError("getadaptersaddresses", err)
+		}
+		if l <= uint32(len(b)) {
+			return nil, os.NewSyscallError("getadaptersaddresses", err)
+		}
+	}
+	var aas []*windows.IpAdapterAddresses
+	for aa := (*windows.IpAdapterAddresses)(unsafe.Pointer(&b[0])); aa != nil; aa = aa.Next {
+		aas = append(aas, aa)
+	}
+	return aas, nil
+}

resolver.go

@@ -5,21 +5,21 @@ import (
 	"errors"
 	"fmt"
 	"net"
-	"strings"
 
 	"github.com/miekg/dns"
 )
 
 const (
-	resolverTypeDOH    = "doh"
-	resolverTypeDOH3   = "doh3"
-	resolverTypeDOT    = "dot"
-	resolverTypeDOQ    = "doq"
-	resolverTypeOS     = "os"
-	resolverTypeLegacy = "legacy"
+	ResolverTypeDOH    = "doh"
+	ResolverTypeDOH3   = "doh3"
+	ResolverTypeDOT    = "dot"
+	ResolverTypeDOQ    = "doq"
+	ResolverTypeOS     = "os"
+	ResolverTypeLegacy = "legacy"
 )
 
 var bootstrapDNS = "76.76.2.0"
+var or = &osResolver{nameservers: nameservers()}
 
 // Resolver is the interface that wraps the basic DNS operations.
 //
@@ -34,44 +34,58 @@ var errUnknownResolver = errors.New("unknown resolver")
 func NewResolver(uc *UpstreamConfig) (Resolver, error) {
 	typ, endpoint := uc.Type, uc.Endpoint
 	switch typ {
-	case resolverTypeDOH, resolverTypeDOH3:
+	case ResolverTypeDOH, ResolverTypeDOH3:
 		return newDohResolver(uc), nil
-	case resolverTypeDOT:
+	case ResolverTypeDOT:
 		return &dotResolver{uc: uc}, nil
-	case resolverTypeDOQ:
+	case ResolverTypeDOQ:
 		return &doqResolver{uc: uc}, nil
-	case resolverTypeOS:
-		return &osResolver{}, nil
-	case resolverTypeLegacy:
+	case ResolverTypeOS:
+		return or, nil
+	case ResolverTypeLegacy:
 		return &legacyResolver{endpoint: endpoint}, nil
 	}
 	return nil, fmt.Errorf("%w: %s", errUnknownResolver, typ)
 }
 
-type osResolver struct{}
+type osResolver struct {
+	nameservers []string
+}
 
+type osResolverResult struct {
+	answer *dns.Msg
+	err    error
+}
+
+// Resolve performs DNS resolvers using OS default nameservers. Nameserver is chosen from
+// available nameservers with a roundrobin algorithm.
 func (o *osResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) {
-	domain := canonicalName(msg.Question[0].Name)
-	addrs, err := net.DefaultResolver.LookupHost(ctx, domain)
-	if err != nil {
-		return nil, err
+	numServers := len(o.nameservers)
+	if numServers == 0 {
+		return nil, errors.New("no nameservers available")
 	}
-	if len(addrs) == 0 {
-		return nil, errors.New("no answer")
-	}
-	answer := new(dns.Msg)
-	answer.SetReply(msg)
-	ip := net.ParseIP(addrs[0])
-	a := &dns.A{
-		A:   ip,
-		Hdr: dns.RR_Header{Name: msg.Question[0].Name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 10},
-	}
-	if ip.To4() != nil {
-		a.Hdr.Rrtype = dns.TypeA
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	dnsClient := &dns.Client{Net: "udp"}
+	ch := make(chan *osResolverResult, numServers)
+	for _, server := range o.nameservers {
+		go func(server string) {
+			answer, _, err := dnsClient.ExchangeContext(ctx, msg, server)
+			ch <- &osResolverResult{answer: answer, err: err}
+		}(server)
 	}
 
-	msg.Answer = append(msg.Answer, a)
-	return msg, nil
+	errs := make([]error, 0, numServers)
+	for res := range ch {
+		if res.err == nil {
+			cancel()
+			return res.answer, res.err
+		}
+		errs = append(errs, res.err)
+	}
+
+	return nil, joinErrors(errs...)
 }
 
 func newDialer(dnsAddress string) *net.Dialer {
@@ -100,13 +114,3 @@ func (r *legacyResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, e
 	answer, _, err := dnsClient.ExchangeContext(ctx, msg, r.endpoint)
 	return answer, err
 }
-
-// canonicalName returns canonical name from FQDN with "." trimmed.
-func canonicalName(fqdn string) string {
-	q := strings.TrimSpace(fqdn)
-	q = strings.TrimSuffix(q, ".")
-	// https://datatracker.ietf.org/doc/html/rfc4343
-	q = strings.ToLower(q)
-
-	return q
-}

scripts/upx.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -ex
+
+binary=$1
+
+if [ -z "$binary" ]; then
+  echo >&2 "Usage: $0 <binary>"
+  exit 1
+fi
+
+case "$binary" in
+  *_freebsd_*)
+    echo >&2 "upx does not work with freebsd binary yet"
+    exit 0
+    ;;
+  *_windows_arm*)
+    echo >&2 "upx does not work with windows arm/arm64 binary yet"
+    exit 0
+    ;;
+esac
+
+upx -- "$binary"

testhelper/config.go

@@ -12,6 +12,7 @@ import (
 func SampleConfig(t *testing.T) *ctrld.Config {
 	v := viper.NewWithOptions(viper.KeyDelimiter("::"))
 	ctrld.InitConfig(v, "test_load_config")
+	v.SetConfigType("toml")
 	require.NoError(t, v.ReadConfig(strings.NewReader(sampleConfigContent)))
 	var cfg ctrld.Config
 	require.NoError(t, v.Unmarshal(&cfg))
@@ -59,6 +60,7 @@ port = 1337
 
 [listener.0.policy]
 name = "My Policy"
+failover_rcodes = ["NXDOMAIN", "SERVFAIL"]
 networks = [
     {"network.0" = ["upstream.1", "upstream.0"]},
     {"network.1" = ["upstream.0"]},