all: rework fetching/generating config in cd mode

Config fetching/generating in cd mode is currently weird, error prone,
and easy for user to break ctrld when using custom config.

This commit reworks the flow:

 - Fetching config from Control D API.
 - No custom config, use the current default config.
 - If custom config presents, but there's no listener, use 0.0.0.0:53.
 - Try listening on current ip+port config, if ok, ctrld could be a
   direct listener with current setup, moving on.
 - If failed, trying 127.0.0.1:53.
 - If failed, trying current ip + port 5354
 - If still failed, pick a random ip:port pair, retry until listening ok.

With this flow, thing is more predictable/stable, and help removing the
Config interface for router.

cmd/ctrld/cli.go

@@ -15,17 +15,21 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/cuonglm/osinfo"
+	"github.com/fsnotify/fsnotify"
 	"github.com/go-playground/validator/v10"
 	"github.com/kardianos/service"
 	"github.com/miekg/dns"
 	"github.com/pelletier/go-toml/v2"
 	"github.com/rs/zerolog"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"tailscale.com/logtail/backoff"
 	"tailscale.com/net/interfaces"
@@ -36,6 +40,7 @@ import (
 	ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
 	"github.com/Control-D-Inc/ctrld/internal/router"
 	"github.com/Control-D-Inc/ctrld/internal/router/ddwrt"
+	"github.com/Control-D-Inc/ctrld/internal/router/firewalla"
 	"github.com/Control-D-Inc/ctrld/internal/router/merlin"
 	"github.com/Control-D-Inc/ctrld/internal/router/tomato"
 )
@@ -126,12 +131,14 @@ func initCLI() {
 			p := &prog{
 				waitCh: waitCh,
 				stopCh: stopCh,
+				cfg:    &cfg,
 			}
 			sockPath := filepath.Join(homedir, ctrldLogUnixSock)
 			if addr, err := net.ResolveUnixAddr("unix", sockPath); err == nil {
 				if conn, err := net.Dial(addr.Network(), addr.String()); err == nil {
-					consoleWriter.Out = io.MultiWriter(os.Stdout, conn)
+					lc := &logConn{conn: conn}
-					p.logConn = conn
+					consoleWriter.Out = io.MultiWriter(os.Stdout, lc)
+					p.logConn = lc
 				}
 			}
 
@@ -177,10 +184,7 @@ func initCLI() {
 				mainLog.Fatal().Msg("network is not up yet")
 			}
 
-			p.router = router.NewDummyRouter()
+			p.router = router.New(&cfg)
-			if setupRouter {
-				p.router = router.New(&cfg)
-			}
 
 			// Processing --cd flag require connecting to ControlD API, which needs valid
 			// time for validating server certificate. Some routers need NTP synchronization
@@ -190,7 +194,22 @@ func initCLI() {
 			}
 
 			oldLogPath := cfg.Service.LogPath
-			processCDFlags(p)
+			if cdUID != "" {
+				processCDFlags()
+			}
+
+			updateListenerConfig()
+
+			if cdUID != "" {
+				processLogAndCacheFlags()
+			}
+
+			if err := writeConfigFile(); err != nil {
+				mainLog.Fatal().Err(err).Msg("failed to write config file")
+			} else {
+				mainLog.Info().Msg("writing config file to: " + defaultConfigFile)
+			}
+
 			if newLogPath := cfg.Service.LogPath; newLogPath != "" && oldLogPath != newLogPath {
 				// After processCDFlags, log config may change, so reset mainLog and re-init logging.
 				mainLog = zerolog.New(io.Discard)
@@ -229,24 +248,38 @@ func initCLI() {
 				os.Exit(0)
 			}
 
-			if setupRouter {
+			p.onStarted = append(p.onStarted, func() {
-				switch platform := router.Name(); {
+				for _, lc := range p.cfg.Listener {
-				case platform == ddwrt.Name:
+					if shouldAllocateLoopbackIP(lc.IP) {
-					rootCertPool = certs.CACertPool()
+						if err := allocateIP(lc.IP); err != nil {
-					fallthrough
+							mainLog.Error().Err(err).Msgf("could not allocate IP: %s", lc.IP)
-				case platform != "":
+						}
-					if !router.IsSupported(platform) {
-						unsupportedPlatformHelp(cmd)
-						os.Exit(1)
 					}
+				}
+			})
+			p.onStopped = append(p.onStopped, func() {
+				for _, lc := range p.cfg.Listener {
+					if shouldAllocateLoopbackIP(lc.IP) {
+						if err := deAllocateIP(lc.IP); err != nil {
+							mainLog.Error().Err(err).Msgf("could not de-allocate IP: %s", lc.IP)
+						}
+					}
+				}
+			})
+			if platform := router.Name(); platform != "" {
+				if platform == ddwrt.Name {
+					rootCertPool = certs.CACertPool()
+				}
+				// Perform router setup/cleanup if ctrld could not be direct listener.
+				if !couldBeDirectListener(cfg.FirstListener()) {
 					p.onStarted = append(p.onStarted, func() {
-						mainLog.Debug().Msg("Router setup")
+						mainLog.Debug().Msg("router setup")
 						if err := p.router.Setup(); err != nil {
 							mainLog.Error().Err(err).Msg("could not configure router")
 						}
 					})
 					p.onStopped = append(p.onStopped, func() {
-						mainLog.Debug().Msg("Router cleanup")
+						mainLog.Debug().Msg("router cleanup")
 						if err := p.router.Cleanup(); err != nil {
 							mainLog.Error().Err(err).Msg("could not cleanup router")
 						}
@@ -278,8 +311,6 @@ func initCLI() {
 	_ = runCmd.Flags().MarkHidden("homedir")
 	runCmd.Flags().StringVarP(&iface, "iface", "", "", `Update DNS setting for iface, "auto" means the default interface gateway`)
 	_ = runCmd.Flags().MarkHidden("iface")
-	runCmd.Flags().BoolVarP(&setupRouter, "router", "", false, `setup for running on router platforms`)
-	_ = runCmd.Flags().MarkHidden("router")
 
 	rootCmd.AddCommand(runCmd)
 
@@ -301,11 +332,10 @@ func initCLI() {
 			setDependencies(sc)
 			sc.Arguments = append([]string{"run"}, osArgs...)
 
-			p := &prog{router: router.NewDummyRouter()}
+			p := &prog{
-			if setupRouter {
+				router: router.New(&cfg),
-				p.router = router.New(&cfg)
+				cfg:    &cfg,
 			}
-
 			if err := p.router.ConfigureService(sc); err != nil {
 				mainLog.Fatal().Err(err).Msg("failed to configure service on router")
 			}
@@ -356,10 +386,6 @@ func initCLI() {
 
 			initLogging()
 
-			processCDFlags(p)
-
-			validateConfig(&cfg)
-
 			// Explicitly passing config, so on system where home directory could not be obtained,
 			// or sub-process env is different with the parent, we still behave correctly and use
 			// the expected config file.
@@ -373,8 +399,10 @@ func initCLI() {
 				return
 			}
 
-			mainLog.Debug().Msg("cleaning up router before installing")
+			if router.Name() != "" && !couldBeDirectListener(cfg.FirstListener()) {
-			_ = p.router.Cleanup()
+				mainLog.Debug().Msg("cleaning up router before installing")
+				_ = p.router.Cleanup()
+			}
 
 			tasks := []task{
 				{s.Stop, false},
@@ -431,8 +459,36 @@ func initCLI() {
 	startCmd.Flags().BoolVarP(&cdDev, "dev", "", false, "Use Control D dev resolver/domain")
 	_ = startCmd.Flags().MarkHidden("dev")
 	startCmd.Flags().StringVarP(&iface, "iface", "", "", `Update DNS setting for iface, "auto" means the default interface gateway`)
-	startCmd.Flags().BoolVarP(&setupRouter, "router", "", false, `setup for running on router platforms`)
+
-	_ = startCmd.Flags().MarkHidden("router")
+	routerCmd := &cobra.Command{
+		Use: "setup",
+		PreRun: func(cmd *cobra.Command, args []string) {
+			initConsoleLogging()
+		},
+		Run: func(cmd *cobra.Command, _ []string) {
+			exe, err := os.Executable()
+			if err != nil {
+				mainLog.Fatal().Msgf("could not find executable path: %v", err)
+				os.Exit(1)
+			}
+			flags := make([]string, 0)
+			cmd.Flags().Visit(func(flag *pflag.Flag) {
+				flags = append(flags, fmt.Sprintf("--%s=%s", flag.Name, flag.Value))
+			})
+			cmdArgs := []string{"start"}
+			cmdArgs = append(cmdArgs, flags...)
+			command := exec.Command(exe, cmdArgs...)
+			command.Stdout = os.Stdout
+			command.Stderr = os.Stderr
+			command.Stdin = os.Stdin
+			if err := command.Run(); err != nil {
+				mainLog.Fatal().Msg(err.Error())
+			}
+		},
+	}
+	routerCmd.Flags().AddFlagSet(startCmd.Flags())
+	routerCmd.Hidden = true
+	rootCmd.AddCommand(routerCmd)
 
 	stopCmd := &cobra.Command{
 		PreRun: func(cmd *cobra.Command, args []string) {
@@ -779,10 +835,7 @@ func processNoConfigFlags(noConfigStart bool) {
 	v.Set("upstream", upstream)
 }
 
-func processCDFlags(p *prog) {
+func processCDFlags() {
-	if cdUID == "" {
-		return
-	}
 	logger := mainLog.With().Str("mode", "cd").Logger()
 	logger.Info().Msgf("fetching Controld D configuration from API: %s", cdUID)
 	resolverConfig, err := controld.FetchResolverConfig(cdUID, rootCmd.Version, cdDev)
@@ -817,48 +870,15 @@ func processCDFlags(p *prog) {
 	}
 
 	logger.Info().Msg("generating ctrld config from Control-D configuration")
-	cfg = ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{
+	cfg = ctrld.Config{}
-		"0": {Port: 53},
+
-	}}
+	// Fetch config, unmarshal to cfg.
 	if resolverConfig.Ctrld.CustomConfig != "" {
 		logger.Info().Msg("using defined custom config of Control-D resolver")
 		readBase64Config(resolverConfig.Ctrld.CustomConfig)
 		if err := v.Unmarshal(&cfg); err != nil {
 			mainLog.Fatal().Msgf("failed to unmarshal config: %v", err)
 		}
-		for _, listener := range cfg.Listener {
-			if listener.IP == "" {
-				listener.IP = randomLocalIP()
-			}
-			if listener.Port == 0 {
-				listener.Port = 53
-			}
-		}
-		switch {
-		case setupRouter:
-			if lc := cfg.Listener["0"]; lc != nil && lc.IP == "" {
-				if err := p.router.Configure(); err != nil {
-					mainLog.Fatal().Err(err).Msg("failed to change ctrld config for router")
-				}
-			}
-		case useSystemdResolved:
-			if lc := cfg.Listener["0"]; lc != nil {
-				if ip := net.ParseIP(lc.IP); ip != nil && ip.IsLoopback() {
-					mainLog.Warn().Msg("using loopback interface do not work with systemd-resolved")
-					// systemd-resolved does not allow forwarding DNS queries from 127.0.0.53 to loopback
-					// ip address, so trying to listen on default route interface address instead.
-					if netIface, _ := net.InterfaceByName(defaultIfaceName()); netIface != nil {
-						addrs, _ := netIface.Addrs()
-						for _, addr := range addrs {
-							if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil {
-								lc.IP = netIP.IP.To4().String()
-								mainLog.Warn().Msgf("use %s as listener address", lc.IP)
-							}
-						}
-					}
-				}
-			}
-		}
 	} else {
 		cfg.Network = make(map[string]*ctrld.NetworkConfig)
 		cfg.Network["0"] = &ctrld.NetworkConfig{
@@ -877,27 +897,18 @@ func processCDFlags(p *prog) {
 		}
 		cfg.Listener = make(map[string]*ctrld.ListenerConfig)
 		lc := &ctrld.ListenerConfig{
-			IP:   "127.0.0.1",
-			Port: 53,
 			Policy: &ctrld.ListenerPolicyConfig{
 				Name:  "My Policy",
 				Rules: rules,
 			},
 		}
 		cfg.Listener["0"] = lc
-		if setupRouter {
-			if err := p.router.Configure(); err != nil {
-				mainLog.Fatal().Err(err).Msg("failed to change ctrld config for router")
-			}
-		}
 	}
-
+	// Set default value.
-	processLogAndCacheFlags()
+	if len(cfg.Listener) == 0 {
-
+		cfg.Listener = map[string]*ctrld.ListenerConfig{
-	if err := writeConfigFile(); err != nil {
+			"0": {IP: "", Port: 0},
-		logger.Fatal().Err(err).Msg("failed to write config file")
+		}
-	} else {
-		logger.Info().Msg("writing config file to: " + defaultConfigFile)
 	}
 }
 
@@ -924,9 +935,11 @@ func processListenFlag() {
 
 func processLogAndCacheFlags() {
 	if logPath != "" {
-		cfg.Service.LogLevel = "debug"
 		cfg.Service.LogPath = logPath
 	}
+	if logPath != "" && cfg.Service.LogLevel == "" {
+		cfg.Service.LogLevel = "debug"
+	}
 
 	if cacheSize != 0 {
 		cfg.Service.CacheEnable = true
@@ -979,8 +992,35 @@ func selfCheckStatus(status service.Status, domain string) service.Status {
 	maxAttempts := 20
 	mainLog.Debug().Msg("Performing self-check")
 
+	var (
+		lcChanged map[string]*ctrld.ListenerConfig
+		mu        sync.Mutex
+	)
+	curCfg := cfg
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		mainLog.Error().Err(err).Msg("could not watch config change")
+		return service.StatusUnknown
+	}
+	defer watcher.Close()
+
+	v.OnConfigChange(func(in fsnotify.Event) {
+		mu.Lock()
+		defer mu.Unlock()
+		if err := v.UnmarshalKey("listener", &lcChanged); err != nil {
+			mainLog.Error().Msgf("failed to unmarshal listener config: %v", err)
+			return
+		}
+	})
+	v.WatchConfig()
 	for i := 0; i < maxAttempts; i++ {
-		lc := cfg.Listener["0"]
+		mu.Lock()
+		if lcChanged != nil {
+			curCfg.Listener = lcChanged
+		}
+		mu.Unlock()
+		lc := curCfg.FirstListener()
+
 		m := new(dns.Msg)
 		m.SetQuestion(domain+".", dns.TypeA)
 		m.RecursionDesired = true
@@ -995,10 +1035,6 @@ func selfCheckStatus(status service.Status, domain string) service.Status {
 	return service.StatusUnknown
 }
 
-func unsupportedPlatformHelp(cmd *cobra.Command) {
-	mainLog.Error().Msg("Unsupported or incorrectly chosen router platform. Please open an issue and provide all relevant information: https://github.com/Control-D-Inc/ctrld/issues/new")
-}
-
 func userHomeDir() (string, error) {
 	switch router.Name() {
 	case ddwrt.Name, merlin.Name, tomato.Name:
@@ -1111,3 +1147,161 @@ func fieldErrorMsg(fe validator.FieldError) string {
 	}
 	return ""
 }
+
+// couldBeDirectListener reports whether ctrld can be a direct listener on port 53.
+// It returns true only if ctrld can listen on port 53 for all interfaces. That means
+// there's no other software listening on port 53.
+//
+// If someone listening on port 53, or ctrld could only listen on port 53 for a specific
+// interface, ctrld could only be configured as a DNS forwarder.
+func couldBeDirectListener(lc *ctrld.ListenerConfig) bool {
+	if lc == nil || lc.Port != 53 {
+		return false
+	}
+	switch lc.IP {
+	case "", "::", "0.0.0.0":
+		return true
+	default:
+		return false
+	}
+
+}
+
+func isLoopback(ipStr string) bool {
+	ip := net.ParseIP(ipStr)
+	if ip == nil {
+		return false
+	}
+	return ip.IsLoopback()
+}
+
+func shouldAllocateLoopbackIP(ipStr string) bool {
+	ip := net.ParseIP(ipStr)
+	if ip == nil || ip.To4() == nil {
+		return false
+	}
+	return ip.IsLoopback() && ip.String() != "127.0.0.1"
+}
+
+// updateListenerConfig updates the config for listeners if not defined,
+// or defined but invalid to be used, e.g: using loopback address other
+// than 127.0.0.1 with sytemd-resolved.
+func updateListenerConfig() {
+	for _, listener := range cfg.Listener {
+		if listener.IP == "" {
+			listener.IP = "0.0.0.0"
+		}
+		if listener.Port == 0 {
+			listener.Port = 53
+		}
+	}
+
+	var closers []io.Closer
+	defer func() {
+		for _, closer := range closers {
+			_ = closer.Close()
+		}
+	}()
+	// listenOk reports whether we can listen on udp/tcp of given address.
+	// Created listeners will be kept in listeners slice above, and close
+	// before function finished.
+	listenOk := func(addr string) bool {
+		udpLn, udpErr := net.ListenPacket("udp", addr)
+		if udpLn != nil {
+			closers = append(closers, udpLn)
+		}
+		tcpLn, tcpErr := net.Listen("tcp", addr)
+		if tcpLn != nil {
+			closers = append(closers, tcpLn)
+		}
+		return udpErr == nil && tcpErr == nil
+	}
+
+	listeners := make([]int, 0, len(cfg.Listener))
+	for k := range cfg.Listener {
+		n, err := strconv.Atoi(k)
+		if err != nil {
+			continue
+		}
+		listeners = append(listeners, n)
+	}
+	sort.Ints(listeners)
+
+	for _, n := range listeners {
+		listener := cfg.Listener[strconv.Itoa(n)]
+		oldIP := listener.IP
+		// Check if we could listen on the current IP + Port, if not, try following thing, pick first one success:
+		//    - Try 127.0.0.1:53
+		//    - Pick a random port until success.
+		localhostIP := func(ipStr string) string {
+			if ip := net.ParseIP(ipStr); ip != nil && ip.To4() == nil {
+				return "::1"
+			}
+			return "127.0.0.1"
+		}
+
+		// On firewalla, we don't need to check localhost, because the lo interface is excluded in dnsmasq
+		// config, so we can always listen on localhost port 53, but no traffic could be routed there.
+		tryLocalhost := !isLoopback(listener.IP) && router.Name() != firewalla.Name
+		tryPort5354 := true
+		attempts := 0
+		maxAttempts := 10
+		for {
+			if attempts == maxAttempts {
+				mainLog.Fatal().Msg("could not find available listen ip and port")
+			}
+			addr := net.JoinHostPort(listener.IP, strconv.Itoa(listener.Port))
+			if listenOk(addr) {
+				break
+			}
+			if tryLocalhost {
+				tryLocalhost = false
+				listener.IP = localhostIP(listener.IP)
+				listener.Port = 53
+				mainLog.Warn().Msgf("could not listen on address: %s, trying localhost: %s", addr, net.JoinHostPort(listener.IP, strconv.Itoa(listener.Port)))
+				continue
+			}
+			if tryPort5354 {
+				tryPort5354 = false
+				listener.IP = oldIP
+				listener.Port = 5354
+				mainLog.Warn().Msgf("could not listen on address: %s, trying port 5354", addr)
+				continue
+			}
+			listener.IP = randomLocalIP()
+			listener.Port = randomPort()
+			mainLog.Warn().Msgf("could not listen on address: %s, pick a random ip+port", addr)
+			attempts++
+		}
+	}
+
+	// Specific case for systemd-resolved.
+	if useSystemdResolved {
+		if listener := cfg.FirstListener(); listener != nil && listener.Port == 53 {
+			// systemd-resolved does not allow forwarding DNS queries from 127.0.0.53 to loopback
+			// ip address, other than "127.0.0.1", so trying to listen on default route interface
+			// address instead.
+			if ip := net.ParseIP(listener.IP); ip != nil && ip.IsLoopback() && ip.String() != "127.0.0.1" {
+				mainLog.Warn().Msg("using loopback interface do not work with systemd-resolved")
+				found := false
+				if netIface, _ := net.InterfaceByName(defaultIfaceName()); netIface != nil {
+					addrs, _ := netIface.Addrs()
+					for _, addr := range addrs {
+						if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil {
+							addr := net.JoinHostPort(netIP.IP.String(), strconv.Itoa(listener.Port))
+							if listenOk(addr) {
+								found = true
+								listener.IP = netIP.IP.String()
+								mainLog.Warn().Msgf("use %s as listener address", listener.IP)
+								break
+							}
+						}
+					}
+				}
+				if !found {
+					mainLog.Fatal().Msgf("could not use %q as DNS nameserver with systemd resolved", listener.IP)
+				}
+			}
+		}
+	}
+}

cmd/ctrld/cli_router.go

@@ -1,100 +0,0 @@
-//go:build linux || freebsd
-
-package main
-
-import (
-	"os"
-	"os/exec"
-	"strings"
-
-	"github.com/spf13/cobra"
-
-	"github.com/Control-D-Inc/ctrld/internal/router"
-)
-
-func initRouterCLI() {
-	validArgs := append(router.SupportedPlatforms(), "auto")
-	var b strings.Builder
-	b.WriteString("Auto-setup Control D on a router.\n\nSupported platforms:\n\n")
-	for _, arg := range validArgs {
-		b.WriteString("    ₒ ")
-		b.WriteString(arg)
-		if arg == "auto" {
-			b.WriteString(" - detect the platform you are running on")
-		}
-		b.WriteString("\n")
-	}
-
-	routerCmd := &cobra.Command{
-		Use:   "setup",
-		Short: b.String(),
-		PreRun: func(cmd *cobra.Command, args []string) {
-			initConsoleLogging()
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			if len(args) == 0 {
-				_ = cmd.Help()
-				return
-			}
-			if len(args) != 1 {
-				_ = cmd.Help()
-				return
-			}
-			platform := args[0]
-			if platform == "auto" {
-				platform = router.Name()
-			}
-			if !router.IsSupported(platform) {
-				unsupportedPlatformHelp(cmd)
-				os.Exit(1)
-			}
-
-			exe, err := os.Executable()
-			if err != nil {
-				mainLog.Fatal().Msgf("could not find executable path: %v", err)
-				os.Exit(1)
-			}
-
-			cmdArgs := []string{"start"}
-			cmdArgs = append(cmdArgs, osArgs(platform)...)
-			cmdArgs = append(cmdArgs, "--router")
-			command := exec.Command(exe, cmdArgs...)
-			command.Stdout = os.Stdout
-			command.Stderr = os.Stderr
-			command.Stdin = os.Stdin
-			if err := command.Run(); err != nil {
-				mainLog.Fatal().Msg(err.Error())
-			}
-		},
-	}
-	// Keep these flags in sync with startCmd, except for "--router".
-	routerCmd.Flags().StringVarP(&configPath, "config", "c", "", "Path to config file")
-	routerCmd.Flags().StringVarP(&configBase64, "base64_config", "", "", "Base64 encoded config")
-	routerCmd.Flags().StringVarP(&listenAddress, "listen", "", "", "Listener address and port, in format: address:port")
-	routerCmd.Flags().StringVarP(&primaryUpstream, "primary_upstream", "", "", "Primary upstream endpoint")
-	routerCmd.Flags().StringVarP(&secondaryUpstream, "secondary_upstream", "", "", "Secondary upstream endpoint")
-	routerCmd.Flags().StringSliceVarP(&domains, "domains", "", nil, "List of domain to apply in a split DNS policy")
-	routerCmd.Flags().StringVarP(&logPath, "log", "", "", "Path to log file")
-	routerCmd.Flags().IntVarP(&cacheSize, "cache_size", "", 0, "Enable cache with size items")
-	routerCmd.Flags().StringVarP(&cdUID, "cd", "", "", "Control D resolver uid")
-	routerCmd.Flags().BoolVarP(&cdDev, "dev", "", false, "Use Control D dev resolver/domain")
-	_ = routerCmd.Flags().MarkHidden("dev")
-	routerCmd.Flags().StringVarP(&iface, "iface", "", "", `Update DNS setting for iface, "auto" means the default interface gateway`)
-
-	tmpl := routerCmd.UsageTemplate()
-	tmpl = strings.Replace(tmpl, "{{.UseLine}}", "{{.UseLine}} [platform]", 1)
-	routerCmd.SetUsageTemplate(tmpl)
-	rootCmd.AddCommand(routerCmd)
-}
-
-func osArgs(platform string) []string {
-	args := os.Args[2:]
-	n := 0
-	for _, x := range args {
-		if x != platform && x != "auto" {
-			args[n] = x
-			n++
-		}
-	}
-	return args[:n]
-}

cmd/ctrld/cli_router_others.go

@@ -1,5 +0,0 @@
-//go:build !linux && !freebsd
-
-package main
-
-func initRouterCLI() {}

cmd/ctrld/conn.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"net"
+	"time"
+)
+
+// logConn wraps a net.Conn, override the Write behavior.
+// runCmd uses this wrapper, so as long as startCmd finished,
+// ctrld log won't be flushed with un-necessary write errors.
+type logConn struct {
+	conn net.Conn
+}
+
+func (lc *logConn) Read(b []byte) (n int, err error) {
+	return lc.conn.Read(b)
+}
+
+func (lc *logConn) Close() error {
+	return lc.conn.Close()
+}
+
+func (lc *logConn) LocalAddr() net.Addr {
+	return lc.conn.LocalAddr()
+}
+
+func (lc *logConn) RemoteAddr() net.Addr {
+	return lc.conn.RemoteAddr()
+}
+
+func (lc *logConn) SetDeadline(t time.Time) error {
+	return lc.conn.SetDeadline(t)
+}
+
+func (lc *logConn) SetReadDeadline(t time.Time) error {
+	return lc.conn.SetReadDeadline(t)
+}
+
+func (lc *logConn) SetWriteDeadline(t time.Time) error {
+	return lc.conn.SetWriteDeadline(t)
+}
+
+func (lc *logConn) Write(b []byte) (int, error) {
+	// Write performs writes with underlying net.Conn, ignore any errors happen.
+	// "ctrld run" command use this wrapper to report errors to "ctrld start".
+	// If no error occurred, "ctrld start" may finish before "ctrld run" attempt
+	// to close the connection, so ignore errors conservatively here, prevent
+	// un-necessary error "write to closed connection" flushed to ctrld log.
+	_, _ = lc.conn.Write(b)
+	return len(b), nil
+}

cmd/ctrld/dns_proxy.go

@@ -93,7 +93,7 @@ func (p *prog) serveDNS(listenerNum string) error {
 			})
 		}
 		g.Go(func() error {
-			s, errCh := runDNSServer(dnsListenAddress(listenerNum, listenerConfig), proto, handler)
+			s, errCh := runDNSServer(dnsListenAddress(listenerConfig), proto, handler)
 			defer s.Shutdown()
 			if listenerConfig.Port == 0 {
 				switch s.Net {
@@ -400,12 +400,13 @@ func needLocalIPv6Listener() bool {
 	return ctrldnet.SupportsIPv6ListenLocal() && runtime.GOOS == "windows"
 }
 
-func dnsListenAddress(lcNum string, lc *ctrld.ListenerConfig) string {
+func dnsListenAddress(lc *ctrld.ListenerConfig) string {
-	addr := net.JoinHostPort(lc.IP, strconv.Itoa(lc.Port))
+	// If we are inside container and the listener loopback address, change
-	// If we are inside container and the listener address is localhost,
+	// the address to something like 0.0.0.0:53, so user can expose the port to outside.
-	// Change it to 0.0.0.0:53, so user can expose the port to outside.
+	if inContainer() {
-	if addr == "127.0.0.1:53" && cdUID != "" && inContainer() {
+		if ip := net.ParseIP(lc.IP); ip != nil && ip.IsLoopback() {
-		return "0.0.0.0:53"
+			return net.JoinHostPort("0.0.0.0", strconv.Itoa(lc.Port))
+		}
 	}
 	return net.JoinHostPort(lc.IP, strconv.Itoa(lc.Port))
 }

cmd/ctrld/main.go

@@ -30,7 +30,6 @@ var (
 	cdDev             bool
 	iface             string
 	ifaceStartStop    string
-	setupRouter       bool
 
 	mainLog       = zerolog.New(io.Discard)
 	consoleWriter zerolog.ConsoleWriter
@@ -39,7 +38,6 @@ var (
 func main() {
 	ctrld.InitConfig(v, "ctrld")
 	initCLI()
-	initRouterCLI()
 	if err := rootCmd.Execute(); err != nil {
 		mainLog.Error().Msg(err.Error())
 		os.Exit(1)

cmd/ctrld/prog.go

@@ -16,10 +16,7 @@ import (
 	"github.com/Control-D-Inc/ctrld/internal/clientinfo"
 	"github.com/Control-D-Inc/ctrld/internal/dnscache"
 	"github.com/Control-D-Inc/ctrld/internal/router"
-	"github.com/Control-D-Inc/ctrld/internal/router/ddwrt"
 	"github.com/Control-D-Inc/ctrld/internal/router/firewalla"
-	"github.com/Control-D-Inc/ctrld/internal/router/openwrt"
-	"github.com/Control-D-Inc/ctrld/internal/router/ubios"
 )
 
 const (
@@ -221,16 +218,7 @@ func (p *prog) deAllocateIP() error {
 }
 
 func (p *prog) setDNS() {
-	switch router.Name() {
+	if cfg.Listener == nil {
-	case ddwrt.Name, openwrt.Name, ubios.Name:
-		// On router, ctrld run as a DNS forwarder, it does not have to change system DNS.
-		// Except for:
-		//   + EdgeOS, which /etc/resolv.conf could be managed by vyatta_update_resolv.pl script.
-		//   + Merlin/Tomato, which has WAN DNS setup on boot for NTP.
-		//   + Synology, which /etc/resolv.conf is not configured to point to localhost.
-		return
-	}
-	if cfg.Listener == nil || cfg.Listener["0"] == nil {
 		return
 	}
 	if iface == "" {
@@ -239,6 +227,10 @@ func (p *prog) setDNS() {
 	if iface == "auto" {
 		iface = defaultIfaceName()
 	}
+	lc := cfg.FirstListener()
+	if lc == nil {
+		return
+	}
 	logger := mainLog.With().Str("iface", iface).Logger()
 	netIface, err := netInterface(iface)
 	if err != nil {
@@ -250,23 +242,29 @@ func (p *prog) setDNS() {
 		return
 	}
 	logger.Debug().Msg("setting DNS for interface")
-	ns := cfg.Listener["0"].IP
+	ns := lc.IP
-	if router.Name() == firewalla.Name && (ns == "127.0.0.1" || ns == "0.0.0.0" || ns == "") {
+	ifaceName := defaultIfaceName()
+	isFirewalla := router.Name() == firewalla.Name
+	if isFirewalla {
 		// On Firewalla, the lo interface is excluded in all dnsmasq settings of all interfaces.
 		// Thus, we use "br0" as the nameserver in /etc/resolv.conf file.
-		if ns == "127.0.0.1" {
+		ifaceName = "br0"
-			logger.Warn().Msg("127.0.0.1 as DNS server won't work on Firewalla")
+		logger.Warn().Msg("using br0 interface IP address as DNS server")
-		} else {
+	}
-			logger.Warn().Msgf("%q could not be used as DNS server", ns)
+	if couldBeDirectListener(lc) {
+		// If ctrld is direct listener, use 127.0.0.1 as nameserver.
+		ns = "127.0.0.1"
+	} else if lc.Port != 53 {
+		logger.Warn().Msg("ctrld is not running on port 53, use default route interface as DNS server")
+		netIface, err := net.InterfaceByName(ifaceName)
+		if err != nil {
+			mainLog.Fatal().Err(err).Msg("failed to get default route interface")
 		}
-		if netIface, err := net.InterfaceByName("br0"); err == nil {
+		addrs, _ := netIface.Addrs()
-			addrs, _ := netIface.Addrs()
+		for _, addr := range addrs {
-			for _, addr := range addrs {
+			if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil {
-				if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil {
+				ns = netIP.IP.To4().String()
-					logger.Warn().Msg("using br0 interface IP address as DNS server")
+				break
-					ns = netIP.IP.To4().String()
-					break
-				}
 			}
 		}
 	}
@@ -278,11 +276,6 @@ func (p *prog) setDNS() {
 }
 
 func (p *prog) resetDNS() {
-	switch router.Name() {
-	case ddwrt.Name, openwrt.Name, ubios.Name:
-		// See comment in p.setDNS method.
-		return
-	}
 	if iface == "" {
 		return
 	}
@@ -312,6 +305,13 @@ func randomLocalIP() string {
 	return fmt.Sprintf("127.0.0.%d", n)
 }
 
+func randomPort() int {
+	max := 1<<16 - 1
+	min := 1025
+	n := rand.Intn(max-min) + min
+	return n
+}
+
 // runLogServer starts a unix listener, use by startCmd to gather log from runCmd.
 func runLogServer(sockPath string) net.Conn {
 	addr, err := net.ResolveUnixAddr("unix", sockPath)

config.go

@@ -12,6 +12,8 @@ import (
 	"net/url"
 	"os"
 	"runtime"
+	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -123,6 +125,25 @@ func (c *Config) HasUpstreamSendClientInfo() bool {
 	return false
 }
 
+// FirstListener returns the first listener config of current config. Listeners are sorted numerically.
+//
+// It panics if Config has no listeners configured.
+func (c *Config) FirstListener() *ListenerConfig {
+	listeners := make([]int, 0, len(c.Listener))
+	for k := range c.Listener {
+		n, err := strconv.Atoi(k)
+		if err != nil {
+			continue
+		}
+		listeners = append(listeners, n)
+	}
+	if len(listeners) == 0 {
+		panic("missing listener config")
+	}
+	sort.Ints(listeners)
+	return c.Listener[strconv.Itoa(listeners[0])]
+}
+
 // ServiceConfig specifies the general ctrld config.
 type ServiceConfig struct {
 	LogLevel              string `mapstructure:"log_level" toml:"log_level,omitempty"`

config_test.go

@@ -1,6 +1,7 @@
 package ctrld_test
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/go-playground/validator/v10"
@@ -56,6 +57,20 @@ func TestLoadDefaultConfig(t *testing.T) {
 	assert.Len(t, cfg.Upstream, 2)
 }
 
+func TestConfigOverride(t *testing.T) {
+	v := viper.NewWithOptions(viper.KeyDelimiter("::"))
+	ctrld.InitConfig(v, "test_load_config")
+	v.SetConfigType("toml")
+	require.NoError(t, v.ReadConfig(strings.NewReader(testhelper.SampleConfigStr(t))))
+	cfg := ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{
+		"0": {IP: "127.0.0.1", Port: 53},
+	}}
+	require.NoError(t, v.Unmarshal(&cfg))
+
+	assert.Equal(t, "10.10.42.69", cfg.Listener["1"].IP)
+	assert.Equal(t, 1337, cfg.Listener["1"].Port)
+}
+
 func TestConfigValidation(t *testing.T) {
 	tests := []struct {
 		name    string

internal/dns/nm.go

@@ -267,5 +267,5 @@ func (m *nmManager) Close() error {
 }
 
 func (m *nmManager) Mode() string {
-	return "network-maanger"
+	return "network-manager"
 }

internal/router/ddwrt/ddwrt.go

@@ -57,12 +57,6 @@ func (d *Ddwrt) PreRun() error {
 	return ntp.Wait()
 }
 
-func (d *Ddwrt) Configure() error {
-	d.cfg.Listener["0"].IP = "127.0.0.1"
-	d.cfg.Listener["0"].Port = 5354
-	return nil
-}
-
 func (d *Ddwrt) Setup() error {
 	// Already setup.
 	if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val == "1" {

internal/router/dnsmasq/dnsmasq.go

@@ -1,6 +1,7 @@
 package dnsmasq
 
 import (
+	"errors"
 	"html/template"
 	"net"
 	"path/filepath"
@@ -60,15 +61,20 @@ type Upstream struct {
 }
 
 func ConfTmpl(tmplText string, cfg *ctrld.Config) (string, error) {
-	upstreams := make([]Upstream, 0, len(cfg.Listener))
+	listener := cfg.FirstListener()
-	for _, listener := range cfg.Listener {
+	if listener == nil {
-		upstreams = append(upstreams, Upstream{Ip: listener.IP, Port: listener.Port})
+		return "", errors.New("missing listener")
 	}
+	ip := listener.IP
+	if ip == "0.0.0.0" || ip == "::" || ip == "" {
+		ip = "127.0.0.1"
+	}
+	upstreams := []Upstream{{Ip: ip, Port: listener.Port}}
 	return confTmpl(tmplText, upstreams, cfg.HasUpstreamSendClientInfo())
 }
 
 func FirewallaConfTmpl(tmplText string, cfg *ctrld.Config) (string, error) {
-	if lc := cfg.Listener["0"]; lc != nil && lc.IP == "0.0.0.0" {
+	if lc := cfg.FirstListener(); lc != nil && (lc.IP == "0.0.0.0" || lc.IP == "") {
 		return confTmpl(tmplText, firewallaUpstreams(lc.Port), cfg.HasUpstreamSendClientInfo())
 	}
 	return ConfTmpl(tmplText, cfg)

internal/router/dummy.go

@@ -4,10 +4,6 @@ import "github.com/kardianos/service"
 
 type dummy struct{}
 
-func NewDummyRouter() Router {
-	return &dummy{}
-}
-
 func (d *dummy) ConfigureService(_ *service.Config) error {
 	return nil
 }

internal/router/edgeos/edgeos.go

@@ -61,12 +61,6 @@ func (e *EdgeOS) PreRun() error {
 	return nil
 }
 
-func (e *EdgeOS) Configure() error {
-	e.cfg.Listener["0"].IP = "127.0.0.1"
-	e.cfg.Listener["0"].Port = 5354
-	return nil
-}
-
 func (e *EdgeOS) Setup() error {
 	if e.isUSG {
 		return e.setupUSG()

internal/router/firewalla/firewalla.go

@@ -53,12 +53,6 @@ func (f *Firewalla) PreRun() error {
 	return nil
 }
 
-func (f *Firewalla) Configure() error {
-	f.cfg.Listener["0"].IP = "0.0.0.0"
-	f.cfg.Listener["0"].Port = 5354
-	return nil
-}
-
 func (f *Firewalla) Setup() error {
 	data, err := dnsmasq.FirewallaConfTmpl(dnsmasq.ConfigContentTmpl, f.cfg)
 	if err != nil {

internal/router/merlin/merlin.go

@@ -48,12 +48,6 @@ func (m *Merlin) PreRun() error {
 	return ntp.Wait()
 }
 
-func (m *Merlin) Configure() error {
-	m.cfg.Listener["0"].IP = "127.0.0.1"
-	m.cfg.Listener["0"].Port = 5354
-	return nil
-}
-
 func (m *Merlin) Setup() error {
 	buf, err := os.ReadFile(dnsmasq.MerlinPostConfPath)
 	// Already setup.

internal/router/openwrt/openwrt.go

@@ -48,12 +48,6 @@ func (o *Openwrt) PreRun() error {
 	return nil
 }
 
-func (o *Openwrt) Configure() error {
-	o.cfg.Listener["0"].IP = "127.0.0.1"
-	o.cfg.Listener["0"].Port = 5354
-	return nil
-}
-
 func (o *Openwrt) Setup() error {
 	// Delete dnsmasq port if set.
 	if _, err := uci("delete", "dhcp.@dnsmasq[0].port"); err != nil && !errors.Is(err, errUCIEntryNotFound) {

internal/router/pfsense/pfsense.go

@@ -54,6 +54,14 @@ func (p *Pfsense) ConfigureService(svc *service.Config) error {
 }
 
 func (p *Pfsense) Install(config *service.Config) error {
+	// pfsense need ".sh" extension for script to be run at boot.
+	// See: https://docs.netgate.com/pfsense/en/latest/development/boot-commands.html#shell-script-option
+	oldname := filepath.Join(rcPath, p.svcName)
+	newname := filepath.Join(rcPath, p.svcName+".sh")
+	_ = os.Remove(newname)
+	if err := os.Symlink(oldname, newname); err != nil {
+		return fmt.Errorf("os.Symlink: %w", err)
+	}
 	return nil
 }
 
@@ -62,16 +70,7 @@ func (p *Pfsense) Uninstall(config *service.Config) error {
 }
 
 func (p *Pfsense) PreRun() error {
-	return nil
+	// TODO: remove this hacky solution.
-}
-
-func (p *Pfsense) Configure() error {
-	p.cfg.Listener["0"].IP = "127.0.0.1"
-	p.cfg.Listener["0"].Port = 53
-	return nil
-}
-
-func (p *Pfsense) Setup() error {
 	// If Pfsense is in DNS Resolver mode, ensure no unbound processes running.
 	_ = exec.Command("killall", "unbound").Run()
 
@@ -80,6 +79,10 @@ func (p *Pfsense) Setup() error {
 	return nil
 }
 
+func (p *Pfsense) Setup() error {
+	return nil
+}
+
 func (p *Pfsense) Cleanup() error {
 	if err := os.Remove(filepath.Join(rcPath, p.svcName+".sh")); err != nil {
 		return fmt.Errorf("os.Remove: %w", err)

internal/router/router.go

@@ -27,15 +27,9 @@ type Service interface {
 	Uninstall(*service.Config) error
 }
 
-// Config is the interface to manage ctrld config on router.
-type Config interface {
-	Configure() error
-}
-
 // Router is the interface for managing ctrld running on router.
 type Router interface {
 	Service
-	Config
 
 	PreRun() error
 	Setup() error
@@ -64,7 +58,7 @@ func New(cfg *ctrld.Config) Router {
 	case firewalla.Name:
 		return firewalla.New(cfg)
 	}
-	return NewDummyRouter()
+	return &dummy{}
 }
 
 // IsGLiNet reports whether the router is an GL.iNet router.
@@ -94,38 +88,6 @@ type router struct {
 	sendClientInfo bool
 }
 
-// IsSupported reports whether the given platform is supported by ctrld.
-func IsSupported(platform string) bool {
-	switch platform {
-	case ddwrt.Name,
-		edgeos.Name,
-		firewalla.Name,
-		merlin.Name,
-		openwrt.Name,
-		pfsense.Name,
-		synology.Name,
-		tomato.Name,
-		ubios.Name:
-		return true
-	}
-	return false
-}
-
-// SupportedPlatforms return all platforms that can be configured to run with ctrld.
-func SupportedPlatforms() []string {
-	return []string{
-		ddwrt.Name,
-		edgeos.Name,
-		firewalla.Name,
-		merlin.Name,
-		openwrt.Name,
-		pfsense.Name,
-		synology.Name,
-		tomato.Name,
-		ubios.Name,
-	}
-}
-
 // Name returns name of the router platform.
 func Name() string {
 	if r := routerPlatform.Load(); r != nil {

internal/router/synology/synology.go

@@ -43,12 +43,6 @@ func (s *Synology) PreRun() error {
 	return nil
 }
 
-func (s *Synology) Configure() error {
-	s.cfg.Listener["0"].IP = "127.0.0.1"
-	s.cfg.Listener["0"].Port = 5354
-	return nil
-}
-
 func (s *Synology) Setup() error {
 	data, err := dnsmasq.ConfTmpl(dnsmasq.ConfigContentTmpl, s.cfg)
 	if err != nil {

internal/router/tomato/tomato.go

@@ -52,12 +52,6 @@ func (f *FreshTomato) PreRun() error {
 	return ntp.Wait()
 }
 
-func (f *FreshTomato) Configure() error {
-	f.cfg.Listener["0"].IP = "127.0.0.1"
-	f.cfg.Listener["0"].Port = 5354
-	return nil
-}
-
 func (f *FreshTomato) Setup() error {
 	// Already setup.
 	if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val == "1" {

internal/router/ubios/ubios.go

@@ -46,12 +46,6 @@ func (u *Ubios) PreRun() error {
 	return nil
 }
 
-func (u *Ubios) Configure() error {
-	u.cfg.Listener["0"].IP = "127.0.0.1"
-	u.cfg.Listener["0"].Port = 5354
-	return nil
-}
-
 func (u *Ubios) Setup() error {
 	data, err := dnsmasq.ConfTmpl(dnsmasq.ConfigContentTmpl, u.cfg)
 	if err != nil {

testhelper/config.go

@@ -19,6 +19,10 @@ func SampleConfig(t *testing.T) *ctrld.Config {
 	return &cfg
 }
 
+func SampleConfigStr(t *testing.T) string {
+	return sampleConfigContent
+}
+
 var sampleConfigContent = `
 [service]
 log_level = "info"