From 7af59ee589216359abd28dcdabd6f962f283e55a Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 7 Jul 2023 21:07:26 +0700 Subject: [PATCH] all: rework fetching/generating config in cd mode Config fetching/generating in cd mode is currently weird, error prone, and easy for user to break ctrld when using custom config. This commit reworks the flow: - Fetching config from Control D API. - No custom config, use the current default config. - If custom config presents, but there's no listener, use 0.0.0.0:53. - Try listening on current ip+port config, if ok, ctrld could be a direct listener with current setup, moving on. - If failed, trying 127.0.0.1:53. - If failed, trying current ip + port 5354 - If still failed, pick a random ip:port pair, retry until listening ok. With this flow, thing is more predictable/stable, and help removing the Config interface for router. --- cmd/ctrld/cli.go | 378 +++++++++++++++++++------ cmd/ctrld/cli_router.go | 100 ------- cmd/ctrld/cli_router_others.go | 5 - cmd/ctrld/conn.go | 51 ++++ cmd/ctrld/dns_proxy.go | 15 +- cmd/ctrld/main.go | 2 - cmd/ctrld/prog.go | 64 ++--- config.go | 21 ++ config_test.go | 15 + internal/dns/nm.go | 2 +- internal/router/ddwrt/ddwrt.go | 6 - internal/router/dnsmasq/dnsmasq.go | 14 +- internal/router/dummy.go | 4 - internal/router/edgeos/edgeos.go | 6 - internal/router/firewalla/firewalla.go | 6 - internal/router/merlin/merlin.go | 6 - internal/router/openwrt/openwrt.go | 6 - internal/router/pfsense/pfsense.go | 23 +- internal/router/router.go | 40 +-- internal/router/synology/synology.go | 6 - internal/router/tomato/tomato.go | 6 - internal/router/ubios/ubios.go | 6 - testhelper/config.go | 4 + 23 files changed, 442 insertions(+), 344 deletions(-) delete mode 100644 cmd/ctrld/cli_router.go delete mode 100644 cmd/ctrld/cli_router_others.go create mode 100644 cmd/ctrld/conn.go diff --git a/cmd/ctrld/cli.go b/cmd/ctrld/cli.go index 1ac9893..e6c40f3 100644 --- a/cmd/ctrld/cli.go +++ b/cmd/ctrld/cli.go @@ -15,17 +15,21 @@ import ( "path/filepath" "reflect" "runtime" + "sort" "strconv" "strings" + "sync" "time" "github.com/cuonglm/osinfo" + "github.com/fsnotify/fsnotify" "github.com/go-playground/validator/v10" "github.com/kardianos/service" "github.com/miekg/dns" "github.com/pelletier/go-toml/v2" "github.com/rs/zerolog" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/spf13/viper" "tailscale.com/logtail/backoff" "tailscale.com/net/interfaces" @@ -36,6 +40,7 @@ import ( ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" "github.com/Control-D-Inc/ctrld/internal/router" "github.com/Control-D-Inc/ctrld/internal/router/ddwrt" + "github.com/Control-D-Inc/ctrld/internal/router/firewalla" "github.com/Control-D-Inc/ctrld/internal/router/merlin" "github.com/Control-D-Inc/ctrld/internal/router/tomato" ) @@ -126,12 +131,14 @@ func initCLI() { p := &prog{ waitCh: waitCh, stopCh: stopCh, + cfg: &cfg, } sockPath := filepath.Join(homedir, ctrldLogUnixSock) if addr, err := net.ResolveUnixAddr("unix", sockPath); err == nil { if conn, err := net.Dial(addr.Network(), addr.String()); err == nil { - consoleWriter.Out = io.MultiWriter(os.Stdout, conn) - p.logConn = conn + lc := &logConn{conn: conn} + consoleWriter.Out = io.MultiWriter(os.Stdout, lc) + p.logConn = lc } } @@ -177,10 +184,7 @@ func initCLI() { mainLog.Fatal().Msg("network is not up yet") } - p.router = router.NewDummyRouter() - if setupRouter { - p.router = router.New(&cfg) - } + p.router = router.New(&cfg) // Processing --cd flag require connecting to ControlD API, which needs valid // time for validating server certificate. Some routers need NTP synchronization @@ -190,7 +194,22 @@ func initCLI() { } oldLogPath := cfg.Service.LogPath - processCDFlags(p) + if cdUID != "" { + processCDFlags() + } + + updateListenerConfig() + + if cdUID != "" { + processLogAndCacheFlags() + } + + if err := writeConfigFile(); err != nil { + mainLog.Fatal().Err(err).Msg("failed to write config file") + } else { + mainLog.Info().Msg("writing config file to: " + defaultConfigFile) + } + if newLogPath := cfg.Service.LogPath; newLogPath != "" && oldLogPath != newLogPath { // After processCDFlags, log config may change, so reset mainLog and re-init logging. mainLog = zerolog.New(io.Discard) @@ -229,24 +248,38 @@ func initCLI() { os.Exit(0) } - if setupRouter { - switch platform := router.Name(); { - case platform == ddwrt.Name: - rootCertPool = certs.CACertPool() - fallthrough - case platform != "": - if !router.IsSupported(platform) { - unsupportedPlatformHelp(cmd) - os.Exit(1) + p.onStarted = append(p.onStarted, func() { + for _, lc := range p.cfg.Listener { + if shouldAllocateLoopbackIP(lc.IP) { + if err := allocateIP(lc.IP); err != nil { + mainLog.Error().Err(err).Msgf("could not allocate IP: %s", lc.IP) + } } + } + }) + p.onStopped = append(p.onStopped, func() { + for _, lc := range p.cfg.Listener { + if shouldAllocateLoopbackIP(lc.IP) { + if err := deAllocateIP(lc.IP); err != nil { + mainLog.Error().Err(err).Msgf("could not de-allocate IP: %s", lc.IP) + } + } + } + }) + if platform := router.Name(); platform != "" { + if platform == ddwrt.Name { + rootCertPool = certs.CACertPool() + } + // Perform router setup/cleanup if ctrld could not be direct listener. + if !couldBeDirectListener(cfg.FirstListener()) { p.onStarted = append(p.onStarted, func() { - mainLog.Debug().Msg("Router setup") + mainLog.Debug().Msg("router setup") if err := p.router.Setup(); err != nil { mainLog.Error().Err(err).Msg("could not configure router") } }) p.onStopped = append(p.onStopped, func() { - mainLog.Debug().Msg("Router cleanup") + mainLog.Debug().Msg("router cleanup") if err := p.router.Cleanup(); err != nil { mainLog.Error().Err(err).Msg("could not cleanup router") } @@ -278,8 +311,6 @@ func initCLI() { _ = runCmd.Flags().MarkHidden("homedir") runCmd.Flags().StringVarP(&iface, "iface", "", "", `Update DNS setting for iface, "auto" means the default interface gateway`) _ = runCmd.Flags().MarkHidden("iface") - runCmd.Flags().BoolVarP(&setupRouter, "router", "", false, `setup for running on router platforms`) - _ = runCmd.Flags().MarkHidden("router") rootCmd.AddCommand(runCmd) @@ -301,11 +332,10 @@ func initCLI() { setDependencies(sc) sc.Arguments = append([]string{"run"}, osArgs...) - p := &prog{router: router.NewDummyRouter()} - if setupRouter { - p.router = router.New(&cfg) + p := &prog{ + router: router.New(&cfg), + cfg: &cfg, } - if err := p.router.ConfigureService(sc); err != nil { mainLog.Fatal().Err(err).Msg("failed to configure service on router") } @@ -356,10 +386,6 @@ func initCLI() { initLogging() - processCDFlags(p) - - validateConfig(&cfg) - // Explicitly passing config, so on system where home directory could not be obtained, // or sub-process env is different with the parent, we still behave correctly and use // the expected config file. @@ -373,8 +399,10 @@ func initCLI() { return } - mainLog.Debug().Msg("cleaning up router before installing") - _ = p.router.Cleanup() + if router.Name() != "" && !couldBeDirectListener(cfg.FirstListener()) { + mainLog.Debug().Msg("cleaning up router before installing") + _ = p.router.Cleanup() + } tasks := []task{ {s.Stop, false}, @@ -431,8 +459,36 @@ func initCLI() { startCmd.Flags().BoolVarP(&cdDev, "dev", "", false, "Use Control D dev resolver/domain") _ = startCmd.Flags().MarkHidden("dev") startCmd.Flags().StringVarP(&iface, "iface", "", "", `Update DNS setting for iface, "auto" means the default interface gateway`) - startCmd.Flags().BoolVarP(&setupRouter, "router", "", false, `setup for running on router platforms`) - _ = startCmd.Flags().MarkHidden("router") + + routerCmd := &cobra.Command{ + Use: "setup", + PreRun: func(cmd *cobra.Command, args []string) { + initConsoleLogging() + }, + Run: func(cmd *cobra.Command, _ []string) { + exe, err := os.Executable() + if err != nil { + mainLog.Fatal().Msgf("could not find executable path: %v", err) + os.Exit(1) + } + flags := make([]string, 0) + cmd.Flags().Visit(func(flag *pflag.Flag) { + flags = append(flags, fmt.Sprintf("--%s=%s", flag.Name, flag.Value)) + }) + cmdArgs := []string{"start"} + cmdArgs = append(cmdArgs, flags...) + command := exec.Command(exe, cmdArgs...) + command.Stdout = os.Stdout + command.Stderr = os.Stderr + command.Stdin = os.Stdin + if err := command.Run(); err != nil { + mainLog.Fatal().Msg(err.Error()) + } + }, + } + routerCmd.Flags().AddFlagSet(startCmd.Flags()) + routerCmd.Hidden = true + rootCmd.AddCommand(routerCmd) stopCmd := &cobra.Command{ PreRun: func(cmd *cobra.Command, args []string) { @@ -779,10 +835,7 @@ func processNoConfigFlags(noConfigStart bool) { v.Set("upstream", upstream) } -func processCDFlags(p *prog) { - if cdUID == "" { - return - } +func processCDFlags() { logger := mainLog.With().Str("mode", "cd").Logger() logger.Info().Msgf("fetching Controld D configuration from API: %s", cdUID) resolverConfig, err := controld.FetchResolverConfig(cdUID, rootCmd.Version, cdDev) @@ -817,48 +870,15 @@ func processCDFlags(p *prog) { } logger.Info().Msg("generating ctrld config from Control-D configuration") - cfg = ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{ - "0": {Port: 53}, - }} + cfg = ctrld.Config{} + + // Fetch config, unmarshal to cfg. if resolverConfig.Ctrld.CustomConfig != "" { logger.Info().Msg("using defined custom config of Control-D resolver") readBase64Config(resolverConfig.Ctrld.CustomConfig) if err := v.Unmarshal(&cfg); err != nil { mainLog.Fatal().Msgf("failed to unmarshal config: %v", err) } - for _, listener := range cfg.Listener { - if listener.IP == "" { - listener.IP = randomLocalIP() - } - if listener.Port == 0 { - listener.Port = 53 - } - } - switch { - case setupRouter: - if lc := cfg.Listener["0"]; lc != nil && lc.IP == "" { - if err := p.router.Configure(); err != nil { - mainLog.Fatal().Err(err).Msg("failed to change ctrld config for router") - } - } - case useSystemdResolved: - if lc := cfg.Listener["0"]; lc != nil { - if ip := net.ParseIP(lc.IP); ip != nil && ip.IsLoopback() { - mainLog.Warn().Msg("using loopback interface do not work with systemd-resolved") - // systemd-resolved does not allow forwarding DNS queries from 127.0.0.53 to loopback - // ip address, so trying to listen on default route interface address instead. - if netIface, _ := net.InterfaceByName(defaultIfaceName()); netIface != nil { - addrs, _ := netIface.Addrs() - for _, addr := range addrs { - if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil { - lc.IP = netIP.IP.To4().String() - mainLog.Warn().Msgf("use %s as listener address", lc.IP) - } - } - } - } - } - } } else { cfg.Network = make(map[string]*ctrld.NetworkConfig) cfg.Network["0"] = &ctrld.NetworkConfig{ @@ -877,27 +897,18 @@ func processCDFlags(p *prog) { } cfg.Listener = make(map[string]*ctrld.ListenerConfig) lc := &ctrld.ListenerConfig{ - IP: "127.0.0.1", - Port: 53, Policy: &ctrld.ListenerPolicyConfig{ Name: "My Policy", Rules: rules, }, } cfg.Listener["0"] = lc - if setupRouter { - if err := p.router.Configure(); err != nil { - mainLog.Fatal().Err(err).Msg("failed to change ctrld config for router") - } - } } - - processLogAndCacheFlags() - - if err := writeConfigFile(); err != nil { - logger.Fatal().Err(err).Msg("failed to write config file") - } else { - logger.Info().Msg("writing config file to: " + defaultConfigFile) + // Set default value. + if len(cfg.Listener) == 0 { + cfg.Listener = map[string]*ctrld.ListenerConfig{ + "0": {IP: "", Port: 0}, + } } } @@ -924,9 +935,11 @@ func processListenFlag() { func processLogAndCacheFlags() { if logPath != "" { - cfg.Service.LogLevel = "debug" cfg.Service.LogPath = logPath } + if logPath != "" && cfg.Service.LogLevel == "" { + cfg.Service.LogLevel = "debug" + } if cacheSize != 0 { cfg.Service.CacheEnable = true @@ -979,8 +992,35 @@ func selfCheckStatus(status service.Status, domain string) service.Status { maxAttempts := 20 mainLog.Debug().Msg("Performing self-check") + var ( + lcChanged map[string]*ctrld.ListenerConfig + mu sync.Mutex + ) + curCfg := cfg + watcher, err := fsnotify.NewWatcher() + if err != nil { + mainLog.Error().Err(err).Msg("could not watch config change") + return service.StatusUnknown + } + defer watcher.Close() + + v.OnConfigChange(func(in fsnotify.Event) { + mu.Lock() + defer mu.Unlock() + if err := v.UnmarshalKey("listener", &lcChanged); err != nil { + mainLog.Error().Msgf("failed to unmarshal listener config: %v", err) + return + } + }) + v.WatchConfig() for i := 0; i < maxAttempts; i++ { - lc := cfg.Listener["0"] + mu.Lock() + if lcChanged != nil { + curCfg.Listener = lcChanged + } + mu.Unlock() + lc := curCfg.FirstListener() + m := new(dns.Msg) m.SetQuestion(domain+".", dns.TypeA) m.RecursionDesired = true @@ -995,10 +1035,6 @@ func selfCheckStatus(status service.Status, domain string) service.Status { return service.StatusUnknown } -func unsupportedPlatformHelp(cmd *cobra.Command) { - mainLog.Error().Msg("Unsupported or incorrectly chosen router platform. Please open an issue and provide all relevant information: https://github.com/Control-D-Inc/ctrld/issues/new") -} - func userHomeDir() (string, error) { switch router.Name() { case ddwrt.Name, merlin.Name, tomato.Name: @@ -1111,3 +1147,161 @@ func fieldErrorMsg(fe validator.FieldError) string { } return "" } + +// couldBeDirectListener reports whether ctrld can be a direct listener on port 53. +// It returns true only if ctrld can listen on port 53 for all interfaces. That means +// there's no other software listening on port 53. +// +// If someone listening on port 53, or ctrld could only listen on port 53 for a specific +// interface, ctrld could only be configured as a DNS forwarder. +func couldBeDirectListener(lc *ctrld.ListenerConfig) bool { + if lc == nil || lc.Port != 53 { + return false + } + switch lc.IP { + case "", "::", "0.0.0.0": + return true + default: + return false + } + +} + +func isLoopback(ipStr string) bool { + ip := net.ParseIP(ipStr) + if ip == nil { + return false + } + return ip.IsLoopback() +} + +func shouldAllocateLoopbackIP(ipStr string) bool { + ip := net.ParseIP(ipStr) + if ip == nil || ip.To4() == nil { + return false + } + return ip.IsLoopback() && ip.String() != "127.0.0.1" +} + +// updateListenerConfig updates the config for listeners if not defined, +// or defined but invalid to be used, e.g: using loopback address other +// than 127.0.0.1 with sytemd-resolved. +func updateListenerConfig() { + for _, listener := range cfg.Listener { + if listener.IP == "" { + listener.IP = "0.0.0.0" + } + if listener.Port == 0 { + listener.Port = 53 + } + } + + var closers []io.Closer + defer func() { + for _, closer := range closers { + _ = closer.Close() + } + }() + // listenOk reports whether we can listen on udp/tcp of given address. + // Created listeners will be kept in listeners slice above, and close + // before function finished. + listenOk := func(addr string) bool { + udpLn, udpErr := net.ListenPacket("udp", addr) + if udpLn != nil { + closers = append(closers, udpLn) + } + tcpLn, tcpErr := net.Listen("tcp", addr) + if tcpLn != nil { + closers = append(closers, tcpLn) + } + return udpErr == nil && tcpErr == nil + } + + listeners := make([]int, 0, len(cfg.Listener)) + for k := range cfg.Listener { + n, err := strconv.Atoi(k) + if err != nil { + continue + } + listeners = append(listeners, n) + } + sort.Ints(listeners) + + for _, n := range listeners { + listener := cfg.Listener[strconv.Itoa(n)] + oldIP := listener.IP + // Check if we could listen on the current IP + Port, if not, try following thing, pick first one success: + // - Try 127.0.0.1:53 + // - Pick a random port until success. + localhostIP := func(ipStr string) string { + if ip := net.ParseIP(ipStr); ip != nil && ip.To4() == nil { + return "::1" + } + return "127.0.0.1" + } + + // On firewalla, we don't need to check localhost, because the lo interface is excluded in dnsmasq + // config, so we can always listen on localhost port 53, but no traffic could be routed there. + tryLocalhost := !isLoopback(listener.IP) && router.Name() != firewalla.Name + tryPort5354 := true + attempts := 0 + maxAttempts := 10 + for { + if attempts == maxAttempts { + mainLog.Fatal().Msg("could not find available listen ip and port") + } + addr := net.JoinHostPort(listener.IP, strconv.Itoa(listener.Port)) + if listenOk(addr) { + break + } + if tryLocalhost { + tryLocalhost = false + listener.IP = localhostIP(listener.IP) + listener.Port = 53 + mainLog.Warn().Msgf("could not listen on address: %s, trying localhost: %s", addr, net.JoinHostPort(listener.IP, strconv.Itoa(listener.Port))) + continue + } + if tryPort5354 { + tryPort5354 = false + listener.IP = oldIP + listener.Port = 5354 + mainLog.Warn().Msgf("could not listen on address: %s, trying port 5354", addr) + continue + } + listener.IP = randomLocalIP() + listener.Port = randomPort() + mainLog.Warn().Msgf("could not listen on address: %s, pick a random ip+port", addr) + attempts++ + } + } + + // Specific case for systemd-resolved. + if useSystemdResolved { + if listener := cfg.FirstListener(); listener != nil && listener.Port == 53 { + // systemd-resolved does not allow forwarding DNS queries from 127.0.0.53 to loopback + // ip address, other than "127.0.0.1", so trying to listen on default route interface + // address instead. + if ip := net.ParseIP(listener.IP); ip != nil && ip.IsLoopback() && ip.String() != "127.0.0.1" { + mainLog.Warn().Msg("using loopback interface do not work with systemd-resolved") + found := false + if netIface, _ := net.InterfaceByName(defaultIfaceName()); netIface != nil { + addrs, _ := netIface.Addrs() + for _, addr := range addrs { + if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil { + addr := net.JoinHostPort(netIP.IP.String(), strconv.Itoa(listener.Port)) + if listenOk(addr) { + found = true + listener.IP = netIP.IP.String() + mainLog.Warn().Msgf("use %s as listener address", listener.IP) + break + } + } + } + } + if !found { + mainLog.Fatal().Msgf("could not use %q as DNS nameserver with systemd resolved", listener.IP) + } + } + } + } +} diff --git a/cmd/ctrld/cli_router.go b/cmd/ctrld/cli_router.go deleted file mode 100644 index 7688e9a..0000000 --- a/cmd/ctrld/cli_router.go +++ /dev/null @@ -1,100 +0,0 @@ -//go:build linux || freebsd - -package main - -import ( - "os" - "os/exec" - "strings" - - "github.com/spf13/cobra" - - "github.com/Control-D-Inc/ctrld/internal/router" -) - -func initRouterCLI() { - validArgs := append(router.SupportedPlatforms(), "auto") - var b strings.Builder - b.WriteString("Auto-setup Control D on a router.\n\nSupported platforms:\n\n") - for _, arg := range validArgs { - b.WriteString(" ₒ ") - b.WriteString(arg) - if arg == "auto" { - b.WriteString(" - detect the platform you are running on") - } - b.WriteString("\n") - } - - routerCmd := &cobra.Command{ - Use: "setup", - Short: b.String(), - PreRun: func(cmd *cobra.Command, args []string) { - initConsoleLogging() - }, - Run: func(cmd *cobra.Command, args []string) { - if len(args) == 0 { - _ = cmd.Help() - return - } - if len(args) != 1 { - _ = cmd.Help() - return - } - platform := args[0] - if platform == "auto" { - platform = router.Name() - } - if !router.IsSupported(platform) { - unsupportedPlatformHelp(cmd) - os.Exit(1) - } - - exe, err := os.Executable() - if err != nil { - mainLog.Fatal().Msgf("could not find executable path: %v", err) - os.Exit(1) - } - - cmdArgs := []string{"start"} - cmdArgs = append(cmdArgs, osArgs(platform)...) - cmdArgs = append(cmdArgs, "--router") - command := exec.Command(exe, cmdArgs...) - command.Stdout = os.Stdout - command.Stderr = os.Stderr - command.Stdin = os.Stdin - if err := command.Run(); err != nil { - mainLog.Fatal().Msg(err.Error()) - } - }, - } - // Keep these flags in sync with startCmd, except for "--router". - routerCmd.Flags().StringVarP(&configPath, "config", "c", "", "Path to config file") - routerCmd.Flags().StringVarP(&configBase64, "base64_config", "", "", "Base64 encoded config") - routerCmd.Flags().StringVarP(&listenAddress, "listen", "", "", "Listener address and port, in format: address:port") - routerCmd.Flags().StringVarP(&primaryUpstream, "primary_upstream", "", "", "Primary upstream endpoint") - routerCmd.Flags().StringVarP(&secondaryUpstream, "secondary_upstream", "", "", "Secondary upstream endpoint") - routerCmd.Flags().StringSliceVarP(&domains, "domains", "", nil, "List of domain to apply in a split DNS policy") - routerCmd.Flags().StringVarP(&logPath, "log", "", "", "Path to log file") - routerCmd.Flags().IntVarP(&cacheSize, "cache_size", "", 0, "Enable cache with size items") - routerCmd.Flags().StringVarP(&cdUID, "cd", "", "", "Control D resolver uid") - routerCmd.Flags().BoolVarP(&cdDev, "dev", "", false, "Use Control D dev resolver/domain") - _ = routerCmd.Flags().MarkHidden("dev") - routerCmd.Flags().StringVarP(&iface, "iface", "", "", `Update DNS setting for iface, "auto" means the default interface gateway`) - - tmpl := routerCmd.UsageTemplate() - tmpl = strings.Replace(tmpl, "{{.UseLine}}", "{{.UseLine}} [platform]", 1) - routerCmd.SetUsageTemplate(tmpl) - rootCmd.AddCommand(routerCmd) -} - -func osArgs(platform string) []string { - args := os.Args[2:] - n := 0 - for _, x := range args { - if x != platform && x != "auto" { - args[n] = x - n++ - } - } - return args[:n] -} diff --git a/cmd/ctrld/cli_router_others.go b/cmd/ctrld/cli_router_others.go deleted file mode 100644 index 4934b5c..0000000 --- a/cmd/ctrld/cli_router_others.go +++ /dev/null @@ -1,5 +0,0 @@ -//go:build !linux && !freebsd - -package main - -func initRouterCLI() {} diff --git a/cmd/ctrld/conn.go b/cmd/ctrld/conn.go new file mode 100644 index 0000000..a627935 --- /dev/null +++ b/cmd/ctrld/conn.go @@ -0,0 +1,51 @@ +package main + +import ( + "net" + "time" +) + +// logConn wraps a net.Conn, override the Write behavior. +// runCmd uses this wrapper, so as long as startCmd finished, +// ctrld log won't be flushed with un-necessary write errors. +type logConn struct { + conn net.Conn +} + +func (lc *logConn) Read(b []byte) (n int, err error) { + return lc.conn.Read(b) +} + +func (lc *logConn) Close() error { + return lc.conn.Close() +} + +func (lc *logConn) LocalAddr() net.Addr { + return lc.conn.LocalAddr() +} + +func (lc *logConn) RemoteAddr() net.Addr { + return lc.conn.RemoteAddr() +} + +func (lc *logConn) SetDeadline(t time.Time) error { + return lc.conn.SetDeadline(t) +} + +func (lc *logConn) SetReadDeadline(t time.Time) error { + return lc.conn.SetReadDeadline(t) +} + +func (lc *logConn) SetWriteDeadline(t time.Time) error { + return lc.conn.SetWriteDeadline(t) +} + +func (lc *logConn) Write(b []byte) (int, error) { + // Write performs writes with underlying net.Conn, ignore any errors happen. + // "ctrld run" command use this wrapper to report errors to "ctrld start". + // If no error occurred, "ctrld start" may finish before "ctrld run" attempt + // to close the connection, so ignore errors conservatively here, prevent + // un-necessary error "write to closed connection" flushed to ctrld log. + _, _ = lc.conn.Write(b) + return len(b), nil +} diff --git a/cmd/ctrld/dns_proxy.go b/cmd/ctrld/dns_proxy.go index fad8e9c..52a5fb1 100644 --- a/cmd/ctrld/dns_proxy.go +++ b/cmd/ctrld/dns_proxy.go @@ -93,7 +93,7 @@ func (p *prog) serveDNS(listenerNum string) error { }) } g.Go(func() error { - s, errCh := runDNSServer(dnsListenAddress(listenerNum, listenerConfig), proto, handler) + s, errCh := runDNSServer(dnsListenAddress(listenerConfig), proto, handler) defer s.Shutdown() if listenerConfig.Port == 0 { switch s.Net { @@ -400,12 +400,13 @@ func needLocalIPv6Listener() bool { return ctrldnet.SupportsIPv6ListenLocal() && runtime.GOOS == "windows" } -func dnsListenAddress(lcNum string, lc *ctrld.ListenerConfig) string { - addr := net.JoinHostPort(lc.IP, strconv.Itoa(lc.Port)) - // If we are inside container and the listener address is localhost, - // Change it to 0.0.0.0:53, so user can expose the port to outside. - if addr == "127.0.0.1:53" && cdUID != "" && inContainer() { - return "0.0.0.0:53" +func dnsListenAddress(lc *ctrld.ListenerConfig) string { + // If we are inside container and the listener loopback address, change + // the address to something like 0.0.0.0:53, so user can expose the port to outside. + if inContainer() { + if ip := net.ParseIP(lc.IP); ip != nil && ip.IsLoopback() { + return net.JoinHostPort("0.0.0.0", strconv.Itoa(lc.Port)) + } } return net.JoinHostPort(lc.IP, strconv.Itoa(lc.Port)) } diff --git a/cmd/ctrld/main.go b/cmd/ctrld/main.go index 9f6ec60..2573f6e 100644 --- a/cmd/ctrld/main.go +++ b/cmd/ctrld/main.go @@ -30,7 +30,6 @@ var ( cdDev bool iface string ifaceStartStop string - setupRouter bool mainLog = zerolog.New(io.Discard) consoleWriter zerolog.ConsoleWriter @@ -39,7 +38,6 @@ var ( func main() { ctrld.InitConfig(v, "ctrld") initCLI() - initRouterCLI() if err := rootCmd.Execute(); err != nil { mainLog.Error().Msg(err.Error()) os.Exit(1) diff --git a/cmd/ctrld/prog.go b/cmd/ctrld/prog.go index e53436f..e94e0e9 100644 --- a/cmd/ctrld/prog.go +++ b/cmd/ctrld/prog.go @@ -16,10 +16,7 @@ import ( "github.com/Control-D-Inc/ctrld/internal/clientinfo" "github.com/Control-D-Inc/ctrld/internal/dnscache" "github.com/Control-D-Inc/ctrld/internal/router" - "github.com/Control-D-Inc/ctrld/internal/router/ddwrt" "github.com/Control-D-Inc/ctrld/internal/router/firewalla" - "github.com/Control-D-Inc/ctrld/internal/router/openwrt" - "github.com/Control-D-Inc/ctrld/internal/router/ubios" ) const ( @@ -221,16 +218,7 @@ func (p *prog) deAllocateIP() error { } func (p *prog) setDNS() { - switch router.Name() { - case ddwrt.Name, openwrt.Name, ubios.Name: - // On router, ctrld run as a DNS forwarder, it does not have to change system DNS. - // Except for: - // + EdgeOS, which /etc/resolv.conf could be managed by vyatta_update_resolv.pl script. - // + Merlin/Tomato, which has WAN DNS setup on boot for NTP. - // + Synology, which /etc/resolv.conf is not configured to point to localhost. - return - } - if cfg.Listener == nil || cfg.Listener["0"] == nil { + if cfg.Listener == nil { return } if iface == "" { @@ -239,6 +227,10 @@ func (p *prog) setDNS() { if iface == "auto" { iface = defaultIfaceName() } + lc := cfg.FirstListener() + if lc == nil { + return + } logger := mainLog.With().Str("iface", iface).Logger() netIface, err := netInterface(iface) if err != nil { @@ -250,23 +242,29 @@ func (p *prog) setDNS() { return } logger.Debug().Msg("setting DNS for interface") - ns := cfg.Listener["0"].IP - if router.Name() == firewalla.Name && (ns == "127.0.0.1" || ns == "0.0.0.0" || ns == "") { + ns := lc.IP + ifaceName := defaultIfaceName() + isFirewalla := router.Name() == firewalla.Name + if isFirewalla { // On Firewalla, the lo interface is excluded in all dnsmasq settings of all interfaces. // Thus, we use "br0" as the nameserver in /etc/resolv.conf file. - if ns == "127.0.0.1" { - logger.Warn().Msg("127.0.0.1 as DNS server won't work on Firewalla") - } else { - logger.Warn().Msgf("%q could not be used as DNS server", ns) + ifaceName = "br0" + logger.Warn().Msg("using br0 interface IP address as DNS server") + } + if couldBeDirectListener(lc) { + // If ctrld is direct listener, use 127.0.0.1 as nameserver. + ns = "127.0.0.1" + } else if lc.Port != 53 { + logger.Warn().Msg("ctrld is not running on port 53, use default route interface as DNS server") + netIface, err := net.InterfaceByName(ifaceName) + if err != nil { + mainLog.Fatal().Err(err).Msg("failed to get default route interface") } - if netIface, err := net.InterfaceByName("br0"); err == nil { - addrs, _ := netIface.Addrs() - for _, addr := range addrs { - if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil { - logger.Warn().Msg("using br0 interface IP address as DNS server") - ns = netIP.IP.To4().String() - break - } + addrs, _ := netIface.Addrs() + for _, addr := range addrs { + if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil { + ns = netIP.IP.To4().String() + break } } } @@ -278,11 +276,6 @@ func (p *prog) setDNS() { } func (p *prog) resetDNS() { - switch router.Name() { - case ddwrt.Name, openwrt.Name, ubios.Name: - // See comment in p.setDNS method. - return - } if iface == "" { return } @@ -312,6 +305,13 @@ func randomLocalIP() string { return fmt.Sprintf("127.0.0.%d", n) } +func randomPort() int { + max := 1<<16 - 1 + min := 1025 + n := rand.Intn(max-min) + min + return n +} + // runLogServer starts a unix listener, use by startCmd to gather log from runCmd. func runLogServer(sockPath string) net.Conn { addr, err := net.ResolveUnixAddr("unix", sockPath) diff --git a/config.go b/config.go index 9c553b2..afc1e49 100644 --- a/config.go +++ b/config.go @@ -12,6 +12,8 @@ import ( "net/url" "os" "runtime" + "sort" + "strconv" "strings" "sync" "sync/atomic" @@ -123,6 +125,25 @@ func (c *Config) HasUpstreamSendClientInfo() bool { return false } +// FirstListener returns the first listener config of current config. Listeners are sorted numerically. +// +// It panics if Config has no listeners configured. +func (c *Config) FirstListener() *ListenerConfig { + listeners := make([]int, 0, len(c.Listener)) + for k := range c.Listener { + n, err := strconv.Atoi(k) + if err != nil { + continue + } + listeners = append(listeners, n) + } + if len(listeners) == 0 { + panic("missing listener config") + } + sort.Ints(listeners) + return c.Listener[strconv.Itoa(listeners[0])] +} + // ServiceConfig specifies the general ctrld config. type ServiceConfig struct { LogLevel string `mapstructure:"log_level" toml:"log_level,omitempty"` diff --git a/config_test.go b/config_test.go index 27f9d40..3591327 100644 --- a/config_test.go +++ b/config_test.go @@ -1,6 +1,7 @@ package ctrld_test import ( + "strings" "testing" "github.com/go-playground/validator/v10" @@ -56,6 +57,20 @@ func TestLoadDefaultConfig(t *testing.T) { assert.Len(t, cfg.Upstream, 2) } +func TestConfigOverride(t *testing.T) { + v := viper.NewWithOptions(viper.KeyDelimiter("::")) + ctrld.InitConfig(v, "test_load_config") + v.SetConfigType("toml") + require.NoError(t, v.ReadConfig(strings.NewReader(testhelper.SampleConfigStr(t)))) + cfg := ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{ + "0": {IP: "127.0.0.1", Port: 53}, + }} + require.NoError(t, v.Unmarshal(&cfg)) + + assert.Equal(t, "10.10.42.69", cfg.Listener["1"].IP) + assert.Equal(t, 1337, cfg.Listener["1"].Port) +} + func TestConfigValidation(t *testing.T) { tests := []struct { name string diff --git a/internal/dns/nm.go b/internal/dns/nm.go index a8ea923..b8bc0c7 100644 --- a/internal/dns/nm.go +++ b/internal/dns/nm.go @@ -267,5 +267,5 @@ func (m *nmManager) Close() error { } func (m *nmManager) Mode() string { - return "network-maanger" + return "network-manager" } diff --git a/internal/router/ddwrt/ddwrt.go b/internal/router/ddwrt/ddwrt.go index cf45b30..f2cffdc 100644 --- a/internal/router/ddwrt/ddwrt.go +++ b/internal/router/ddwrt/ddwrt.go @@ -57,12 +57,6 @@ func (d *Ddwrt) PreRun() error { return ntp.Wait() } -func (d *Ddwrt) Configure() error { - d.cfg.Listener["0"].IP = "127.0.0.1" - d.cfg.Listener["0"].Port = 5354 - return nil -} - func (d *Ddwrt) Setup() error { // Already setup. if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val == "1" { diff --git a/internal/router/dnsmasq/dnsmasq.go b/internal/router/dnsmasq/dnsmasq.go index 6089c43..0b051aa 100644 --- a/internal/router/dnsmasq/dnsmasq.go +++ b/internal/router/dnsmasq/dnsmasq.go @@ -1,6 +1,7 @@ package dnsmasq import ( + "errors" "html/template" "net" "path/filepath" @@ -60,15 +61,20 @@ type Upstream struct { } func ConfTmpl(tmplText string, cfg *ctrld.Config) (string, error) { - upstreams := make([]Upstream, 0, len(cfg.Listener)) - for _, listener := range cfg.Listener { - upstreams = append(upstreams, Upstream{Ip: listener.IP, Port: listener.Port}) + listener := cfg.FirstListener() + if listener == nil { + return "", errors.New("missing listener") } + ip := listener.IP + if ip == "0.0.0.0" || ip == "::" || ip == "" { + ip = "127.0.0.1" + } + upstreams := []Upstream{{Ip: ip, Port: listener.Port}} return confTmpl(tmplText, upstreams, cfg.HasUpstreamSendClientInfo()) } func FirewallaConfTmpl(tmplText string, cfg *ctrld.Config) (string, error) { - if lc := cfg.Listener["0"]; lc != nil && lc.IP == "0.0.0.0" { + if lc := cfg.FirstListener(); lc != nil && (lc.IP == "0.0.0.0" || lc.IP == "") { return confTmpl(tmplText, firewallaUpstreams(lc.Port), cfg.HasUpstreamSendClientInfo()) } return ConfTmpl(tmplText, cfg) diff --git a/internal/router/dummy.go b/internal/router/dummy.go index 71d7a82..dea54e0 100644 --- a/internal/router/dummy.go +++ b/internal/router/dummy.go @@ -4,10 +4,6 @@ import "github.com/kardianos/service" type dummy struct{} -func NewDummyRouter() Router { - return &dummy{} -} - func (d *dummy) ConfigureService(_ *service.Config) error { return nil } diff --git a/internal/router/edgeos/edgeos.go b/internal/router/edgeos/edgeos.go index c6b1e3c..f84e3b1 100644 --- a/internal/router/edgeos/edgeos.go +++ b/internal/router/edgeos/edgeos.go @@ -61,12 +61,6 @@ func (e *EdgeOS) PreRun() error { return nil } -func (e *EdgeOS) Configure() error { - e.cfg.Listener["0"].IP = "127.0.0.1" - e.cfg.Listener["0"].Port = 5354 - return nil -} - func (e *EdgeOS) Setup() error { if e.isUSG { return e.setupUSG() diff --git a/internal/router/firewalla/firewalla.go b/internal/router/firewalla/firewalla.go index fd4635f..4e177ed 100644 --- a/internal/router/firewalla/firewalla.go +++ b/internal/router/firewalla/firewalla.go @@ -53,12 +53,6 @@ func (f *Firewalla) PreRun() error { return nil } -func (f *Firewalla) Configure() error { - f.cfg.Listener["0"].IP = "0.0.0.0" - f.cfg.Listener["0"].Port = 5354 - return nil -} - func (f *Firewalla) Setup() error { data, err := dnsmasq.FirewallaConfTmpl(dnsmasq.ConfigContentTmpl, f.cfg) if err != nil { diff --git a/internal/router/merlin/merlin.go b/internal/router/merlin/merlin.go index 9e84298..18b07c5 100644 --- a/internal/router/merlin/merlin.go +++ b/internal/router/merlin/merlin.go @@ -48,12 +48,6 @@ func (m *Merlin) PreRun() error { return ntp.Wait() } -func (m *Merlin) Configure() error { - m.cfg.Listener["0"].IP = "127.0.0.1" - m.cfg.Listener["0"].Port = 5354 - return nil -} - func (m *Merlin) Setup() error { buf, err := os.ReadFile(dnsmasq.MerlinPostConfPath) // Already setup. diff --git a/internal/router/openwrt/openwrt.go b/internal/router/openwrt/openwrt.go index bd08b12..1c8860d 100644 --- a/internal/router/openwrt/openwrt.go +++ b/internal/router/openwrt/openwrt.go @@ -48,12 +48,6 @@ func (o *Openwrt) PreRun() error { return nil } -func (o *Openwrt) Configure() error { - o.cfg.Listener["0"].IP = "127.0.0.1" - o.cfg.Listener["0"].Port = 5354 - return nil -} - func (o *Openwrt) Setup() error { // Delete dnsmasq port if set. if _, err := uci("delete", "dhcp.@dnsmasq[0].port"); err != nil && !errors.Is(err, errUCIEntryNotFound) { diff --git a/internal/router/pfsense/pfsense.go b/internal/router/pfsense/pfsense.go index d724c23..ec2b890 100644 --- a/internal/router/pfsense/pfsense.go +++ b/internal/router/pfsense/pfsense.go @@ -54,6 +54,14 @@ func (p *Pfsense) ConfigureService(svc *service.Config) error { } func (p *Pfsense) Install(config *service.Config) error { + // pfsense need ".sh" extension for script to be run at boot. + // See: https://docs.netgate.com/pfsense/en/latest/development/boot-commands.html#shell-script-option + oldname := filepath.Join(rcPath, p.svcName) + newname := filepath.Join(rcPath, p.svcName+".sh") + _ = os.Remove(newname) + if err := os.Symlink(oldname, newname); err != nil { + return fmt.Errorf("os.Symlink: %w", err) + } return nil } @@ -62,16 +70,7 @@ func (p *Pfsense) Uninstall(config *service.Config) error { } func (p *Pfsense) PreRun() error { - return nil -} - -func (p *Pfsense) Configure() error { - p.cfg.Listener["0"].IP = "127.0.0.1" - p.cfg.Listener["0"].Port = 53 - return nil -} - -func (p *Pfsense) Setup() error { + // TODO: remove this hacky solution. // If Pfsense is in DNS Resolver mode, ensure no unbound processes running. _ = exec.Command("killall", "unbound").Run() @@ -80,6 +79,10 @@ func (p *Pfsense) Setup() error { return nil } +func (p *Pfsense) Setup() error { + return nil +} + func (p *Pfsense) Cleanup() error { if err := os.Remove(filepath.Join(rcPath, p.svcName+".sh")); err != nil { return fmt.Errorf("os.Remove: %w", err) diff --git a/internal/router/router.go b/internal/router/router.go index 257e3b4..839151e 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -27,15 +27,9 @@ type Service interface { Uninstall(*service.Config) error } -// Config is the interface to manage ctrld config on router. -type Config interface { - Configure() error -} - // Router is the interface for managing ctrld running on router. type Router interface { Service - Config PreRun() error Setup() error @@ -64,7 +58,7 @@ func New(cfg *ctrld.Config) Router { case firewalla.Name: return firewalla.New(cfg) } - return NewDummyRouter() + return &dummy{} } // IsGLiNet reports whether the router is an GL.iNet router. @@ -94,38 +88,6 @@ type router struct { sendClientInfo bool } -// IsSupported reports whether the given platform is supported by ctrld. -func IsSupported(platform string) bool { - switch platform { - case ddwrt.Name, - edgeos.Name, - firewalla.Name, - merlin.Name, - openwrt.Name, - pfsense.Name, - synology.Name, - tomato.Name, - ubios.Name: - return true - } - return false -} - -// SupportedPlatforms return all platforms that can be configured to run with ctrld. -func SupportedPlatforms() []string { - return []string{ - ddwrt.Name, - edgeos.Name, - firewalla.Name, - merlin.Name, - openwrt.Name, - pfsense.Name, - synology.Name, - tomato.Name, - ubios.Name, - } -} - // Name returns name of the router platform. func Name() string { if r := routerPlatform.Load(); r != nil { diff --git a/internal/router/synology/synology.go b/internal/router/synology/synology.go index e1d51bd..78551e4 100644 --- a/internal/router/synology/synology.go +++ b/internal/router/synology/synology.go @@ -43,12 +43,6 @@ func (s *Synology) PreRun() error { return nil } -func (s *Synology) Configure() error { - s.cfg.Listener["0"].IP = "127.0.0.1" - s.cfg.Listener["0"].Port = 5354 - return nil -} - func (s *Synology) Setup() error { data, err := dnsmasq.ConfTmpl(dnsmasq.ConfigContentTmpl, s.cfg) if err != nil { diff --git a/internal/router/tomato/tomato.go b/internal/router/tomato/tomato.go index 937f8ba..4c1824d 100644 --- a/internal/router/tomato/tomato.go +++ b/internal/router/tomato/tomato.go @@ -52,12 +52,6 @@ func (f *FreshTomato) PreRun() error { return ntp.Wait() } -func (f *FreshTomato) Configure() error { - f.cfg.Listener["0"].IP = "127.0.0.1" - f.cfg.Listener["0"].Port = 5354 - return nil -} - func (f *FreshTomato) Setup() error { // Already setup. if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val == "1" { diff --git a/internal/router/ubios/ubios.go b/internal/router/ubios/ubios.go index 61ab8e0..06194d9 100644 --- a/internal/router/ubios/ubios.go +++ b/internal/router/ubios/ubios.go @@ -46,12 +46,6 @@ func (u *Ubios) PreRun() error { return nil } -func (u *Ubios) Configure() error { - u.cfg.Listener["0"].IP = "127.0.0.1" - u.cfg.Listener["0"].Port = 5354 - return nil -} - func (u *Ubios) Setup() error { data, err := dnsmasq.ConfTmpl(dnsmasq.ConfigContentTmpl, u.cfg) if err != nil { diff --git a/testhelper/config.go b/testhelper/config.go index 0b739f0..5c2e5f4 100644 --- a/testhelper/config.go +++ b/testhelper/config.go @@ -19,6 +19,10 @@ func SampleConfig(t *testing.T) *ctrld.Config { return &cfg } +func SampleConfigStr(t *testing.T) string { + return sampleConfigContent +} + var sampleConfigContent = ` [service] log_level = "info"