From 4d996e317b568b352d3496786b0fe9f3bcb56a5c Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 20 Dec 2023 23:08:11 +0700 Subject: [PATCH 01/16] Fix wrong toml struct tag for arp discovery --- config.go | 2 +- config_test.go | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/config.go b/config.go index 5baa10d..0b5d39c 100644 --- a/config.go +++ b/config.go @@ -189,7 +189,7 @@ type ServiceConfig struct { DHCPLeaseFile string `mapstructure:"dhcp_lease_file_path" toml:"dhcp_lease_file_path" validate:"omitempty,file"` DHCPLeaseFileFormat string `mapstructure:"dhcp_lease_file_format" toml:"dhcp_lease_file_format" validate:"required_unless=DHCPLeaseFile '',omitempty,oneof=dnsmasq isc-dhcp"` DiscoverMDNS *bool `mapstructure:"discover_mdns" toml:"discover_mdns,omitempty"` - DiscoverARP *bool `mapstructure:"discover_arp" toml:"discover_dhcp,omitempty"` + DiscoverARP *bool `mapstructure:"discover_arp" toml:"discover_arp,omitempty"` DiscoverDHCP *bool `mapstructure:"discover_dhcp" toml:"discover_dhcp,omitempty"` DiscoverPtr *bool `mapstructure:"discover_ptr" toml:"discover_ptr,omitempty"` DiscoverHosts *bool `mapstructure:"discover_hosts" toml:"discover_hosts,omitempty"` diff --git a/config_test.go b/config_test.go index ff20bc2..d66556f 100644 --- a/config_test.go +++ b/config_test.go @@ -121,6 +121,29 @@ func TestConfigValidation(t *testing.T) { } } +func TestConfigDiscoverOverride(t *testing.T) { + v := viper.NewWithOptions(viper.KeyDelimiter("::")) + ctrld.InitConfig(v, "test_config_discover_override") + v.SetConfigType("toml") + configStr := ` +[service] +discover_arp = false +discover_dhcp = false +discover_hosts = false +discover_mdns = false +discover_ptr = false +` + require.NoError(t, v.ReadConfig(strings.NewReader(configStr))) + cfg := ctrld.Config{} + require.NoError(t, v.Unmarshal(&cfg)) + + require.False(t, *cfg.Service.DiscoverARP) + require.False(t, *cfg.Service.DiscoverDHCP) + require.False(t, *cfg.Service.DiscoverHosts) + require.False(t, *cfg.Service.DiscoverMDNS) + require.False(t, *cfg.Service.DiscoverPtr) +} + func defaultConfig(t *testing.T) *ctrld.Config { v := viper.New() ctrld.InitConfig(v, "test_load_default_config") From cb445825f4ffd07fe2038747acea86e02f7bd682 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 4 Jan 2024 00:23:41 +0700 Subject: [PATCH 02/16] internal/clientinfo: add NDP discovery --- internal/clientinfo/client_info.go | 29 +++++-- internal/clientinfo/ndp.go | 126 +++++++++++++++++++++++++++++ internal/clientinfo/ndp_linux.go | 24 ++++++ internal/clientinfo/ndp_others.go | 31 +++++++ internal/clientinfo/ndp_test.go | 64 +++++++++++++++ 5 files changed, 266 insertions(+), 8 deletions(-) create mode 100644 internal/clientinfo/ndp.go create mode 100644 internal/clientinfo/ndp_linux.go create mode 100644 internal/clientinfo/ndp_others.go create mode 100644 internal/clientinfo/ndp_test.go diff --git a/internal/clientinfo/client_info.go b/internal/clientinfo/client_info.go index 07e4cf0..f51cf88 100644 --- a/internal/clientinfo/client_info.go +++ b/internal/clientinfo/client_info.go @@ -74,6 +74,7 @@ type Table struct { dhcp *dhcp merlin *merlinDiscover arp *arpDiscover + ndp *ndpDiscover ptr *ptrDiscover mdns *mdns hf *hostsFile @@ -172,16 +173,28 @@ func (t *Table) init() { } go t.dhcp.watchChanges() } - // ARP table. + // ARP/NDP table. if t.discoverARP() { t.arp = &arpDiscover{} + t.ndp = &ndpDiscover{} ctrld.ProxyLogger.Load().Debug().Msg("start arp discovery") - if err := t.arp.refresh(); err != nil { - ctrld.ProxyLogger.Load().Error().Err(err).Msg("could not init ARP discover") - } else { - t.ipResolvers = append(t.ipResolvers, t.arp) - t.macResolvers = append(t.macResolvers, t.arp) - t.refreshers = append(t.refreshers, t.arp) + discovers := map[string]interface { + refresher + IpResolver + MacResolver + }{ + "ARP": t.arp, + "NDP": t.ndp, + } + + for protocol, discover := range discovers { + if err := discover.refresh(); err != nil { + ctrld.ProxyLogger.Load().Error().Err(err).Msgf("could not init %s discover", protocol) + } else { + t.ipResolvers = append(t.ipResolvers, discover) + t.macResolvers = append(t.macResolvers, discover) + t.refreshers = append(t.refreshers, discover) + } } } // PTR lookup. @@ -328,7 +341,7 @@ func (t *Table) ListClients() []*Client { _ = r.refresh() } ipMap := make(map[string]*Client) - il := []ipLister{t.dhcp, t.arp, t.ptr, t.mdns, t.vni} + il := []ipLister{t.dhcp, t.arp, t.ndp, t.ptr, t.mdns, t.vni} for _, ir := range il { for _, ip := range ir.List() { c, ok := ipMap[ip] diff --git a/internal/clientinfo/ndp.go b/internal/clientinfo/ndp.go new file mode 100644 index 0000000..337c414 --- /dev/null +++ b/internal/clientinfo/ndp.go @@ -0,0 +1,126 @@ +package clientinfo + +import ( + "bufio" + "io" + "net" + "strings" + "sync" +) + +// ndpDiscover provides client discovery functionality using NDP protocol. +type ndpDiscover struct { + mac sync.Map // ip => mac + ip sync.Map // mac => ip +} + +// refresh re-scans the NDP table. +func (nd *ndpDiscover) refresh() error { + nd.scan() + return nil +} + +// LookupIP returns the ipv6 associated with the input MAC address. +func (nd *ndpDiscover) LookupIP(mac string) string { + val, ok := nd.ip.Load(mac) + if !ok { + return "" + } + return val.(string) +} + +// LookupMac returns the MAC address of the given IP address. +func (nd *ndpDiscover) LookupMac(ip string) string { + val, ok := nd.mac.Load(ip) + if !ok { + return "" + } + return val.(string) +} + +// String returns human-readable format of ndpDiscover. +func (nd *ndpDiscover) String() string { + return "ndp" +} + +// List returns all known IP addresses. +func (nd *ndpDiscover) List() []string { + if nd == nil { + return nil + } + var ips []string + nd.ip.Range(func(key, value any) bool { + ips = append(ips, value.(string)) + return true + }) + nd.mac.Range(func(key, value any) bool { + ips = append(ips, key.(string)) + return true + }) + return ips +} + +// scanWindows populates NDP table using information from "netsh" command. +func (nd *ndpDiscover) scanWindows(r io.Reader) { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 3 { + continue + } + if mac := parseMAC(fields[1]); mac != "" { + nd.mac.Store(fields[0], mac) + nd.ip.Store(mac, fields[0]) + } + } +} + +// scanUnix populates NDP table using information from "ndp" command. +func (nd *ndpDiscover) scanUnix(r io.Reader) { + scanner := bufio.NewScanner(r) + scanner.Scan() // skip header + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 2 { + continue + } + if mac := parseMAC(fields[1]); mac != "" { + ip := fields[0] + if idx := strings.IndexByte(ip, '%'); idx != -1 { + ip = ip[:idx] + } + nd.mac.Store(ip, mac) + nd.ip.Store(mac, ip) + } + } +} + +// normalizeMac ensure the given MAC address have the proper format +// before being parsed. +// +// Example, changing "00:0:00:0:00:01" to "00:00:00:00:00:01", which +// can be seen on Darwin. +func normalizeMac(mac string) string { + if len(mac) == 17 { + return mac + } + // Windows use "-" instead of ":" as separator. + mac = strings.ReplaceAll(mac, "-", ":") + parts := strings.Split(mac, ":") + if len(parts) != 6 { + return "" + } + for i, c := range parts { + if len(c) == 1 { + parts[i] = "0" + c + } + } + return strings.Join(parts, ":") +} + +// parseMAC parses the input MAC, doing normalization, +// and return the result after calling net.ParseMac function. +func parseMAC(mac string) string { + hw, _ := net.ParseMAC(normalizeMac(mac)) + return hw.String() +} diff --git a/internal/clientinfo/ndp_linux.go b/internal/clientinfo/ndp_linux.go new file mode 100644 index 0000000..713a7e3 --- /dev/null +++ b/internal/clientinfo/ndp_linux.go @@ -0,0 +1,24 @@ +package clientinfo + +import ( + "github.com/vishvananda/netlink" + + "github.com/Control-D-Inc/ctrld" +) + +// scan populates NDP table using information from system mappings. +func (nd *ndpDiscover) scan() { + neighs, err := netlink.NeighList(0, netlink.FAMILY_V6) + if err != nil { + ctrld.ProxyLogger.Load().Warn().Err(err).Msg("could not get neigh list") + return + } + + for _, n := range neighs { + ip := n.IP.String() + mac := n.HardwareAddr.String() + nd.mac.Store(ip, mac) + nd.ip.Store(mac, ip) + } + +} diff --git a/internal/clientinfo/ndp_others.go b/internal/clientinfo/ndp_others.go new file mode 100644 index 0000000..05ac322 --- /dev/null +++ b/internal/clientinfo/ndp_others.go @@ -0,0 +1,31 @@ +//go:build !linux + +package clientinfo + +import ( + "bytes" + "os/exec" + "runtime" + + "github.com/Control-D-Inc/ctrld" +) + +// scan populates NDP table using information from system mappings. +func (nd *ndpDiscover) scan() { + switch runtime.GOOS { + case "windows": + data, err := exec.Command("netsh", "interface", "ipv6", "show", "neighbors").Output() + if err != nil { + ctrld.ProxyLogger.Load().Warn().Err(err).Msg("could not query ndp table") + return + } + nd.scanWindows(bytes.NewReader(data)) + default: + data, err := exec.Command("ndp", "-an").Output() + if err != nil { + ctrld.ProxyLogger.Load().Warn().Err(err).Msg("could not query ndp table") + return + } + nd.scanUnix(bytes.NewReader(data)) + } +} diff --git a/internal/clientinfo/ndp_test.go b/internal/clientinfo/ndp_test.go new file mode 100644 index 0000000..c8cd398 --- /dev/null +++ b/internal/clientinfo/ndp_test.go @@ -0,0 +1,64 @@ +package clientinfo + +import ( + "strings" + "sync" + "testing" +) + +func Test_ndpDiscover_scanUnix(t *testing.T) { + r := strings.NewReader(`Neighbor Linklayer Address Netif Expire St Flgs Prbs +2405:4802:1f90:fda0:1459:ec89:523d:3583 00:0:00:0:00:01 en0 permanent R +2405:4802:1f90:fda0:186b:c54a:1370:c196 (incomplete) en0 expired N +2405:4802:1f90:fda0:88de:14ef:6a8c:579a 00:0:00:0:00:02 en0 permanent R +fe80::1%lo0 (incomplete) lo0 permanent R +`) + nd := &ndpDiscover{} + nd.scanUnix(r) + + for _, m := range []*sync.Map{&nd.mac, &nd.ip} { + count := 0 + m.Range(func(key, value any) bool { + count++ + return true + }) + if count != 2 { + t.Errorf("unexpected count, want 2, got: %d", count) + } + } +} + +func Test_ndpDiscover_scanWindows(t *testing.T) { + r := strings.NewReader(`Interface 14: Wi-Fi + + +Internet Address Physical Address Type +-------------------------------------------- ----------------- ----------- +2405:4802:1f90:fda0:ffff:ffff:ffff:ff88 00-00-00-00-00-00 Unreachable +fe80::1 60-57-47-21-dd-00 Reachable (Router) +fe80::6257:47ff:fe21:dd00 60-57-47-21-dd-00 Reachable (Router) +ff02::1 33-33-00-00-00-01 Permanent +ff02::2 33-33-00-00-00-02 Permanent +ff02::c 33-33-00-00-00-0c Permanent +`) + nd := &ndpDiscover{} + nd.scanWindows(r) + + count := 0 + nd.mac.Range(func(key, value any) bool { + count++ + return true + }) + if count != 6 { + t.Errorf("unexpected count, want 6, got: %d", count) + } + + count = 0 + nd.ip.Range(func(key, value any) bool { + count++ + return true + }) + if count != 5 { + t.Errorf("unexpected count, want 5, got: %d", count) + } +} From dc700bbd5291c2868a85799cc877d6c39ff8cf1f Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 3 Jan 2024 09:24:45 +0700 Subject: [PATCH 03/16] internal/router: use max-cache-ttl=0 on some routers On some routers, dnsmasq config may change cache-size dynamically after ctrld starts, causing dnsmasq crashes. Fixing this by using max-cache-ttl, which have the same effect with setting cache-size=0 but won't conflict with existing routers config. --- internal/router/dnsmasq/dnsmasq.go | 66 +++++++------------------- internal/router/edgeos/edgeos.go | 4 +- internal/router/firewalla/firewalla.go | 10 ---- internal/router/ubios/ubios.go | 10 +--- 4 files changed, 19 insertions(+), 71 deletions(-) diff --git a/internal/router/dnsmasq/dnsmasq.go b/internal/router/dnsmasq/dnsmasq.go index 9fab8b6..50c7d0e 100644 --- a/internal/router/dnsmasq/dnsmasq.go +++ b/internal/router/dnsmasq/dnsmasq.go @@ -1,12 +1,9 @@ package dnsmasq import ( - "bytes" "errors" - "fmt" "html/template" "net" - "os" "path/filepath" "strings" @@ -24,6 +21,8 @@ add-subnet=32,128 {{- end}} {{- if .CacheDisabled}} cache-size=0 +{{- else}} +max-cache-ttl=0 {{- end}} ` @@ -72,11 +71,18 @@ type Upstream struct { Port int } +// ConfTmpl generates dnsmasq configuration from ctrld config. func ConfTmpl(tmplText string, cfg *ctrld.Config) (string, error) { - return ConfTmplWitchCacheDisabled(tmplText, cfg, true) + return ConfTmplWithCacheDisabled(tmplText, cfg, true) } -func ConfTmplWitchCacheDisabled(tmplText string, cfg *ctrld.Config, cacheDisabled bool) (string, error) { +// ConfTmplWithCacheDisabled is like ConfTmpl, but the caller can control whether +// dnsmasq cache is disabled using cacheDisabled parameter. +// +// Generally, the caller should use ConfTmpl, but on some routers which dnsmasq config may be changed +// after ctrld started (like EdgeOS/Ubios, Firewalla ...), dnsmasq cache should not be disabled because +// the cache-size=0 generated by ctrld will conflict with router's generated config. +func ConfTmplWithCacheDisabled(tmplText string, cfg *ctrld.Config, cacheDisabled bool) (string, error) { listener := cfg.FirstListener() if listener == nil { return "", errors.New("missing listener") @@ -89,11 +95,14 @@ func ConfTmplWitchCacheDisabled(tmplText string, cfg *ctrld.Config, cacheDisable return confTmpl(tmplText, upstreams, cfg.HasUpstreamSendClientInfo(), cacheDisabled) } +// FirewallaConfTmpl generates dnsmasq config for Firewalla routers. func FirewallaConfTmpl(tmplText string, cfg *ctrld.Config) (string, error) { + // If ctrld listen on all interfaces, generating config for all of them. if lc := cfg.FirstListener(); lc != nil && (lc.IP == "0.0.0.0" || lc.IP == "") { - return confTmpl(tmplText, firewallaUpstreams(lc.Port), cfg.HasUpstreamSendClientInfo(), true) + return confTmpl(tmplText, firewallaUpstreams(lc.Port), cfg.HasUpstreamSendClientInfo(), false) } - return ConfTmpl(tmplText, cfg) + // Otherwise, generating config for the specific listener from ctrld's config. + return ConfTmplWithCacheDisabled(tmplText, cfg, false) } func confTmpl(tmplText string, upstreams []Upstream, sendClientInfo, cacheDisabled bool) (string, error) { @@ -136,20 +145,6 @@ func firewallaDnsmasqConfFiles() ([]string, error) { return filepath.Glob("/home/pi/firerouter/etc/dnsmasq.dns.*.conf") } -// firewallUpdateConf updates all firewall config files using given function. -func firewallUpdateConf(update func(conf string) error) error { - confFiles, err := firewallaDnsmasqConfFiles() - if err != nil { - return err - } - for _, conf := range confFiles { - if err := update(conf); err != nil { - return fmt.Errorf("%s: %w", conf, err) - } - } - return nil -} - // FirewallaSelfInterfaces returns list of interfaces that will be configured with default dnsmasq setup on Firewalla. func FirewallaSelfInterfaces() []*net.Interface { matches, err := firewallaDnsmasqConfFiles() @@ -166,32 +161,3 @@ func FirewallaSelfInterfaces() []*net.Interface { } return ifaces } - -// FirewallaDisableCache comments out "cache-size" line in all firewalla dnsmasq config files. -func FirewallaDisableCache() error { - return firewallUpdateConf(DisableCache) -} - -// FirewallaEnableCache un-comments out "cache-size" line in all firewalla dnsmasq config files. -func FirewallaEnableCache() error { - return firewallUpdateConf(EnableCache) -} - -// DisableCache comments out "cache-size" line in dnsmasq config file. -func DisableCache(conf string) error { - return replaceFileContent(conf, "\ncache-size=", "\n#cache-size=") -} - -// EnableCache un-comments "cache-size" line in dnsmasq config file. -func EnableCache(conf string) error { - return replaceFileContent(conf, "\n#cache-size=", "\ncache-size=") -} - -func replaceFileContent(filename, old, new string) error { - content, err := os.ReadFile(filename) - if err != nil { - return err - } - content = bytes.ReplaceAll(content, []byte(old), []byte(new)) - return os.WriteFile(filename, content, 0644) -} diff --git a/internal/router/edgeos/edgeos.go b/internal/router/edgeos/edgeos.go index 3e7003b..df7b57b 100644 --- a/internal/router/edgeos/edgeos.go +++ b/internal/router/edgeos/edgeos.go @@ -109,7 +109,7 @@ func (e *EdgeOS) setupUSG() error { sb.WriteString(line) } - data, err := dnsmasq.ConfTmplWitchCacheDisabled(dnsmasq.ConfigContentTmpl, e.cfg, false) + data, err := dnsmasq.ConfTmplWithCacheDisabled(dnsmasq.ConfigContentTmpl, e.cfg, false) if err != nil { return err } @@ -127,7 +127,7 @@ func (e *EdgeOS) setupUSG() error { } func (e *EdgeOS) setupUDM() error { - data, err := dnsmasq.ConfTmplWitchCacheDisabled(dnsmasq.ConfigContentTmpl, e.cfg, false) + data, err := dnsmasq.ConfTmplWithCacheDisabled(dnsmasq.ConfigContentTmpl, e.cfg, false) if err != nil { return err } diff --git a/internal/router/firewalla/firewalla.go b/internal/router/firewalla/firewalla.go index 66cd15e..cdf6586 100644 --- a/internal/router/firewalla/firewalla.go +++ b/internal/router/firewalla/firewalla.go @@ -65,11 +65,6 @@ func (f *Firewalla) Setup() error { return fmt.Errorf("writing ctrld config: %w", err) } - // Disable dnsmasq cache. - if err := dnsmasq.FirewallaDisableCache(); err != nil { - return err - } - // Restart dnsmasq service. if err := restartDNSMasq(); err != nil { return fmt.Errorf("restartDNSMasq: %w", err) @@ -87,11 +82,6 @@ func (f *Firewalla) Cleanup() error { return fmt.Errorf("removing ctrld config: %w", err) } - // Enable dnsmasq cache. - if err := dnsmasq.FirewallaEnableCache(); err != nil { - return err - } - // Restart dnsmasq service. if err := restartDNSMasq(); err != nil { return fmt.Errorf("restartDNSMasq: %w", err) diff --git a/internal/router/ubios/ubios.go b/internal/router/ubios/ubios.go index 32c7576..6513657 100644 --- a/internal/router/ubios/ubios.go +++ b/internal/router/ubios/ubios.go @@ -51,17 +51,13 @@ func (u *Ubios) Setup() error { if u.cfg.FirstListener().IsDirectDnsListener() { return nil } - data, err := dnsmasq.ConfTmpl(dnsmasq.ConfigContentTmpl, u.cfg) + data, err := dnsmasq.ConfTmplWithCacheDisabled(dnsmasq.ConfigContentTmpl, u.cfg, false) if err != nil { return err } if err := os.WriteFile(ubiosDNSMasqConfigPath, []byte(data), 0600); err != nil { return err } - // Disable dnsmasq cache. - if err := dnsmasq.DisableCache(ubiosDNSMasqDnsConfigPath); err != nil { - return err - } // Restart dnsmasq service. if err := restartDNSMasq(); err != nil { return err @@ -77,10 +73,6 @@ func (u *Ubios) Cleanup() error { if err := os.Remove(ubiosDNSMasqConfigPath); err != nil { return err } - // Enable dnsmasq cache. - if err := dnsmasq.EnableCache(ubiosDNSMasqDnsConfigPath); err != nil { - return err - } // Restart dnsmasq service. if err := restartDNSMasq(); err != nil { return err From 0a6d9d44548683f4c7cc844959b9ed43a8b77002 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 3 Jan 2024 17:56:04 +0700 Subject: [PATCH 04/16] internal/clientinfo: add Ubios custom device name --- internal/clientinfo/client_info.go | 25 +++++++--- internal/clientinfo/ubios.go | 78 ++++++++++++++++++++++++++++++ internal/clientinfo/ubios_test.go | 43 ++++++++++++++++ 3 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 internal/clientinfo/ubios.go create mode 100644 internal/clientinfo/ubios_test.go diff --git a/internal/clientinfo/client_info.go b/internal/clientinfo/client_info.go index f51cf88..72ef971 100644 --- a/internal/clientinfo/client_info.go +++ b/internal/clientinfo/client_info.go @@ -73,6 +73,7 @@ type Table struct { dhcp *dhcp merlin *merlinDiscover + ubios *ubiosDiscover arp *arpDiscover ndp *ndpDiscover ptr *ptrDiscover @@ -138,14 +139,26 @@ func (t *Table) init() { // Otherwise, process all possible sources in order, that means // the first result of IP/MAC/Hostname lookup will be used. // - // Merlin custom clients. + // Routers custom clients: + // - Merlin + // - Ubios if t.discoverDHCP() || t.discoverARP() { t.merlin = &merlinDiscover{} - if err := t.merlin.refresh(); err != nil { - ctrld.ProxyLogger.Load().Error().Err(err).Msg("could not init Merlin discover") - } else { - t.hostnameResolvers = append(t.hostnameResolvers, t.merlin) - t.refreshers = append(t.refreshers, t.merlin) + t.ubios = &ubiosDiscover{} + discovers := map[string]interface { + refresher + HostnameResolver + }{ + "Merlin": t.merlin, + "Ubios": t.ubios, + } + for platform, discover := range discovers { + if err := discover.refresh(); err != nil { + ctrld.ProxyLogger.Load().Error().Err(err).Msgf("could not init %s discover", platform) + } else { + t.hostnameResolvers = append(t.hostnameResolvers, discover) + t.refreshers = append(t.refreshers, discover) + } } } // Hosts file mapping. diff --git a/internal/clientinfo/ubios.go b/internal/clientinfo/ubios.go new file mode 100644 index 0000000..1a60de0 --- /dev/null +++ b/internal/clientinfo/ubios.go @@ -0,0 +1,78 @@ +package clientinfo + +import ( + "bytes" + "encoding/json" + "io" + "os/exec" + "strings" + "sync" + + "github.com/Control-D-Inc/ctrld/internal/router" + "github.com/Control-D-Inc/ctrld/internal/router/ubios" +) + +// ubiosDiscover provides client discovery functionality on Ubios routers. +type ubiosDiscover struct { + hostname sync.Map // mac => hostname +} + +// refresh reloads unifi devices from database. +func (u *ubiosDiscover) refresh() error { + if router.Name() != ubios.Name { + return nil + } + return u.refreshDevices() +} + +// LookupHostnameByIP returns hostname for given IP. +func (u *ubiosDiscover) LookupHostnameByIP(ip string) string { + return "" +} + +// LookupHostnameByMac returns unifi device custom hostname for the given MAC address. +func (u *ubiosDiscover) LookupHostnameByMac(mac string) string { + val, ok := u.hostname.Load(mac) + if !ok { + return "" + } + return val.(string) +} + +// refreshDevices updates unifi devices name from local mongodb. +func (u *ubiosDiscover) refreshDevices() error { + cmd := exec.Command("/usr/bin/mongo", "localhost:27117/ace", "--quiet", "--eval", ` + DBQuery.shellBatchSize = 256; + db.user.find({name: {$exists: true, $ne: ""}}, {_id:0, mac:1, name:1});`) + b, err := cmd.Output() + if err != nil { + return err + } + return u.storeDevices(bytes.NewReader(b)) +} + +// storeDevices saves unifi devices name for caching. +func (u *ubiosDiscover) storeDevices(r io.Reader) error { + decoder := json.NewDecoder(r) + device := struct { + MAC string + Name string + }{} + for { + err := decoder.Decode(&device) + if err == io.EOF { + break + } + if err != nil { + return err + } + mac := strings.ToLower(device.MAC) + u.hostname.Store(mac, normalizeHostname(device.Name)) + } + return nil +} + +// String returns human-readable format of ubiosDiscover. +func (u *ubiosDiscover) String() string { + return "ubios" +} diff --git a/internal/clientinfo/ubios_test.go b/internal/clientinfo/ubios_test.go new file mode 100644 index 0000000..657cf18 --- /dev/null +++ b/internal/clientinfo/ubios_test.go @@ -0,0 +1,43 @@ +package clientinfo + +import ( + "strings" + "testing" +) + +func Test_ubiosDiscover_storeDevices(t *testing.T) { + ud := &ubiosDiscover{} + r := strings.NewReader(`{ "mac": "00:00:00:00:00:01", "name": "device 1" } +{ "mac": "00:00:00:00:00:02", "name": "device 2" } +`) + if err := ud.storeDevices(r); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + mac string + hostname string + }{ + {"device 1", "00:00:00:00:00:01", "device 1"}, + {"device 2", "00:00:00:00:00:02", "device 2"}, + {"non-existed", "00:00:00:00:00:03", ""}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := ud.LookupHostnameByMac(tc.mac); got != tc.hostname { + t.Errorf("hostname mismatched, want: %q, got: %q", tc.hostname, got) + } + }) + } + + // Test for invalid input. + r = strings.NewReader(`{ "mac": "00:00:00:00:00:01", "name": "device 1"`) + if err := ud.storeDevices(r); err == nil { + t.Fatal("expected error, got nil") + } else { + t.Log(err) + } +} From 51b235b61ab50f91aa951b30c0bdc75fe72a94ba Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 4 Jan 2024 20:26:42 +0700 Subject: [PATCH 05/16] internal/clientinfo: implement ndp listen So when new clients join the network, ctrld can really the event and update client information to NDP table quickly. --- go.mod | 1 + go.sum | 2 + internal/clientinfo/client_info.go | 6 ++ internal/clientinfo/ndp.go | 93 ++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+) diff --git a/go.mod b/go.mod index fec32ef..4d38445 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/jaytaylor/go-hostsfile v0.0.0-20220426042432-61485ac1fa6c github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 github.com/kardianos/service v1.2.1 + github.com/mdlayher/ndp v1.0.1 github.com/miekg/dns v1.1.55 github.com/olekukonko/tablewriter v0.0.5 github.com/pelletier/go-toml/v2 v2.0.8 diff --git a/go.sum b/go.sum index c792103..a64357a 100644 --- a/go.sum +++ b/go.sum @@ -201,6 +201,8 @@ github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWV github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7 h1:lez6TS6aAau+8wXUP3G9I3TGlmPFEq2CTxBaRqY6AGE= github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y= +github.com/mdlayher/ndp v1.0.1 h1:+yAD79/BWyFlvAoeG5ncPS0ItlHP/eVbH7bQ6/+LVA4= +github.com/mdlayher/ndp v1.0.1/go.mod h1:rf3wKaWhAYJEXFKpgF8kQ2AxypxVbfNcZbqoAo6fVzk= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= github.com/mdlayher/raw v0.0.0-20190606142536-fef19f00fc18/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= diff --git a/internal/clientinfo/client_info.go b/internal/clientinfo/client_info.go index 72ef971..1fe1083 100644 --- a/internal/clientinfo/client_info.go +++ b/internal/clientinfo/client_info.go @@ -209,6 +209,12 @@ func (t *Table) init() { t.refreshers = append(t.refreshers, discover) } } + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-t.quitCh + cancel() + }() + go t.ndp.listen(ctx) } // PTR lookup. if t.discoverPTR() { diff --git a/internal/clientinfo/ndp.go b/internal/clientinfo/ndp.go index 337c414..600b54c 100644 --- a/internal/clientinfo/ndp.go +++ b/internal/clientinfo/ndp.go @@ -2,10 +2,19 @@ package clientinfo import ( "bufio" + "context" + "errors" + "fmt" "io" "net" + "net/netip" "strings" "sync" + "time" + + "github.com/mdlayher/ndp" + + "github.com/Control-D-Inc/ctrld" ) // ndpDiscover provides client discovery functionality using NDP protocol. @@ -60,6 +69,55 @@ func (nd *ndpDiscover) List() []string { return ips } +// listen listens on ipv6 link local for Neighbor Solicitation message +// to update new neighbors information to ndp table. +func (nd *ndpDiscover) listen(ctx context.Context) { + ifi, err := firstInterfaceWithV6LinkLocal() + if err != nil { + ctrld.ProxyLogger.Load().Debug().Err(err).Msg("failed to find valid ipv6") + return + } + c, ip, err := ndp.Listen(ifi, ndp.LinkLocal) + if err != nil { + ctrld.ProxyLogger.Load().Debug().Err(err).Msg("ndp listen failed") + return + } + defer c.Close() + ctrld.ProxyLogger.Load().Debug().Msgf("listening ndp on: %s", ip.String()) + for { + select { + case <-ctx.Done(): + return + default: + } + _ = c.SetReadDeadline(time.Now().Add(30 * time.Second)) + msg, _, from, readErr := c.ReadFrom() + if readErr != nil { + var opErr *net.OpError + if errors.As(readErr, &opErr) && (opErr.Timeout() || opErr.Temporary()) { + continue + } + ctrld.ProxyLogger.Load().Debug().Err(readErr).Msg("ndp read loop error") + return + } + + // Only looks for neighbor solicitation message, since new clients + // which join network will broadcast this message to us. + am, ok := msg.(*ndp.NeighborSolicitation) + if !ok { + continue + } + fromIP := from.String() + for _, opt := range am.Options { + if lla, ok := opt.(*ndp.LinkLayerAddress); ok { + mac := lla.Addr.String() + nd.mac.Store(fromIP, mac) + nd.ip.Store(mac, fromIP) + } + } + } +} + // scanWindows populates NDP table using information from "netsh" command. func (nd *ndpDiscover) scanWindows(r io.Reader) { scanner := bufio.NewScanner(r) @@ -124,3 +182,38 @@ func parseMAC(mac string) string { hw, _ := net.ParseMAC(normalizeMac(mac)) return hw.String() } + +// firstInterfaceWithV6LinkLocal returns the first interface which is capable of using NDP. +func firstInterfaceWithV6LinkLocal() (*net.Interface, error) { + ifis, err := net.Interfaces() + if err != nil { + return nil, err + } + + for _, ifi := range ifis { + // Skip if iface is down/loopback/non-multicast. + if ifi.Flags&net.FlagUp == 0 || ifi.Flags&net.FlagLoopback != 0 || ifi.Flags&net.FlagMulticast == 0 { + continue + } + + addrs, err := ifi.Addrs() + if err != nil { + return nil, err + } + + for _, addr := range addrs { + ipNet, ok := addr.(*net.IPNet) + if !ok { + continue + } + ip, ok := netip.AddrFromSlice(ipNet.IP) + if !ok { + return nil, fmt.Errorf("invalid ip address: %s", ipNet.String()) + } + if ip.Is6() && !ip.Is4In6() { + return &ifi, nil + } + } + } + return nil, errors.New("no interface can be used") +} From cfaf32f71aa1eee6327470a87f9f16742ff103ef Mon Sep 17 00:00:00 2001 From: Ginder Singh Date: Sat, 6 Jan 2024 00:42:41 -0500 Subject: [PATCH 06/16] Added upstream proto option to mobile library Changed android listener IP to 0.0.0.0 --- cmd/cli/cli.go | 24 ++++++++++++++++++------ cmd/cli/library.go | 9 +++++---- cmd/ctrld_library/main.go | 11 ++++++----- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 75249cd..7176356 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -753,6 +753,11 @@ func isMobile() bool { return runtime.GOOS == "android" || runtime.GOOS == "ios" } +// isAndroid reports whether the current OS is Android. +func isAndroid() bool { + return runtime.GOOS == "android" +} + // RunCobraCommand runs ctrld cli. func RunCobraCommand(cmd *cobra.Command) { noConfigStart = isNoConfigStart(cmd) @@ -771,7 +776,7 @@ func RunMobile(appConfig *AppConfig, appCallback *AppCallback, stopCh chan struc homedir = appConfig.HomeDir verbose = appConfig.Verbose cdUID = appConfig.CdUID - cdUpstreamProto = ctrld.ResolverTypeDOH + cdUpstreamProto = appConfig.UpstreamProto logPath = appConfig.LogPath run(appCallback, stopCh) } @@ -1618,10 +1623,18 @@ type listenerConfigCheck struct { // mobileListenerPort returns hardcoded port for mobile platforms. func mobileListenerPort() int { - if runtime.GOOS == "ios" { - return 53 + if isAndroid() { + return 5354 } - return 5354 + return 53 +} + +// mobileListenerIp returns hardcoded listener ip for mobile platforms +func mobileListenerIp() string { + if isAndroid() { + return "0.0.0.0" + } + return "127.0.0.1" } // updateListenerConfig updates the config for listeners if not defined, @@ -1670,9 +1683,8 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, fata delete(cfg.Listener, k) } } - // In cd mode, always use 127.0.0.1:5354. if cdMode { - firstLn.IP = "127.0.0.1" // Mobile platforms allows running listener only on loop back address. + firstLn.IP = mobileListenerIp() firstLn.Port = mobileListenerPort() // TODO: use clear(lcc) once upgrading to go 1.21 for k := range lcc { diff --git a/cmd/cli/library.go b/cmd/cli/library.go index 80612c9..d302644 100644 --- a/cmd/cli/library.go +++ b/cmd/cli/library.go @@ -11,8 +11,9 @@ type AppCallback struct { // AppConfig allows overwriting ctrld cli flags from mobile platforms. type AppConfig struct { - CdUID string - HomeDir string - Verbose int - LogPath string + CdUID string + HomeDir string + UpstreamProto string + Verbose int + LogPath string } diff --git a/cmd/ctrld_library/main.go b/cmd/ctrld_library/main.go index 526dd3b..9bcc151 100644 --- a/cmd/ctrld_library/main.go +++ b/cmd/ctrld_library/main.go @@ -28,14 +28,15 @@ type AppCallback interface { // Start configures utility with config.toml from provided directory. // This function will block until Stop is called // Check port availability prior to calling it. -func (c *Controller) Start(CdUID string, HomeDir string, logLevel int, logPath string) { +func (c *Controller) Start(CdUID string, HomeDir string, UpstreamProto string, logLevel int, logPath string) { if c.stopCh == nil { c.stopCh = make(chan struct{}) c.Config = cli.AppConfig{ - CdUID: CdUID, - HomeDir: HomeDir, - Verbose: logLevel, - LogPath: logPath, + CdUID: CdUID, + HomeDir: HomeDir, + UpstreamProto: UpstreamProto, + Verbose: logLevel, + LogPath: logPath, } appCallback := mapCallback(c.AppCallback) cli.RunMobile(&c.Config, &appCallback, c.stopCh) From eaad24e5e563faf7dafb1216a463f1d8a68cfd58 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 5 Jan 2024 21:04:18 +0700 Subject: [PATCH 07/16] internal/clientinfo: add host_entries.conf parser --- internal/clientinfo/hostsfile.go | 73 +++++++++++++++++++++++---- internal/clientinfo/hostsfile_test.go | 44 ++++++++++++++++ 2 files changed, 106 insertions(+), 11 deletions(-) diff --git a/internal/clientinfo/hostsfile.go b/internal/clientinfo/hostsfile.go index 8c86987..cf7798e 100644 --- a/internal/clientinfo/hostsfile.go +++ b/internal/clientinfo/hostsfile.go @@ -1,8 +1,12 @@ package clientinfo import ( + "bufio" + "bytes" + "io" "net/netip" "os" + "strings" "sync" "github.com/fsnotify/fsnotify" @@ -12,9 +16,10 @@ import ( ) const ( - ipv4LocalhostName = "localhost" - ipv6LocalhostName = "ip6-localhost" - ipv6LoopbackName = "ip6-loopback" + ipv4LocalhostName = "localhost" + ipv6LocalhostName = "ip6-localhost" + ipv6LoopbackName = "ip6-loopback" + hostEntriesConfPath = "/var/unbound/host_entries.conf" ) // hostsFile provides client discovery functionality using system hosts file. @@ -34,14 +39,9 @@ func (hf *hostsFile) init() error { if err := hf.watcher.Add(hostsfile.HostsPath); err != nil { return err } - m, err := hostsfile.ParseHosts(hostsfile.ReadHostsFile()) - if err != nil { - return err - } - hf.mu.Lock() - hf.m = m - hf.mu.Unlock() - return nil + // Conservatively adding hostEntriesConfPath, since it is not available everywhere. + _ = hf.watcher.Add(hostEntriesConfPath) + return hf.refresh() } // refresh reloads hosts file entries. @@ -52,6 +52,14 @@ func (hf *hostsFile) refresh() error { } hf.mu.Lock() hf.m = m + // override hosts file with host_entries.conf content if present. + hem, err := parseHostEntriesConf(hostEntriesConfPath) + if err != nil && !os.IsNotExist(err) { + ctrld.ProxyLogger.Load().Debug().Err(err).Msg("could not read host_entries.conf file") + } + for k, v := range hem { + hf.m[k] = v + } hf.mu.Unlock() return nil } @@ -137,3 +145,46 @@ func isLocalhostName(hostname string) bool { return false } } + +// parseHostEntriesConf parses host_entries.conf file and returns parsed result. +func parseHostEntriesConf(path string) (map[string][]string, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return parseHostEntriesConfFromReader(bytes.NewReader(b)), nil +} + +// parseHostEntriesConfFromReader is like parseHostEntriesConf, but read from an io.Reader instead of file. +func parseHostEntriesConfFromReader(r io.Reader) map[string][]string { + hostsMap := map[string][]string{} + scanner := bufio.NewScanner(r) + + localZone := "" + for scanner.Scan() { + line := scanner.Text() + if after, found := strings.CutPrefix(line, "local-zone:"); found { + after = strings.TrimSpace(after) + fields := strings.Fields(after) + if len(fields) > 1 { + localZone = strings.Trim(fields[0], `""`) + } + continue + } + // Only read "local-data-ptr: ..." line, it has all necessary information. + after, found := strings.CutPrefix(line, "local-data-ptr:") + if !found { + continue + } + after = strings.TrimSpace(after) + after = strings.Trim(after, `"`) + fields := strings.Fields(after) + if len(fields) != 2 { + continue + } + ip := fields[0] + name := strings.TrimSuffix(fields[1], "."+localZone) + hostsMap[ip] = append(hostsMap[ip], name) + } + return hostsMap +} diff --git a/internal/clientinfo/hostsfile_test.go b/internal/clientinfo/hostsfile_test.go index f67fcef..27c8b90 100644 --- a/internal/clientinfo/hostsfile_test.go +++ b/internal/clientinfo/hostsfile_test.go @@ -1,6 +1,7 @@ package clientinfo import ( + "strings" "testing" ) @@ -31,3 +32,46 @@ func Test_hostsFile_LookupHostnameByIP(t *testing.T) { }) } } + +func Test_parseHostEntriesConfFromReader(t *testing.T) { + const content = `local-zone: "localdomain" transparent +local-data-ptr: "127.0.0.1 localhost" +local-data: "localhost A 127.0.0.1" +local-data: "localhost.localdomain A 127.0.0.1" +local-data-ptr: "::1 localhost" +local-data: "localhost AAAA ::1" +local-data: "localhost.localdomain AAAA ::1" +local-data-ptr: "10.0.10.227 OPNsense.localdomain" +local-data: "OPNsense.localdomain A 10.0.10.227" +local-data: "OPNsense A 10.0.10.227" +local-data-ptr: "fe80::5a78:4e29:caa3:f9f7 OPNsense.localdomain" +local-data: "OPNsense.localdomain AAAA fe80::5a78:4e29:caa3:f9f7" +local-data: "OPNsense AAAA fe80::5a78:4e29:caa3:f9f7" +local-data-ptr: "1.1.1.1 banana-party.local.com" +local-data: "banana-party.local.com IN A 1.1.1.1" +local-data-ptr: "1.1.1.1 cheese-land.lan" +local-data: "cheese-land.lan IN A 1.1.1.1" +` + r := strings.NewReader(content) + hostsMap := parseHostEntriesConfFromReader(r) + if len(hostsMap) != 5 { + t.Fatalf("unexpected number of entries, want 5, got: %d", len(hostsMap)) + } + for ip, names := range hostsMap { + switch ip { + case "1.1.1.1": + for _, name := range names { + if name != "banana-party.local.com" && name != "cheese-land.lan" { + t.Fatalf("unexpected names for 1.1.1.1: %v", names) + } + } + case "10.0.10.227": + if len(names) != 1 { + t.Fatalf("unexpected names for 10.0.10.227: %v", names) + } + if names[0] != "OPNsense" { + t.Fatalf("unexpected name: %s", names[0]) + } + } + } +} From 9c1665a7599d014fe7bc4f307f04352ac8fcaeba Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 5 Jan 2024 23:39:31 +0700 Subject: [PATCH 08/16] internal/clientinfo: add kea-dhcp4 parser --- client_info.go | 1 + internal/clientinfo/dhcp.go | 59 ++++++++++++++++++++++++- internal/clientinfo/dhcp_lease_files.go | 1 + internal/clientinfo/dhcp_test.go | 41 +++++++++++++++++ 4 files changed, 100 insertions(+), 2 deletions(-) diff --git a/client_info.go b/client_info.go index 05d2910..24e09c3 100644 --- a/client_info.go +++ b/client_info.go @@ -18,4 +18,5 @@ type LeaseFileFormat string const ( Dnsmasq LeaseFileFormat = "dnsmasq" IscDhcpd LeaseFileFormat = "isc-dhcpd" + KeaDHCP4 LeaseFileFormat = "kea-dhcp4" ) diff --git a/internal/clientinfo/dhcp.go b/internal/clientinfo/dhcp.go index ebbeb77..1bbe405 100644 --- a/internal/clientinfo/dhcp.go +++ b/internal/clientinfo/dhcp.go @@ -3,6 +3,7 @@ package clientinfo import ( "bufio" "bytes" + "encoding/csv" "fmt" "io" "net" @@ -202,7 +203,8 @@ func (d *dhcp) dnsmasqReadClientInfoFile(name string) error { } -// dnsmasqReadClientInfoReader likes ctrld.Dnsmasq, but reading from an io.Reader instead of file. +// dnsmasqReadClientInfoReader performs the same task as dnsmasqReadClientInfoFile, +// but by reading from an io.Reader instead of file. func (d *dhcp) dnsmasqReadClientInfoReader(reader io.Reader) error { return lineread.Reader(reader, func(line []byte) error { fields := bytes.Fields(line) @@ -244,7 +246,8 @@ func (d *dhcp) iscDHCPReadClientInfoFile(name string) error { return d.iscDHCPReadClientInfoReader(f) } -// iscDHCPReadClientInfoReader likes ctrld.IscDhcpd, but reading from an io.Reader instead of file. +// iscDHCPReadClientInfoReader performs the same task as iscDHCPReadClientInfoFile, +// but by reading from an io.Reader instead of file. func (d *dhcp) iscDHCPReadClientInfoReader(reader io.Reader) error { s := bufio.NewScanner(reader) var ip, mac, hostname string @@ -287,6 +290,58 @@ func (d *dhcp) iscDHCPReadClientInfoReader(reader io.Reader) error { return nil } +// keaDhcp4ReadClientInfoFile populates dhcp table with client info reading from kea dhcp4 lease file. +func (d *dhcp) keaDhcp4ReadClientInfoFile(name string) error { + f, err := os.Open(name) + if err != nil { + return err + } + defer f.Close() + return d.keaDhcp4ReadClientInfoReader(bufio.NewReader(f)) + +} + +// keaDhcp4ReadClientInfoReader performs the same task as keaDhcp4ReadClientInfoFile, +// but by reading from an io.Reader instead of file. +func (d *dhcp) keaDhcp4ReadClientInfoReader(r io.Reader) error { + cr := csv.NewReader(r) + for { + record, err := cr.Read() + if err == io.EOF { + break + } + if err != nil { + return err + } + if len(record) < 9 { + continue // hostname is at 9th field, so skipping record with not enough fields. + } + if record[0] == "address" { + continue // skip header. + } + mac := record[1] + if _, err := net.ParseMAC(mac); err != nil { // skip invalid MAC + continue + } + ip := normalizeIP(record[0]) + if net.ParseIP(ip) == nil { + ctrld.ProxyLogger.Load().Warn().Msgf("invalid ip address entry: %q", ip) + ip = "" + } + + d.mac.Store(ip, mac) + d.ip.Store(mac, ip) + hostname := record[8] + if hostname == "*" { + continue + } + name := normalizeHostname(hostname) + d.mac2name.Store(mac, name) + d.ip2name.Store(ip, name) + } + return nil +} + // addSelf populates current host info to dhcp, so queries from // the host itself can be attached with proper client info. func (d *dhcp) addSelf() { diff --git a/internal/clientinfo/dhcp_lease_files.go b/internal/clientinfo/dhcp_lease_files.go index 4932a4b..1b5d829 100644 --- a/internal/clientinfo/dhcp_lease_files.go +++ b/internal/clientinfo/dhcp_lease_files.go @@ -15,4 +15,5 @@ var clientInfoFiles = map[string]ctrld.LeaseFileFormat{ "/run/dhcpd.leases": ctrld.IscDhcpd, // EdgeOS "/var/dhcpd/var/db/dhcpd.leases": ctrld.IscDhcpd, // Pfsense "/home/pi/.router/run/dhcp/dnsmasq.leases": ctrld.Dnsmasq, // Firewalla + "/var/lib/kea/dhcp4.leases": ctrld.KeaDHCP4, // Pfsense } diff --git a/internal/clientinfo/dhcp_test.go b/internal/clientinfo/dhcp_test.go index 359f441..07dbf5a 100644 --- a/internal/clientinfo/dhcp_test.go +++ b/internal/clientinfo/dhcp_test.go @@ -67,6 +67,41 @@ lease 192.168.1.2 { "00:00:00:00:00:04", "example", }, + { + "kea-dhcp4 good", + `address,hwaddr,client_id,valid_lifetime,expire,subnet_id,fqdn_fwd,fqdn_rev,hostname,state,user_context,pool_id +192.168.0.123,00:00:00:00:00:05,00:00:00:00:00:05,7200,1703290639,1,0,0,foo,0,,0 +`, + d.keaDhcp4ReadClientInfoReader, + "00:00:00:00:00:05", + "foo", + }, + { + "kea-dhcp4 no-header", + `192.168.0.123,00:00:00:00:00:05,00:00:00:00:00:05,7200,1703290639,1,0,0,foo,0,,0`, + d.keaDhcp4ReadClientInfoReader, + "00:00:00:00:00:05", + "foo", + }, + { + "kea-dhcp4 hostname *", + `address,hwaddr,client_id,valid_lifetime,expire,subnet_id,fqdn_fwd,fqdn_rev,hostname,state,user_context,pool_id +192.168.0.123,00:00:00:00:00:05,00:00:00:00:00:05,7200,1703290639,1,0,0,*,0,,0 +`, + d.keaDhcp4ReadClientInfoReader, + "00:00:00:00:00:05", + "*", + }, + { + "kea-dhcp4 bad", + `address,hwaddr,client_id,valid_lifetime,expire,subnet_id,fqdn_fwd,fqdn_rev,hostname,state,user_context,pool_id +192.168.0.123,00:00:00:00:00:05,00:00:00:00:00:05,7200,1703290639,1,0,0,foo,0,,0 +192.168.0.124,invalid_MAC,00:00:00:00:00:05,7200,1703290639,1,0,0,foo,0,,0 +`, + d.keaDhcp4ReadClientInfoReader, + "00:00:00:00:00:05", + "foo", + }, } for _, tc := range tests { @@ -76,6 +111,12 @@ lease 192.168.1.2 { t.Errorf("readClientInfoReader() error = %v", err) } val, existed := d.mac2name.Load(tc.mac) + if tc.hostname == "*" { + if existed { + t.Errorf("* hostname must be skipped") + } + return + } if !existed { t.Error("client info missing") } From af38623590a9dc4f924add726eb733c1196eadcb Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 11 Jan 2024 21:10:54 +0700 Subject: [PATCH 09/16] internal/clientinfo: read mdns data from avahi-daemon cache When avahi-daemon is avaibale, reading data from its cache help ctrld populate the mdns data with already known services within local network, allowing discover client info more quickly. --- internal/clientinfo/mdns.go | 44 ++++++++++++++++++++++++++++++++ internal/clientinfo/mdns_test.go | 27 ++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 internal/clientinfo/mdns_test.go diff --git a/internal/clientinfo/mdns.go b/internal/clientinfo/mdns.go index 3f0a311..3fb004e 100644 --- a/internal/clientinfo/mdns.go +++ b/internal/clientinfo/mdns.go @@ -1,11 +1,16 @@ package clientinfo import ( + "bufio" + "bytes" "context" "errors" + "io" "net" "net/netip" "os" + "os/exec" + "strings" "sync" "syscall" "time" @@ -107,6 +112,7 @@ func (m *mdns) init(quitCh chan struct{}) error { go m.probeLoop(v4ConnList, mdnsV4Addr, quitCh) go m.probeLoop(v6ConnList, mdnsV6Addr, quitCh) + go m.getDataFromAvahiDaemonCache() return nil } @@ -212,6 +218,44 @@ func (m *mdns) probe(conns []*net.UDPConn, remoteAddr net.Addr) error { return err } +// getDataFromAvahiDaemonCache reads entries from avahi-daemon cache to update mdns data. +func (m *mdns) getDataFromAvahiDaemonCache() { + if _, err := exec.LookPath("avahi-browse"); err != nil { + ctrld.ProxyLogger.Load().Debug().Err(err).Msg("could not find avahi-browse binary, skipping.") + return + } + // Run avahi-browse to discover services from cache: + // - "-a" -> all services. + // - "-r" -> resolve found services. + // - "-p" -> parseable format. + // - "-c" -> read from cache. + out, err := exec.Command("avahi-browse", "-a", "-r", "-p", "-c").Output() + if err != nil { + ctrld.ProxyLogger.Load().Debug().Err(err).Msg("could not browse services from avahi cache") + return + } + m.storeDataFromAvahiBrowseOutput(bytes.NewReader(out)) +} + +// storeDataFromAvahiBrowseOutput parses avahi-browse output from reader, then updating found data to mdns table. +func (m *mdns) storeDataFromAvahiBrowseOutput(r io.Reader) { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + fields := strings.FieldsFunc(scanner.Text(), func(r rune) bool { + return r == ';' + }) + if len(fields) < 8 || fields[0] != "=" { + continue + } + ip := fields[7] + name := normalizeHostname(fields[6]) + // Only using cache value if we don't have existed one. + if _, loaded := m.name.LoadOrStore(ip, name); !loaded { + ctrld.ProxyLogger.Load().Debug().Msgf("found hostname: %q, ip: %q via avahi cache", name, ip) + } + } +} + func multicastInterfaces() ([]net.Interface, error) { ifaces, err := net.Interfaces() if err != nil { diff --git a/internal/clientinfo/mdns_test.go b/internal/clientinfo/mdns_test.go new file mode 100644 index 0000000..e6f8698 --- /dev/null +++ b/internal/clientinfo/mdns_test.go @@ -0,0 +1,27 @@ +package clientinfo + +import ( + "strings" + "testing" +) + +func Test_mdns_storeDataFromAvahiBrowseOutput(t *testing.T) { + const content = `+;wlp0s20f3;IPv6;Foo\032\0402\041;_companion-link._tcp;local ++;wlp0s20f3;IPv4;Foo\032\0402\041;_companion-link._tcp;local +=;wlp0s20f3;IPv6;Foo\032\0402\041;_companion-link._tcp;local;Foo-2.local;192.168.1.123;64842;"rpBA=00:00:00:00:00:01" "rpHI=e6ae2cbbca0e" "rpAD=36566f4d850f" "rpVr=510.71.1" "rpHA=0ddc20fdddc8" "rpFl=0x30000" "rpHN=1d4a03afdefa" "rpMac=0" +=;wlp0s20f3;IPv4;Foo\032\0402\041;_companion-link._tcp;local;Foo-2.local;192.168.1.123;64842;"rpBA=00:00:00:00:00:01" "rpHI=e6ae2cbbca0e" "rpAD=36566f4d850f" "rpVr=510.71.1" "rpHA=0ddc20fdddc8" "rpFl=0x30000" "rpHN=1d4a03afdefa" "rpMac=0" +` + m := &mdns{} + m.storeDataFromAvahiBrowseOutput(strings.NewReader(content)) + ip := "192.168.1.123" + val, loaded := m.name.LoadOrStore(ip, "") + if !loaded { + t.Fatal("missing Foo-2 data from mdns table") + } + + wantHostname := "Foo-2" + hostname := val.(string) + if hostname != wantHostname { + t.Fatalf("unexpected hostname, want: %q, got: %q", wantHostname, hostname) + } +} From 44352f80067e840201fc92af334397bfe00d6731 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 11 Jan 2024 22:18:04 +0700 Subject: [PATCH 10/16] all: make discovery refresh interval configurable --- config.go | 35 +++++++++++++++--------------- docs/config.md | 8 +++++++ internal/clientinfo/client_info.go | 19 +++++++++++----- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/config.go b/config.go index 0b5d39c..2ffec37 100644 --- a/config.go +++ b/config.go @@ -179,23 +179,24 @@ func (c *Config) FirstUpstream() *UpstreamConfig { // ServiceConfig specifies the general ctrld config. type ServiceConfig struct { - LogLevel string `mapstructure:"log_level" toml:"log_level,omitempty"` - LogPath string `mapstructure:"log_path" toml:"log_path,omitempty"` - CacheEnable bool `mapstructure:"cache_enable" toml:"cache_enable,omitempty"` - CacheSize int `mapstructure:"cache_size" toml:"cache_size,omitempty"` - CacheTTLOverride int `mapstructure:"cache_ttl_override" toml:"cache_ttl_override,omitempty"` - CacheServeStale bool `mapstructure:"cache_serve_stale" toml:"cache_serve_stale,omitempty"` - MaxConcurrentRequests *int `mapstructure:"max_concurrent_requests" toml:"max_concurrent_requests,omitempty" validate:"omitempty,gte=0"` - DHCPLeaseFile string `mapstructure:"dhcp_lease_file_path" toml:"dhcp_lease_file_path" validate:"omitempty,file"` - DHCPLeaseFileFormat string `mapstructure:"dhcp_lease_file_format" toml:"dhcp_lease_file_format" validate:"required_unless=DHCPLeaseFile '',omitempty,oneof=dnsmasq isc-dhcp"` - DiscoverMDNS *bool `mapstructure:"discover_mdns" toml:"discover_mdns,omitempty"` - DiscoverARP *bool `mapstructure:"discover_arp" toml:"discover_arp,omitempty"` - DiscoverDHCP *bool `mapstructure:"discover_dhcp" toml:"discover_dhcp,omitempty"` - DiscoverPtr *bool `mapstructure:"discover_ptr" toml:"discover_ptr,omitempty"` - DiscoverHosts *bool `mapstructure:"discover_hosts" toml:"discover_hosts,omitempty"` - ClientIDPref string `mapstructure:"client_id_preference" toml:"client_id_preference,omitempty" validate:"omitempty,oneof=host mac"` - Daemon bool `mapstructure:"-" toml:"-"` - AllocateIP bool `mapstructure:"-" toml:"-"` + LogLevel string `mapstructure:"log_level" toml:"log_level,omitempty"` + LogPath string `mapstructure:"log_path" toml:"log_path,omitempty"` + CacheEnable bool `mapstructure:"cache_enable" toml:"cache_enable,omitempty"` + CacheSize int `mapstructure:"cache_size" toml:"cache_size,omitempty"` + CacheTTLOverride int `mapstructure:"cache_ttl_override" toml:"cache_ttl_override,omitempty"` + CacheServeStale bool `mapstructure:"cache_serve_stale" toml:"cache_serve_stale,omitempty"` + MaxConcurrentRequests *int `mapstructure:"max_concurrent_requests" toml:"max_concurrent_requests,omitempty" validate:"omitempty,gte=0"` + DHCPLeaseFile string `mapstructure:"dhcp_lease_file_path" toml:"dhcp_lease_file_path" validate:"omitempty,file"` + DHCPLeaseFileFormat string `mapstructure:"dhcp_lease_file_format" toml:"dhcp_lease_file_format" validate:"required_unless=DHCPLeaseFile '',omitempty,oneof=dnsmasq isc-dhcp"` + DiscoverMDNS *bool `mapstructure:"discover_mdns" toml:"discover_mdns,omitempty"` + DiscoverARP *bool `mapstructure:"discover_arp" toml:"discover_arp,omitempty"` + DiscoverDHCP *bool `mapstructure:"discover_dhcp" toml:"discover_dhcp,omitempty"` + DiscoverPtr *bool `mapstructure:"discover_ptr" toml:"discover_ptr,omitempty"` + DiscoverHosts *bool `mapstructure:"discover_hosts" toml:"discover_hosts,omitempty"` + DiscoverRefreshInterval int `mapstructure:"discover_refresh_interval" toml:"discover_refresh_interval,omitempty"` + ClientIDPref string `mapstructure:"client_id_preference" toml:"client_id_preference,omitempty" validate:"omitempty,oneof=host mac"` + Daemon bool `mapstructure:"-" toml:"-"` + AllocateIP bool `mapstructure:"-" toml:"-"` } // NetworkConfig specifies configuration for networks where ctrld will handle requests. diff --git a/docs/config.md b/docs/config.md index 90c15e0..b92e320 100644 --- a/docs/config.md +++ b/docs/config.md @@ -200,6 +200,14 @@ Perform LAN client discovery using hosts file. - Required: no - Default: true +### discover_refresh_interval +Time in seconds between each discovery refresh loop to update new client information data. +The default value is 120 seconds, lower this value to make the discovery process run more aggressively. + +- Type: integer +- Required: no +- Default: 120 + ### dhcp_lease_file_path Relative or absolute path to a custom DHCP leases file location. diff --git a/internal/clientinfo/client_info.go b/internal/clientinfo/client_info.go index 1fe1083..0e5c157 100644 --- a/internal/clientinfo/client_info.go +++ b/internal/clientinfo/client_info.go @@ -70,6 +70,7 @@ type Table struct { hostnameResolvers []HostnameResolver refreshers []refresher initOnce sync.Once + refreshInterval int dhcp *dhcp merlin *merlinDiscover @@ -88,12 +89,17 @@ type Table struct { } func NewTable(cfg *ctrld.Config, selfIP, cdUID string, ns []string) *Table { + refreshInterval := cfg.Service.DiscoverRefreshInterval + if refreshInterval <= 0 { + refreshInterval = 2 * 60 // 2 minutes + } return &Table{ - svcCfg: cfg.Service, - quitCh: make(chan struct{}), - selfIP: selfIP, - cdUID: cdUID, - ptrNameservers: ns, + svcCfg: cfg.Service, + quitCh: make(chan struct{}), + selfIP: selfIP, + cdUID: cdUID, + ptrNameservers: ns, + refreshInterval: refreshInterval, } } @@ -104,8 +110,9 @@ func (t *Table) AddLeaseFile(name string, format ctrld.LeaseFileFormat) { clientInfoFiles[name] = format } +// RefreshLoop runs all the refresher to update new client info data. func (t *Table) RefreshLoop(ctx context.Context) { - timer := time.NewTicker(time.Minute * 5) + timer := time.NewTicker(time.Second * time.Duration(t.refreshInterval)) defer timer.Stop() for { select { From 71f26a6d81b5901b4dc0441f048c58e44289baad Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Tue, 9 Jan 2024 01:42:50 +0700 Subject: [PATCH 11/16] Add prometheus exporter Updates #6 --- cmd/cli/cli.go | 13 ++- cmd/cli/control_server.go | 21 ++++ cmd/cli/dns_proxy.go | 66 ++++++++++--- cmd/cli/dns_proxy_test.go | 4 +- cmd/cli/metrics.go | 150 +++++++++++++++++++++++++++++ cmd/cli/prog.go | 12 +++ cmd/cli/prometheus.go | 57 +++++++++++ config.go | 2 + docs/config.md | 14 +++ go.mod | 10 ++ go.sum | 22 +++++ internal/clientinfo/client_info.go | 10 +- 12 files changed, 363 insertions(+), 18 deletions(-) create mode 100644 cmd/cli/metrics.go create mode 100644 cmd/cli/prometheus.go diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 7176356..57761f2 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -719,6 +719,10 @@ NOTE: Uninstalling will set DNS to values provided by DHCP.`, sort.Strings(s) return s } + // If metrics is enabled, server set this for all clients, so we can check only the first one. + // Ideally, we may have a field in response to indicate that query count should be shown, but + // it would break earlier version of ctrld, which only look list of clients in response. + withQueryCount := len(clients) > 0 && clients[0].IncludeQueryCount data := make([][]string, len(clients)) for i, c := range clients { row := []string{ @@ -727,10 +731,17 @@ NOTE: Uninstalling will set DNS to values provided by DHCP.`, c.Mac, strings.Join(map2Slice(c.Source), ","), } + if withQueryCount { + row = append(row, strconv.FormatInt(c.QueryCount, 10)) + } data[i] = row } table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"IP", "Hostname", "Mac", "Discovered"}) + headers := []string{"IP", "Hostname", "Mac", "Discovered"} + if withQueryCount { + headers = append(headers, "Queries") + } + table.SetHeader(headers) table.SetAutoFormatHeaders(false) table.AppendBulk(data) table.Render() diff --git a/cmd/cli/control_server.go b/cmd/cli/control_server.go index 5ee7112..36749c1 100644 --- a/cmd/cli/control_server.go +++ b/cmd/cli/control_server.go @@ -10,6 +10,8 @@ import ( "sort" "time" + dto "github.com/prometheus/client_model/go" + "github.com/Control-D-Inc/ctrld" ) @@ -66,6 +68,25 @@ func (p *prog) registerControlServerHandler() { sort.Slice(clients, func(i, j int) bool { return clients[i].IP.Less(clients[j].IP) }) + if p.cfg.Service.MetricsQueryStats { + for _, client := range clients { + client.IncludeQueryCount = true + dm := &dto.Metric{} + m, err := statsClientQueriesCount.MetricVec.GetMetricWithLabelValues( + client.IP.String(), + client.Mac, + client.Hostname, + ) + if err != nil { + mainLog.Load().Debug().Err(err).Msgf("could not get metrics for client: %v", client) + continue + } + if err := m.Write(dm); err == nil { + client.QueryCount = int64(dm.Counter.GetValue()) + } + } + } + if err := json.NewEncoder(w).Encode(&clients); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index fc319f2..0b6282e 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -54,6 +54,14 @@ type proxyRequest struct { ufr *upstreamForResult } +// proxyResponse contains data for proxying a DNS response from upstream. +type proxyResponse struct { + answer *dns.Msg + cached bool + clientInfo bool + upstream string +} + // upstreamForResult represents the result of processing rules for a request. type upstreamForResult struct { upstreams []string @@ -101,26 +109,49 @@ func (p *prog) serveDNS(listenerNum string) error { fmtSrcToDest := fmtRemoteToLocal(listenerNum, ci.Hostname, remoteAddr.String()) t := time.Now() ctrld.Log(ctx, mainLog.Load().Info(), "QUERY: %s: %s %s", fmtSrcToDest, dns.TypeToString[q.Qtype], domain) - res := p.upstreamFor(ctx, listenerNum, listenerConfig, remoteAddr, ci.Mac, domain) + ur := p.upstreamFor(ctx, listenerNum, listenerConfig, remoteAddr, ci.Mac, domain) + + labelValues := make([]string, 0, len(statsQueriesCountLabels)) + labelValues = append(labelValues, net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port))) + labelValues = append(labelValues, ci.IP) + labelValues = append(labelValues, ci.Mac) + labelValues = append(labelValues, ci.Hostname) + var answer *dns.Msg - if !res.matched && listenerConfig.Restricted { + if !ur.matched && listenerConfig.Restricted { ctrld.Log(ctx, mainLog.Load().Info(), "query refused, %s does not match any network policy", remoteAddr.String()) answer = new(dns.Msg) answer.SetRcode(m, dns.RcodeRefused) + labelValues = append(labelValues, "") // no upstream } else { var failoverRcode []int if listenerConfig.Policy != nil { failoverRcode = listenerConfig.Policy.FailoverRcodeNumbers } - answer = p.proxy(ctx, &proxyRequest{ + pr := p.proxy(ctx, &proxyRequest{ msg: m, ci: ci, failoverRcodes: failoverRcode, - ufr: res, + ufr: ur, }) + answer = pr.answer rtt := time.Since(t) ctrld.Log(ctx, mainLog.Load().Debug(), "received response of %d bytes in %s", answer.Len(), rtt) + upstream := pr.upstream + switch { + case pr.cached: + upstream = "cache" + case pr.clientInfo: + upstream = "client_info_table" + } + labelValues = append(labelValues, upstream) } + labelValues = append(labelValues, dns.TypeToString[q.Qtype]) + labelValues = append(labelValues, dns.RcodeToString[answer.Rcode]) + go func() { + p.WithLabelValuesInc(statsQueriesCount, labelValues...) + p.WithLabelValuesInc(statsClientQueriesCount, []string{ci.IP, ci.Mac, ci.Hostname}...) + }() if err := w.WriteMsg(answer); err != nil { ctrld.Log(ctx, mainLog.Load().Error().Err(err), "serveDNS: failed to send DNS response to client") } @@ -360,7 +391,7 @@ func (p *prog) proxyLanHostnameQuery(ctx context.Context, msg *dns.Msg) *dns.Msg return nil } -func (p *prog) proxy(ctx context.Context, req *proxyRequest) *dns.Msg { +func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { var staleAnswer *dns.Msg upstreams := req.ufr.upstreams serveStaleCache := p.cache != nil && p.cfg.Service.CacheServeStale @@ -370,6 +401,8 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *dns.Msg { upstreams = []string{upstreamOS} } + res := &proxyResponse{} + // LAN/PTR lookup flow: // // 1. If there's matching rule, follow it. @@ -384,14 +417,18 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *dns.Msg { case isPrivatePtrLookup(req.msg): isLanOrPtrQuery = true if answer := p.proxyPrivatePtrLookup(ctx, req.msg); answer != nil { - return answer + res.answer = answer + res.clientInfo = true + return res } upstreams, upstreamConfigs = p.upstreamsAndUpstreamConfigForLanAndPtr(upstreams, upstreamConfigs) ctrld.Log(ctx, mainLog.Load().Debug(), "private PTR lookup, using upstreams: %v", upstreams) case isLanHostnameQuery(req.msg): isLanOrPtrQuery = true if answer := p.proxyLanHostnameQuery(ctx, req.msg); answer != nil { - return answer + res.answer = answer + res.clientInfo = true + return res } upstreams, upstreamConfigs = p.upstreamsAndUpstreamConfigForLanAndPtr(upstreams, upstreamConfigs) ctrld.Log(ctx, mainLog.Load().Debug(), "lan hostname lookup, using upstreams: %v", upstreams) @@ -413,7 +450,9 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *dns.Msg { if cachedValue.Expire.After(now) { ctrld.Log(ctx, mainLog.Load().Debug(), "hit cached response") setCachedAnswerTTL(answer, now, cachedValue.Expire) - return answer + res.answer = answer + res.cached = true + return res } staleAnswer = answer } @@ -475,7 +514,9 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *dns.Msg { ctrld.Log(ctx, mainLog.Load().Debug(), "serving stale cached response") now := time.Now() setCachedAnswerTTL(staleAnswer, now, now.Add(staleTTL)) - return staleAnswer + res.answer = staleAnswer + res.cached = true + return res } continue } @@ -509,12 +550,15 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *dns.Msg { hostname = req.ci.Hostname } ctrld.Log(ctx, mainLog.Load().Info(), "REPLY: %s -> %s (%s): %s", upstreams[n], req.ufr.srcAddr, hostname, dns.RcodeToString[answer.Rcode]) - return answer + res.answer = answer + res.upstream = upstreamConfig.Endpoint + return res } ctrld.Log(ctx, mainLog.Load().Error(), "all %v endpoints failed", upstreams) answer := new(dns.Msg) answer.SetRcode(req.msg, dns.RcodeServerFailure) - return answer + res.answer = answer + return res } func (p *prog) upstreamsAndUpstreamConfigForLanAndPtr(upstreams []string, upstreamConfigs []*ctrld.UpstreamConfig) ([]string, []*ctrld.UpstreamConfig) { diff --git a/cmd/cli/dns_proxy_test.go b/cmd/cli/dns_proxy_test.go index bd73d17..52d3edb 100644 --- a/cmd/cli/dns_proxy_test.go +++ b/cmd/cli/dns_proxy_test.go @@ -187,8 +187,8 @@ func TestCache(t *testing.T) { got1 := prog.proxy(context.Background(), req1) got2 := prog.proxy(context.Background(), req2) assert.NotSame(t, got1, got2) - assert.Equal(t, answer1.Rcode, got1.Rcode) - assert.Equal(t, answer2.Rcode, got2.Rcode) + assert.Equal(t, answer1.Rcode, got1.answer.Rcode) + assert.Equal(t, answer2.Rcode, got2.answer.Rcode) } func Test_ipAndMacFromMsg(t *testing.T) { diff --git a/cmd/cli/metrics.go b/cmd/cli/metrics.go new file mode 100644 index 0000000..ee64975 --- /dev/null +++ b/cmd/cli/metrics.go @@ -0,0 +1,150 @@ +package cli + +import ( + "context" + "encoding/json" + "net" + "net/http" + "runtime" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/prom2json" +) + +// metricsServer represents a server to expose Prometheus metrics via HTTP. +type metricsServer struct { + server *http.Server + mux *http.ServeMux + reg *prometheus.Registry + addr string + started bool +} + +// newMetricsServer returns new metrics server. +func newMetricsServer(addr string, reg *prometheus.Registry) (*metricsServer, error) { + mux := http.NewServeMux() + ms := &metricsServer{ + server: &http.Server{Handler: mux}, + mux: mux, + reg: reg, + } + ms.addr = addr + ms.registerMetricsServerHandler() + return ms, nil +} + +// register adds handlers for given pattern. +func (ms *metricsServer) register(pattern string, handler http.Handler) { + ms.mux.Handle(pattern, handler) +} + +// registerMetricsServerHandler adds handlers for metrics server. +func (ms *metricsServer) registerMetricsServerHandler() { + ms.register("/metrics", promhttp.HandlerFor( + ms.reg, + promhttp.HandlerOpts{ + EnableOpenMetrics: true, + Timeout: 10 * time.Second, + }, + )) + ms.register("/metrics/json", jsonResponse(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + g := prometheus.ToTransactionalGatherer(ms.reg) + mfs, done, err := g.Gather() + defer done() + if err != nil { + msg := "could not gather metrics" + mainLog.Load().Warn().Err(err).Msg(msg) + http.Error(w, msg, http.StatusInternalServerError) + return + } + result := make([]*prom2json.Family, 0, len(mfs)) + for _, mf := range mfs { + result = append(result, prom2json.NewFamily(mf)) + } + if err := json.NewEncoder(w).Encode(result); err != nil { + msg := "could not marshal metrics result" + mainLog.Load().Warn().Err(err).Msg(msg) + http.Error(w, msg, http.StatusInternalServerError) + return + } + }))) +} + +// start runs the metricsServer. +func (ms *metricsServer) start() error { + listener, err := net.Listen("tcp", ms.addr) + if err != nil { + return err + } + go ms.server.Serve(listener) + ms.started = true + return nil +} + +// stop shutdowns the metricsServer within 2 seconds timeout. +func (ms *metricsServer) stop() error { + if !ms.started { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*1) + defer cancel() + return ms.server.Shutdown(ctx) +} + +// runMetricsServer initializes metrics stats and runs the metrics server if enabled. +func (p *prog) runMetricsServer(ctx context.Context, reloadCh chan struct{}) { + if !p.metricsEnabled() { + return + } + + // Reset all stats. + statsVersion.Reset() + statsQueriesCount.Reset() + statsClientQueriesCount.Reset() + + reg := prometheus.NewRegistry() + // Register queries count stats if enabled. + if cfg.Service.MetricsQueryStats { + reg.MustRegister(statsQueriesCount) + reg.MustRegister(statsClientQueriesCount) + } + + addr := p.cfg.Service.MetricsListener + ms, err := newMetricsServer(addr, reg) + if err != nil { + mainLog.Load().Warn().Err(err).Msg("could not create new metrics server") + return + } + // Only start listener address if defined. + if addr != "" { + // Go runtime stats. + reg.MustRegister(collectors.NewBuildInfoCollector()) + reg.MustRegister(collectors.NewGoCollector( + collectors.WithGoCollectorRuntimeMetrics(collectors.MetricsAll), + )) + // ctrld stats. + reg.MustRegister(statsVersion) + statsVersion.WithLabelValues(commit, runtime.Version(), curVersion()).Inc() + reg.MustRegister(statsTimeStart) + statsTimeStart.Set(float64(time.Now().Unix())) + mainLog.Load().Debug().Msgf("starting metrics server on: %s", addr) + if err := ms.start(); err != nil { + mainLog.Load().Warn().Err(err).Msg("could not start metrics server") + return + } + } + + select { + case <-p.stopCh: + case <-ctx.Done(): + case <-reloadCh: + } + + if err := ms.stop(); err != nil { + mainLog.Load().Warn().Err(err).Msg("could not stop metrics server") + return + } +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 0d4f645..4b3968f 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -348,6 +348,13 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { p.checkDnsLoopTicker(ctx) }() + wg.Add(1) + // Prometheus exporter goroutine. + go func() { + defer wg.Done() + p.runMetricsServer(ctx, reloadCh) + }() + if !reload { // Stop writing log to unix socket. consoleWriter.Out = os.Stdout @@ -365,6 +372,11 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { wg.Wait() } +// metricsEnabled reports whether prometheus exporter is enabled/disabled. +func (p *prog) metricsEnabled() bool { + return p.cfg.Service.MetricsQueryStats || p.cfg.Service.MetricsListener != "" +} + func (p *prog) Stop(s service.Service) error { mainLog.Load().Info().Msg("Service stopped") close(p.stopCh) diff --git a/cmd/cli/prometheus.go b/cmd/cli/prometheus.go new file mode 100644 index 0000000..fc2fc5d --- /dev/null +++ b/cmd/cli/prometheus.go @@ -0,0 +1,57 @@ +package cli + +import "github.com/prometheus/client_golang/prometheus" + +const ( + metricsLabelListener = "listener" + metricsLabelClientSourceIP = "client_source_ip" + metricsLabelClientMac = "client_mac" + metricsLabelClientHostname = "client_hostname" + metricsLabelUpstream = "upstream" + metricsLabelRRType = "rr_type" + metricsLabelRCode = "rcode" +) + +// statsVersion represent ctrld version. +var statsVersion = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "ctrld_build_info", + Help: "Version of ctrld process.", +}, []string{"gitref", "goversion", "version"}) + +// statsTimeStart represents start time of ctrld service. +var statsTimeStart = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "ctrld_time_seconds", + Help: "Start time of the ctrld process since unix epoch in seconds.", +}) + +var statsQueriesCountLabels = []string{ + metricsLabelListener, + metricsLabelClientSourceIP, + metricsLabelClientMac, + metricsLabelClientHostname, + metricsLabelUpstream, + metricsLabelRRType, + metricsLabelRCode, +} + +// statsQueriesCount counts total number of queries. +var statsQueriesCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "ctrld_queries_count", + Help: "Total number of queries.", +}, statsQueriesCountLabels) + +// statsClientQueriesCount counts total number of queries of a client. +// +// The labels "client_source_ip", "client_mac", "client_hostname" are unbounded, +// thus this stat is highly inefficient if there are many devices. +var statsClientQueriesCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "ctrld_client_queries_count", + Help: "Total number queries of a client.", +}, []string{metricsLabelClientSourceIP, metricsLabelClientMac, metricsLabelClientHostname}) + +// WithLabelValuesInc increases prometheus counter by 1 if query stats is enabled. +func (p *prog) WithLabelValuesInc(c *prometheus.CounterVec, lvs ...string) { + if p.cfg.Service.MetricsQueryStats { + c.WithLabelValues(lvs...).Inc() + } +} diff --git a/config.go b/config.go index 2ffec37..cb38096 100644 --- a/config.go +++ b/config.go @@ -195,6 +195,8 @@ type ServiceConfig struct { DiscoverHosts *bool `mapstructure:"discover_hosts" toml:"discover_hosts,omitempty"` DiscoverRefreshInterval int `mapstructure:"discover_refresh_interval" toml:"discover_refresh_interval,omitempty"` ClientIDPref string `mapstructure:"client_id_preference" toml:"client_id_preference,omitempty" validate:"omitempty,oneof=host mac"` + MetricsQueryStats bool `mapstructure:"metrics_query_stats" toml:"metrics_query_stats,omitempty"` + MetricsListener string `mapstructure:"metrics_listener" toml:"metrics_listener,omitempty"` Daemon bool `mapstructure:"-" toml:"-"` AllocateIP bool `mapstructure:"-" toml:"-"` } diff --git a/docs/config.md b/docs/config.md index b92e320..4f4d897 100644 --- a/docs/config.md +++ b/docs/config.md @@ -234,6 +234,20 @@ Else -> client ID will use both Mac and Hostname i.e. `hash(mac + host) - Valid values: `mac`, `host` - Default: "" +### metrics_query_stats +If set to `true`, collect and export the query counters, and show them in `clients list` command. + +- Type: boolean +- Required: no +- Default: false + +### metrics_listener +Specifying the `ip` and `port` of the metrics server. + +- Type: string +- Required: no +- Default: "" + ## Upstream The `[upstream]` section specifies the DNS upstream servers that `ctrld` will forward DNS requests to. diff --git a/go.mod b/go.mod index 4d38445..ebb62d2 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,8 @@ require ( github.com/miekg/dns v1.1.55 github.com/olekukonko/tablewriter v0.0.5 github.com/pelletier/go-toml/v2 v2.0.8 + github.com/prometheus/client_golang v1.15.1 + github.com/prometheus/prom2json v1.3.3 github.com/quic-go/quic-go v0.38.0 github.com/rs/zerolog v1.28.0 github.com/spf13/cobra v1.7.0 @@ -35,11 +37,14 @@ require ( require ( github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/golang/mock v1.6.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -52,6 +57,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.18 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7 // indirect github.com/mdlayher/netlink v1.7.2 // indirect github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065 // indirect @@ -60,6 +66,9 @@ require ( github.com/onsi/ginkgo/v2 v2.9.5 // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-20 v0.3.2 // indirect github.com/rivo/uniseg v0.4.4 // indirect @@ -76,6 +85,7 @@ require ( golang.org/x/mod v0.10.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/tools v0.9.1 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a64357a..bbfe6c7 100644 --- a/go.sum +++ b/go.sum @@ -42,7 +42,11 @@ github.com/Windscribe/zerolog v0.0.0-20230503170159-e6aa153233be h1:qBKVRi7Mom5h github.com/Windscribe/zerolog v0.0.0-20230503170159-e6aa153233be/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -114,7 +118,9 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -126,6 +132,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -199,6 +206,8 @@ github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7 h1:lez6TS6aAau+8wXUP3G9I3TGlmPFEq2CTxBaRqY6AGE= github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y= github.com/mdlayher/ndp v1.0.1 h1:+yAD79/BWyFlvAoeG5ncPS0ItlHP/eVbH7bQ6/+LVA4= @@ -229,7 +238,17 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/prom2json v1.3.3 h1:IYfSMiZ7sSOfliBoo89PcufjWO4eAR0gznGcETyaUgo= +github.com/prometheus/prom2json v1.3.3/go.mod h1:Pv4yIPktEkK7btWsrUTWDDDrnpUrAELaOCj+oFwlgmc= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI= @@ -610,7 +629,10 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/internal/clientinfo/client_info.go b/internal/clientinfo/client_info.go index 0e5c157..7f41fca 100644 --- a/internal/clientinfo/client_info.go +++ b/internal/clientinfo/client_info.go @@ -58,10 +58,12 @@ type ipLister interface { } type Client struct { - IP netip.Addr - Mac string - Hostname string - Source map[string]struct{} + IP netip.Addr + Mac string + Hostname string + Source map[string]struct{} + QueryCount int64 + IncludeQueryCount bool } type Table struct { From 32709dc64cb62f440d49fa69238ec96784f212f9 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Tue, 16 Jan 2024 21:53:06 +0700 Subject: [PATCH 12/16] internal/router: use daemon -r option So if ctrld is killed unexpectedly, daemon will respawn new ctrld and keep the system DNS working. --- internal/router/os_freebsd.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/internal/router/os_freebsd.go b/internal/router/os_freebsd.go index 3b7c0a3..c38eebc 100644 --- a/internal/router/os_freebsd.go +++ b/internal/router/os_freebsd.go @@ -137,20 +137,9 @@ rcvar="${name}_enable" pidfile="/var/run/${name}.pid" child_pidfile="/var/run/${name}_child.pid" command="/usr/sbin/daemon" -daemon_args="-P ${pidfile} -p ${child_pidfile} -t \"${name}: daemon\"{{if .WorkingDirectory}} -c {{.WorkingDirectory}}{{end}}" +daemon_args="-r -P ${pidfile} -p ${child_pidfile} -t \"${name}: daemon\"{{if .WorkingDirectory}} -c {{.WorkingDirectory}}{{end}}" command_args="${daemon_args} {{.Path}}{{range .Arguments}} {{.}}{{end}}" -stop_cmd="ctrld_stop" - -ctrld_stop() { - pid=$(cat ${pidfile}) - child_pid=$(cat ${child_pidfile}) - if [ -e "${child_pidfile}" ]; then - kill -s TERM "${child_pid}" - wait_for_pids "${child_pid}" "${pidfile}" - fi -} - load_rc_config "${name}" run_rc_command "$1" ` From 251255c74630770073aa739e2d1a27308556b031 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 17 Jan 2024 14:50:29 +0700 Subject: [PATCH 13/16] all: change bootstrap DNS for ipv4/ipv6 --- internal/net/net.go | 4 ++-- resolver.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/net/net.go b/internal/net/net.go index 5f2c509..770c3db 100644 --- a/internal/net/net.go +++ b/internal/net/net.go @@ -16,8 +16,8 @@ import ( const ( controldIPv6Test = "ipv6.controld.io" - v4BootstrapDNS = "76.76.2.0:53" - v6BootstrapDNS = "[2606:1a40::]:53" + v4BootstrapDNS = "76.76.2.22:53" + v6BootstrapDNS = "[2606:1a40::22]:53" ) var Dialer = &net.Dialer{ diff --git a/resolver.go b/resolver.go index 750679c..0a4569e 100644 --- a/resolver.go +++ b/resolver.go @@ -30,7 +30,7 @@ const ( ResolverTypePrivate = "private" ) -var bootstrapDNS = "76.76.2.0" +const bootstrapDNS = "76.76.2.22" // or is the Resolver used for ResolverTypeOS. var or = &osResolver{nameservers: defaultNameservers()} From 28984090e537699103c00ddabbb8b51916e99cb7 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 18 Jan 2024 18:55:42 +0700 Subject: [PATCH 14/16] internal/router: report error if DNS shield is enabled in UniFi OS --- internal/router/edgeos/edgeos.go | 19 +++++++++++++++++++ internal/router/ubios/ubios.go | 4 ++++ 2 files changed, 23 insertions(+) diff --git a/internal/router/edgeos/edgeos.go b/internal/router/edgeos/edgeos.go index df7b57b..2e229ac 100644 --- a/internal/router/edgeos/edgeos.go +++ b/internal/router/edgeos/edgeos.go @@ -20,11 +20,15 @@ const ( usgDNSMasqConfigPath = "/etc/dnsmasq.conf" usgDNSMasqBackupConfigPath = "/etc/dnsmasq.conf.bak" toggleContentFilteringLink = "https://community.ui.com/questions/UDM-Pro-disable-enable-DNS-filtering/e2cc4060-e56a-4139-b200-62d7f773ff8f" + toggleDnsShieldLink = "https://community.ui.com/questions/UniFi-OS-3-2-7-DNS-Shield-Missing/d3a85905-4ce0-4fe4-8bf0-6cb04f21371d" ) var ErrContentFilteringEnabled = fmt.Errorf(`the "Content Filtering" feature" is enabled, which is conflicted with ctrld.\n To disable it, folowing instruction here: %s`, toggleContentFilteringLink) +var ErrDnsShieldEnabled = fmt.Errorf(`the "DNS Shield" feature" is enabled, which is conflicted with ctrld.\n +To disable it, folowing screenshot here: %s`, toggleDnsShieldLink) + type EdgeOS struct { cfg *ctrld.Config isUSG bool @@ -50,6 +54,11 @@ func (e *EdgeOS) Install(_ *service.Config) error { if ContentFilteringEnabled() { return ErrContentFilteringEnabled } + // If "DNS Shield" is enabled, UniFi OS will spawn dnscrypt-proxy process, and route all DNS queries to it. So + // reporting an error and guiding users to disable the feature using UniFi OS web UI. + if DnsShieldEnabled() { + return ErrDnsShieldEnabled + } return nil } @@ -169,6 +178,16 @@ func ContentFilteringEnabled() bool { return err == nil && !st.IsDir() } +// DnsShieldEnabled reports whether DNS Shield is enabled. +// See: https://community.ui.com/releases/UniFi-OS-Dream-Machines-3-2-7/251dfc1e-f4dd-4264-a080-3be9d8b9e02b +func DnsShieldEnabled() bool { + buf, err := os.ReadFile("/var/run/dnsmasq.conf.d/dns.conf") + if err != nil { + return false + } + return bytes.Contains(buf, []byte("server=127.0.0.1#5053")) +} + func LeaseFileDir() string { if checkUSG() { return "" diff --git a/internal/router/ubios/ubios.go b/internal/router/ubios/ubios.go index 6513657..a1f0b6c 100644 --- a/internal/router/ubios/ubios.go +++ b/internal/router/ubios/ubios.go @@ -36,6 +36,10 @@ func (u *Ubios) Install(config *service.Config) error { if edgeos.ContentFilteringEnabled() { return edgeos.ErrContentFilteringEnabled } + // See comment in (*edgeos.EdgeOS).Install method. + if edgeos.DnsShieldEnabled() { + return edgeos.ErrDnsShieldEnabled + } return nil } From e42554f89291945ed7e08d8b8b0ba2b4aa5e5483 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 22 Jan 2024 14:29:07 +0700 Subject: [PATCH 15/16] internal/router/dnsmasq: always include client's mac/ip Since ctrld now supports MAC rules, the client's mac and ip must always be sent to ctrld. Otherwise, the mac policy won't work when ctrld is an upstream of dnsmasq. --- internal/router/dnsmasq/dnsmasq.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/internal/router/dnsmasq/dnsmasq.go b/internal/router/dnsmasq/dnsmasq.go index 50c7d0e..c2f8845 100644 --- a/internal/router/dnsmasq/dnsmasq.go +++ b/internal/router/dnsmasq/dnsmasq.go @@ -15,10 +15,8 @@ no-resolv {{- range .Upstreams}} server={{ .IP }}#{{ .Port }} {{- end}} -{{- if .SendClientInfo}} add-mac add-subnet=32,128 -{{- end}} {{- if .CacheDisabled}} cache-size=0 {{- else}} @@ -44,12 +42,10 @@ if [ -n "$pid" ] && [ -f "/proc/${pid}/cmdline" ]; then {{- range .Upstreams}} pc_append "server={{ .IP }}#{{ .Port }}" "$config_file" {{- end}} - {{- if .SendClientInfo}} pc_delete "add-mac" "$config_file" pc_delete "add-subnet" "$config_file" pc_append "add-mac" "$config_file" # add client mac pc_append "add-subnet=32,128" "$config_file" # add client ip - {{- end}} pc_delete "dnssec" "$config_file" # disable DNSSEC pc_delete "trust-anchor=" "$config_file" # disable DNSSEC pc_delete "cache-size=" "$config_file" @@ -92,29 +88,27 @@ func ConfTmplWithCacheDisabled(tmplText string, cfg *ctrld.Config, cacheDisabled ip = "127.0.0.1" } upstreams := []Upstream{{IP: ip, Port: listener.Port}} - return confTmpl(tmplText, upstreams, cfg.HasUpstreamSendClientInfo(), cacheDisabled) + return confTmpl(tmplText, upstreams, cacheDisabled) } // FirewallaConfTmpl generates dnsmasq config for Firewalla routers. func FirewallaConfTmpl(tmplText string, cfg *ctrld.Config) (string, error) { // If ctrld listen on all interfaces, generating config for all of them. if lc := cfg.FirstListener(); lc != nil && (lc.IP == "0.0.0.0" || lc.IP == "") { - return confTmpl(tmplText, firewallaUpstreams(lc.Port), cfg.HasUpstreamSendClientInfo(), false) + return confTmpl(tmplText, firewallaUpstreams(lc.Port), false) } // Otherwise, generating config for the specific listener from ctrld's config. return ConfTmplWithCacheDisabled(tmplText, cfg, false) } -func confTmpl(tmplText string, upstreams []Upstream, sendClientInfo, cacheDisabled bool) (string, error) { +func confTmpl(tmplText string, upstreams []Upstream, cacheDisabled bool) (string, error) { tmpl := template.Must(template.New("").Parse(tmplText)) var to = &struct { - SendClientInfo bool - Upstreams []Upstream - CacheDisabled bool + Upstreams []Upstream + CacheDisabled bool }{ - SendClientInfo: sendClientInfo, - Upstreams: upstreams, - CacheDisabled: cacheDisabled, + Upstreams: upstreams, + CacheDisabled: cacheDisabled, } var sb strings.Builder if err := tmpl.Execute(&sb, to); err != nil { From 6d3c86c0bea8f1842a26573884f1d59ab6483c07 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 22 Jan 2024 23:51:52 +0700 Subject: [PATCH 16/16] internal/clientinfo: add kea-dhcp4 to readLeaseFile While at it, also removing duplicated characters in cutset of strings.Trim function. --- internal/clientinfo/dhcp.go | 2 ++ internal/clientinfo/hostsfile.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/clientinfo/dhcp.go b/internal/clientinfo/dhcp.go index 1bbe405..9d1f339 100644 --- a/internal/clientinfo/dhcp.go +++ b/internal/clientinfo/dhcp.go @@ -188,6 +188,8 @@ func (d *dhcp) readLeaseFile(name string, format ctrld.LeaseFileFormat) error { return d.dnsmasqReadClientInfoFile(name) case ctrld.IscDhcpd: return d.iscDHCPReadClientInfoFile(name) + case ctrld.KeaDHCP4: + return d.keaDhcp4ReadClientInfoFile(name) } return fmt.Errorf("unsupported format: %s, file: %s", format, name) } diff --git a/internal/clientinfo/hostsfile.go b/internal/clientinfo/hostsfile.go index cf7798e..c758f3b 100644 --- a/internal/clientinfo/hostsfile.go +++ b/internal/clientinfo/hostsfile.go @@ -167,7 +167,7 @@ func parseHostEntriesConfFromReader(r io.Reader) map[string][]string { after = strings.TrimSpace(after) fields := strings.Fields(after) if len(fields) > 1 { - localZone = strings.Trim(fields[0], `""`) + localZone = strings.Trim(fields[0], `"`) } continue }