diff --git a/cmd/cli/change_logger.go b/cmd/cli/change_logger.go new file mode 100644 index 0000000..52601d8 --- /dev/null +++ b/cmd/cli/change_logger.go @@ -0,0 +1,205 @@ +package cli + +import ( + "maps" + "slices" + "sync" + "tailscale.com/net/netmon" + "time" +) + +// noiseSummaryInterval bounds a storm of noise deltas to one journal line, +// because AirDrop and virtual adapters can churn for hours. +const noiseSummaryInterval = 10 * time.Minute + +// wakeReportInterval merges the reports of the wake sources. netmon polls for +// a time jump every 15 s, so one wake can reach ctrld over half a minute. +const wakeReportInterval = 30 * time.Second + +// repeatLogger holds the last value of each named line, so a line that +// repeats every few seconds reaches the log on change only. +type repeatLogger struct { + mu sync.Mutex + last map[string]string + repeats map[string]uint64 +} + +// changed reports whether the value of a key differs from the value before it. +// The repeat count tells how many lines the caller held back, so the next line +// it writes can carry them. +func (r *repeatLogger) changed(key, value string) (changed bool, repeats uint64) { + r.mu.Lock() + defer r.mu.Unlock() + if r.last == nil { + r.last = make(map[string]string) + r.repeats = make(map[string]uint64) + } + if previous, seen := r.last[key]; seen && previous == value { + r.repeats[key]++ + return false, r.repeats[key] + } + repeats = r.repeats[key] + r.last[key] = value + r.repeats[key] = 0 + return true, repeats +} + +// wakeHoldInterval is the time that a wake report without a gap waits for the +// detector. The detector ticks every two seconds, so it reports well inside +// this hold when it runs. +const wakeHoldInterval = 8 * time.Second + +// wakeReport is one wake as a source saw it. +type wakeReport struct { + source string + gap time.Duration + state *netmon.State +} + +// wakeReporter keeps one report per wake. netmon measures no gap, so its +// report waits for the detector, which measures one. When no detector report +// comes, the held report goes out as it is. +type wakeReporter struct { + mu sync.Mutex + last time.Time + held *wakeReport + // after arms the hold timer. nil means time.AfterFunc, and a test fires + // the hold itself. + after func(time.Duration, func()) +} + +// note takes one report of a wake and hands the report that the journal gets +// to emit. The first source of a window reports. A report with a gap replaces +// a held report without one. Every other report of the window is the same +// wake and drops. +func (w *wakeReporter) note(now time.Time, report wakeReport, emit func(wakeReport)) { + w.mu.Lock() + defer w.mu.Unlock() + gapKnown := report.gap > 0 + if !w.last.IsZero() && now.Sub(w.last) < wakeReportInterval { + if w.held != nil && gapKnown { + w.held = nil + emit(report) + } + return + } + w.last = now + if gapKnown { + w.held = nil + emit(report) + return + } + held := &wakeReport{source: report.source, gap: report.gap, state: report.state} + w.held = held + w.afterFunc()(wakeHoldInterval, func() { w.release(held, emit) }) +} + +// release emits a held report when no source with a gap replaced it. +func (w *wakeReporter) release(held *wakeReport, emit func(wakeReport)) { + w.mu.Lock() + defer w.mu.Unlock() + if w.held != held { + return + } + w.held = nil + emit(*held) +} + +func (w *wakeReporter) afterFunc() func(time.Duration, func()) { + if w.after != nil { + return w.after + } + return func(d time.Duration, fn func()) { time.AfterFunc(d, fn) } +} + +// noiseSummary describes the noise deltas that one journal line covers. +type noiseSummary struct { + Count int + First time.Time + Last time.Time + Interfaces []string +} + +// noiseCoalescer counts the noise deltas of a storm and reports when the +// journal needs a line for them. The count, the first time, and the names +// cover the open window; last covers the storm, so a quiet period ends it. +type noiseCoalescer struct { + mu sync.Mutex + count int + first time.Time + last time.Time + emitted time.Time + names map[string]struct{} +} + +// add records one noise delta and reports whether to log a summary now. The +// summary covers the deltas since the last one, so a storm of any length +// keeps to one line per interval. +func (c *noiseCoalescer) add(interfaces []string, now time.Time) (summary noiseSummary, emit bool) { + c.mu.Lock() + defer c.mu.Unlock() + newRun := c.stormEnded(now) + if newRun { + c.clearWindow() + } + c.record(interfaces, now) + if !newRun && now.Sub(c.emitted) < noiseSummaryInterval { + return noiseSummary{}, false + } + summary = c.summary() + c.emitted = now + c.clearWindow() + return summary, true +} + +// flush reports the open window and closes it. A storm ends with a delta that +// the daemon acts on, so the last window of the storm reaches the journal +// there instead of waiting for the next storm. +func (c *noiseCoalescer) flush(now time.Time) (summary noiseSummary, emit bool) { + c.mu.Lock() + defer c.mu.Unlock() + if c.count == 0 { + return noiseSummary{}, false + } + summary = c.summary() + c.emitted = now + c.clearWindow() + return summary, true +} + +// stormEnded is true when the deltas stopped for longer than the interval, so +// a new storm does not inherit the count of the storm before it. +func (c *noiseCoalescer) stormEnded(now time.Time) bool { + return c.last.IsZero() || now.Sub(c.last) > noiseSummaryInterval +} + +func (c *noiseCoalescer) record(interfaces []string, now time.Time) { + if c.count == 0 { + c.first = now + } + c.count++ + c.last = now + if c.names == nil { + c.names = make(map[string]struct{}) + } + for _, name := range interfaces { + c.names[name] = struct{}{} + } +} + +// summary sorts the names, because a log line that keeps its order reads as +// one value across a capture. +func (c *noiseCoalescer) summary() noiseSummary { + return noiseSummary{ + Count: c.count, + First: c.first, + Last: c.last, + Interfaces: slices.Sorted(maps.Keys(c.names)), + } +} + +func (c *noiseCoalescer) clearWindow() { + c.count = 0 + c.first = time.Time{} + c.names = nil +} diff --git a/cmd/cli/change_logger_test.go b/cmd/cli/change_logger_test.go new file mode 100644 index 0000000..4292c4c --- /dev/null +++ b/cmd/cli/change_logger_test.go @@ -0,0 +1,198 @@ +package cli + +import ( + "slices" + "testing" + "time" +) + +func Test_repeatLogger(t *testing.T) { + var repeats repeatLogger + for _, step := range []struct { + name string + key string + value string + wantChanged bool + wantRepeats uint64 + }{ + {"the first value of a key is a change", "pf anchor", "intact", true, 0}, + {"the same value repeats", "pf anchor", "intact", false, 1}, + {"the same value repeats again", "pf anchor", "intact", false, 2}, + {"a new value reports the run before it", "pf anchor", "missing", true, 2}, + {"the new value repeats", "pf anchor", "missing", false, 1}, + {"the value before is a change again", "pf anchor", "intact", true, 1}, + {"a second key starts on its own", "tunnels", "utun0", true, 0}, + {"the second key repeats", "tunnels", "utun0", false, 1}, + {"the first key keeps its own count", "pf anchor", "intact", false, 1}, + } { + t.Run(step.name, func(t *testing.T) { + changed, count := repeats.changed(step.key, step.value) + if changed != step.wantChanged || count != step.wantRepeats { + t.Errorf("changed(%q, %q) = %v, %d, want %v, %d", + step.key, step.value, changed, count, step.wantChanged, step.wantRepeats) + } + }) + } +} + +func Test_wakeReporter(t *testing.T) { + base := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + var wake wakeReporter + var armed []func() + wake.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + var emitted []wakeReport + emit := func(report wakeReport) { emitted = append(emitted, report) } + fireHeld := func() { + for _, fn := range armed { + fn() + } + armed = nil + } + + // netmon reports first and knows no gap, so its report waits. + wake.note(base, wakeReport{source: "netmon"}, emit) + if len(emitted) != 0 || len(armed) != 1 { + t.Fatalf("a report without a gap must wait: emitted %d, timers %d", len(emitted), len(armed)) + } + // The detector reports inside the hold with the gap, so it replaces the + // held report. + wake.note(base.Add(2*time.Second), wakeReport{source: "detector", gap: 90 * time.Second}, emit) + fireHeld() + if len(emitted) != 1 || emitted[0].source != "detector" { + t.Fatalf("emitted %+v, want the detector report alone", emitted) + } + // Every other report of this wake is the same wake. + wake.note(base.Add(10*time.Second), wakeReport{source: "netmon"}, emit) + wake.note(base.Add(12*time.Second), wakeReport{source: "detector", gap: 91 * time.Second}, emit) + if len(emitted) != 1 { + t.Fatalf("emitted %d reports inside the window, want 1", len(emitted)) + } + + // A wake that only netmon sees reports without a gap when the hold ends. + wake.note(base.Add(40*time.Second), wakeReport{source: "netmon"}, emit) + if len(emitted) != 1 || len(armed) != 1 { + t.Fatalf("a new wake must arm one hold: emitted %d, timers %d", len(emitted), len(armed)) + } + fireHeld() + if len(emitted) != 2 || emitted[1].source != "netmon" || emitted[1].gap != 0 { + t.Fatalf("emitted %+v, want the netmon report after the hold", emitted) + } + // A gap that arrives after the hold is too late: one event per wake. + wake.note(base.Add(50*time.Second), wakeReport{source: "detector", gap: 92 * time.Second}, emit) + if len(emitted) != 2 { + t.Fatalf("emitted %d reports, want no second report of one wake", len(emitted)) + } + + // The detector first reports at once, and netmon adds nothing. + wake.note(base.Add(80*time.Second), wakeReport{source: "detector", gap: 93 * time.Second}, emit) + wake.note(base.Add(82*time.Second), wakeReport{source: "netmon"}, emit) + fireHeld() + if len(emitted) != 3 || emitted[2].source != "detector" || len(armed) != 0 { + t.Fatalf("emitted %+v with %d timers, want one detector report and no hold", emitted, len(armed)) + } +} + +func Test_noiseCoalescer(t *testing.T) { + base := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + + t.Run("a storm reports once every ten minutes", func(t *testing.T) { + const deltas = 100 + const cadence = 15 * time.Second + var noise noiseCoalescer + var got []noiseSummary + for i := 0; i < deltas; i++ { + interfaces := []string{"awdl0"} + if i%2 == 1 { + interfaces = []string{"llw0", "awdl0"} + } + summary, emit := noise.add(interfaces, base.Add(time.Duration(i)*cadence)) + if !emit { + continue + } + got = append(got, summary) + } + want := []noiseSummary{{ + Count: 1, + First: base, + Last: base, + Interfaces: []string{"awdl0"}, + }, { + Count: 40, + First: base.Add(cadence), + Last: base.Add(10 * time.Minute), + Interfaces: []string{"awdl0", "llw0"}, + }, { + Count: 40, + First: base.Add(10*time.Minute + cadence), + Last: base.Add(20 * time.Minute), + Interfaces: []string{"awdl0", "llw0"}, + }} + assertSummaries(t, got, want) + }) + + t.Run("a flush reports the open window", func(t *testing.T) { + var noise noiseCoalescer + if _, emit := noise.flush(base); emit { + t.Fatal("an empty coalescer reported a summary") + } + for _, offset := range []time.Duration{0, time.Minute, 2 * time.Minute} { + noise.add([]string{"awdl0"}, base.Add(offset)) + } + summary, emit := noise.flush(base.Add(3 * time.Minute)) + if !emit { + t.Fatal("the flush reported no summary for an open window") + } + assertSummaries(t, []noiseSummary{summary}, []noiseSummary{{ + Count: 2, + First: base.Add(time.Minute), + Last: base.Add(2 * time.Minute), + Interfaces: []string{"awdl0"}, + }}) + if _, emit = noise.flush(base.Add(4 * time.Minute)); emit { + t.Fatal("the flush reported the same window twice") + } + }) + + t.Run("a gap longer than the interval starts a new run", func(t *testing.T) { + var noise noiseCoalescer + var got []noiseSummary + for _, offset := range []time.Duration{0, time.Minute, 2 * time.Minute, 17 * time.Minute} { + summary, emit := noise.add([]string{"awdl0"}, base.Add(offset)) + if !emit { + continue + } + got = append(got, summary) + } + want := []noiseSummary{{ + Count: 1, + First: base, + Last: base, + Interfaces: []string{"awdl0"}, + }, { + Count: 1, + First: base.Add(17 * time.Minute), + Last: base.Add(17 * time.Minute), + Interfaces: []string{"awdl0"}, + }} + assertSummaries(t, got, want) + }) +} + +func assertSummaries(t *testing.T, got, want []noiseSummary) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("got %d summaries, want %d: %+v", len(got), len(want), got) + } + for i := range want { + if !sameSummary(got[i], want[i]) { + t.Errorf("summary %d = %+v, want %+v", i, got[i], want[i]) + } + } +} + +func sameSummary(got, want noiseSummary) bool { + return got.Count == want.Count && + got.First.Equal(want.First) && + got.Last.Equal(want.Last) && + slices.Equal(got.Interfaces, want.Interfaces) +} diff --git a/cmd/cli/dns64.go b/cmd/cli/dns64.go index 352a1b1..b8d3989 100644 --- a/cmd/cli/dns64.go +++ b/cmd/cli/dns64.go @@ -293,6 +293,15 @@ func (p *prog) activeDNS64Prefix() (netip.Prefix, bool) { return p.dns64.prefix, p.dns64.prefix.IsValid() } +// dns64PrefixSnapshot reports the stored NAT64 prefix and whether synthesis +// runs. It reads the state only, because a report of the network must not +// reclassify it or start a discovery query. +func (p *prog) dns64PrefixSnapshot() (netip.Prefix, bool) { + p.dns64.mu.Lock() + defer p.dns64.mu.Unlock() + return p.dns64.prefix, p.dns64.active +} + func dns64CacheVariant(prefix netip.Prefix) string { return "dns64:" + prefix.String() } @@ -398,13 +407,40 @@ func (p *prog) discoverNAT64Prefix(generation uint64) { } prefix, ok := nat64PrefixFromAnswer(answer) if !ok { - mainLog.Load().Debug().Msg("dns64: no NAT64 prefix present (not a DNS64 network)") + p.logNAT64Result("") return } if !p.storeDiscoveredNAT64Prefix(generation, prefix) { return } - mainLog.Load().Info().Msgf("dns64: discovered NAT64 prefix %s; enabling AAAA synthesis for IPv6-only network without CLAT", prefix) + p.logNAT64Result(prefix.String()) +} + +// logNAT64Result reports one discovery result. An empty prefix means that the +// network has none. Both results report a change of the network, so both go to +// the journal at info. +func (p *prog) logNAT64Result(prefix string) { + if !p.noteNAT64Result(prefix) { + return + } + if prefix == "" { + journal(mainLog.Load().Info()).Msg("dns64: no NAT64 prefix present (not a DNS64 network)") + return + } + journal(mainLog.Load().Info()).Msgf("dns64: discovered NAT64 prefix %s; enabling AAAA synthesis for IPv6-only network without CLAT", prefix) +} + +// noteNAT64Result reports whether this NAT64 result differs from the one that +// the journal already holds. An empty prefix means no prefix. Discovery runs +// every recheck window, and the journal keeps the changes only. +func (p *prog) noteNAT64Result(prefix string) bool { + p.lastNAT64Mu.Lock() + defer p.lastNAT64Mu.Unlock() + if p.lastNAT64Logged == prefix { + return false + } + p.lastNAT64Logged = prefix + return true } // maybeDNS64 applies DNS64 synthesis to an already-filtered answer when the diff --git a/cmd/cli/dns64_test.go b/cmd/cli/dns64_test.go index 292450e..3985aea 100644 --- a/cmd/cli/dns64_test.go +++ b/cmd/cli/dns64_test.go @@ -364,3 +364,61 @@ func TestMaybeDNS64DropsStaleInFlightPrefix(t *testing.T) { t.Fatal("in-flight synthesis used a prefix invalidated by a network change") } } + +// TestDNS64PrefixSnapshotReadsTheStoredStateOnly covers the read that a report +// of the network uses. The active read reclassifies the network and can start +// a discovery query, and a report must change nothing. +func TestDNS64PrefixSnapshotReadsTheStoredStateOnly(t *testing.T) { + classReads := 0 + old := dns64NetworkClassFn + dns64NetworkClassFn = func() (bool, bool, error) { + classReads++ + return false, false, nil + } + t.Cleanup(func() { dns64NetworkClassFn = old }) + + p := &prog{} + p.dns64.prefix = netip.MustParsePrefix("64:ff9b::/96") + p.dns64.active = true + + prefix, active := p.dns64PrefixSnapshot() + + if !active || prefix != p.dns64.prefix { + t.Fatalf("snapshot = %v, %v, want %v, true", prefix, active, p.dns64.prefix) + } + if classReads != 0 { + t.Fatalf("the snapshot classified the network %d times, want 0", classReads) + } + if p.dns64.discovering { + t.Fatal("the snapshot started a NAT64 prefix discovery") + } +} + +// TestDNS64ResultLinesReachTheJournalAtInfo covers the level of the two NAT64 +// result lines. Both report a change, and a debug line never leaves a host +// that runs at the normal level, so the journal lost the result of a network +// that has no prefix. +func TestDNS64ResultLinesReachTheJournalAtInfo(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + + // The first result of a run reports nothing when the network has no + // prefix, so the prefix has to arrive before it goes away. + p.logNAT64Result("64:ff9b::/96") + p.logNAT64Result("64:ff9b::/96") + p.logNAT64Result("") + + absent := jsonLogEvents(t, logs, "dns64: no NAT64 prefix present (not a DNS64 network)") + if len(absent) != 1 { + t.Fatalf("got %d no-prefix lines, want 1", len(absent)) + } + wantField(t, absent[0], "level", "info") + wantField(t, absent[0], "journal", true) + + found := jsonLogEvents(t, logs, "dns64: discovered NAT64 prefix 64:ff9b::/96; enabling AAAA synthesis for IPv6-only network without CLAT") + if len(found) != 1 { + t.Fatalf("got %d discovered lines for two identical results, want 1", len(found)) + } + wantField(t, found[0], "level", "info") + wantField(t, found[0], "journal", true) +} diff --git a/cmd/cli/dns_config.go b/cmd/cli/dns_config.go new file mode 100644 index 0000000..22c44ef --- /dev/null +++ b/cmd/cli/dns_config.go @@ -0,0 +1,360 @@ +package cli + +import ( + "bufio" + "errors" + "io" + "slices" + "strconv" + "strings" + "sync" + "time" +) + +const ( + // dnsConfigFastInterval is the poll period that follows a network event. + dnsConfigFastInterval = 15 * time.Second + + // dnsConfigSlowInterval is the poll period of a quiet host. + dnsConfigSlowInterval = 5 * time.Minute + + // dnsConfigFastWindow is how long the fast period lasts. A resolver change + // that follows a network event arrives late, after the OS settles. + dnsConfigFastWindow = 10 * time.Minute +) + +// scutilTablePrefix starts a resolver table. macOS prints the table for scoped +// queries after the main one, with the same line. +const scutilTablePrefix = "DNS configuration" + +// scutilResolverPrefix starts one resolver block inside a table. +const scutilResolverPrefix = "resolver #" + +// scutilDNSOutputLimit bounds the text of one read. A resolver table of a host +// is a few kilobytes, so a larger output does not come from a table. +const scutilDNSOutputLimit = 1 << 20 + +// errSCUtilDNSTooLarge ends a read that passed the bound. +var errSCUtilDNSTooLarge = errors.New("scutil --dns output is too large") + +// dnsResolverEntry is one resolver of the DNS configuration of the host. +type dnsResolverEntry struct { + Order int + Nameservers []string + IfIndex int + Interface string + Flags string + SearchDomains []string + Reachable string + // Scoped marks a resolver of the table for scoped queries, which serves one + // interface. A second service keeps its own resolver there while the + // unscoped table stays the same. + Scoped bool + // Action tells what the diff saw: added, changed, or removed. A resolver + // without nameservers looks the same whether it came or went, so the event + // needs the word. + Action string +} + +// The actions of a changed resolver. +const ( + resolverActionAdded = "added" + resolverActionChanged = "changed" + resolverActionRemoved = "removed" +) + +// parseSCUtilDNS reads the resolver tables of "scutil --dns": the unscoped +// table that answers a query of the host, then the table for scoped queries +// with one resolver per interface. Both tables count, because a second +// service can change its resolver while the unscoped table stays the same. +// +// A read that ends early or passes the bound gives an error and no table. A +// part of a table looks like resolvers that went away, and the caller must +// keep the table of the read before instead. +func parseSCUtilDNS(r io.Reader) ([]dnsResolverEntry, error) { + var entries []dnsResolverEntry + scoped := false + var entry *dnsResolverEntry + closeEntry := func() { + if entry == nil { + return + } + entry.Scoped = scoped + entries = append(entries, *entry) + entry = nil + } + read := 0 + scanner := bufio.NewScanner(io.LimitReader(r, scutilDNSOutputLimit+1)) + for scanner.Scan() { + read += len(scanner.Bytes()) + 1 + if read > scutilDNSOutputLimit { + return nil, errSCUtilDNSTooLarge + } + line := strings.TrimSpace(scanner.Text()) + switch { + case strings.HasPrefix(line, scutilTablePrefix): + closeEntry() + scoped = strings.Contains(line, "scoped") + case strings.HasPrefix(line, scutilResolverPrefix): + closeEntry() + entry = &dnsResolverEntry{} + case entry != nil: + applySCUtilField(entry, line) + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + closeEntry() + return entries, nil +} + +func applySCUtilField(entry *dnsResolverEntry, line string) { + name, value, ok := strings.Cut(line, ":") + if !ok { + return + } + name, value = strings.TrimSpace(name), strings.TrimSpace(value) + switch { + case strings.HasPrefix(name, "nameserver["): + entry.Nameservers = append(entry.Nameservers, value) + case strings.HasPrefix(name, "search domain["): + entry.SearchDomains = append(entry.SearchDomains, value) + case name == "if_index": + entry.IfIndex, entry.Interface = parseSCUtilIfIndex(value) + case name == "flags": + entry.Flags = value + case name == "reach": + entry.Reachable = scutilBracketText(value) + case name == "order": + entry.Order, _ = strconv.Atoi(value) + } +} + +// parseSCUtilIfIndex reads an "if_index" value of the form "14 (en0)". +func parseSCUtilIfIndex(value string) (int, string) { + number, name, _ := strings.Cut(value, " ") + index, _ := strconv.Atoi(number) + return index, scutilBracketText(name) +} + +// scutilBracketText returns the text in brackets, where scutil puts the +// readable form of a number. +func scutilBracketText(value string) string { + start := strings.Index(value, "(") + end := strings.LastIndex(value, ")") + if start < 0 || end < start { + return "" + } + return value[start+1 : end] +} + +// resolverKey identifies one resolver across two reads. An unscoped resolver +// has an order, and several can share it, so the count of earlier entries +// with that order completes the key. A scoped resolver has no order line and +// serves one interface, so its interface index is its identity. The two +// tables are two key spaces. +type resolverKey struct { + scoped bool + order int + ifIndex int + occurrence int +} + +func resolverKeys(entries []dnsResolverEntry) []resolverKey { + counts := make(map[resolverKey]int, len(entries)) + keys := make([]resolverKey, len(entries)) + for i, entry := range entries { + base := resolverKey{scoped: entry.Scoped, order: entry.Order} + if entry.Scoped { + base = resolverKey{scoped: true, ifIndex: entry.IfIndex} + } + key := base + key.occurrence = counts[base] + keys[i] = key + counts[base]++ + } + return keys +} + +func equalResolverEntry(a, b dnsResolverEntry) bool { + return a.IfIndex == b.IfIndex && + a.Interface == b.Interface && + a.Flags == b.Flags && + a.Reachable == b.Reachable && + slices.Equal(a.Nameservers, b.Nameservers) && + slices.Equal(a.SearchDomains, b.SearchDomains) +} + +// diffResolverTables returns the resolvers that differ between two reads. A +// resolver that vanished comes back without nameservers, so the journal shows +// that the host dropped it. +func diffResolverTables(before, after []dnsResolverEntry) []dnsResolverEntry { + beforeKeys, afterKeys := resolverKeys(before), resolverKeys(after) + previous := make(map[resolverKey]dnsResolverEntry, len(before)) + for i, key := range beforeKeys { + previous[key] = before[i] + } + current := make(map[resolverKey]struct{}, len(after)) + for _, key := range afterKeys { + current[key] = struct{}{} + } + var changed []dnsResolverEntry + for i, key := range afterKeys { + known, ok := previous[key] + if ok && equalResolverEntry(known, after[i]) { + continue + } + entry := after[i] + entry.Action = resolverActionChanged + if !ok { + entry.Action = resolverActionAdded + } + changed = append(changed, entry) + } + for i, key := range beforeKeys { + if _, ok := current[key]; ok { + continue + } + vanished := before[i] + vanished.Nameservers = nil + vanished.Action = resolverActionRemoved + changed = append(changed, vanished) + } + return changed +} + +// dnsConfigPoller reads the DNS configuration of the host on a timer and +// reports the resolvers that changed since the read before. +type dnsConfigPoller struct { + run func() ([]dnsResolverEntry, error) + now func() time.Time + mu sync.Mutex + last []dnsResolverEntry + lastActivity time.Time + started bool + readFailed bool + // kick wakes a loop that waits on the slow delay. One pending signal is + // enough, so a storm of events does not queue polls. + kick chan struct{} +} + +func newDNSConfigPoller(run func() ([]dnsResolverEntry, error)) *dnsConfigPoller { + return &dnsConfigPoller{run: run, now: time.Now, kick: make(chan struct{}, 1)} +} + +// noteActivity keeps the poll fast for a while after the network moved, and +// wakes a loop that waits on the slow delay. +func (p *dnsConfigPoller) noteActivity(now time.Time) { + p.markActivity(now) + select { + case p.kick <- struct{}{}: + default: + } +} + +// markActivity records the time of the last network event. +func (p *dnsConfigPoller) markActivity(now time.Time) { + p.mu.Lock() + defer p.mu.Unlock() + p.lastActivity = now +} + +func (p *dnsConfigPoller) nextDelay(now time.Time) time.Duration { + p.mu.Lock() + defer p.mu.Unlock() + if p.lastActivity.IsZero() || now.Sub(p.lastActivity) >= dnsConfigFastWindow { + return dnsConfigSlowInterval + } + return dnsConfigFastInterval +} + +// pollOnce returns the resolvers that changed. The first table is the baseline +// of the process and reports no change. A failed read keeps the table of the +// read before, so a short failure does not report the whole table twice. The +// first failure and the read that ends it log once, so a host whose read +// never works says so. +func (p *dnsConfigPoller) pollOnce(now time.Time) []dnsResolverEntry { + table, err := p.run() + p.mu.Lock() + defer p.mu.Unlock() + if err != nil { + if !p.readFailed { + mainLog.Load().Warn().Err(err).Msg("DNS configuration read failed; the journal keeps the last table") + } + p.readFailed = true + return nil + } + if p.readFailed { + mainLog.Load().Info().Msg("DNS configuration read works again") + p.readFailed = false + } + if !p.started { + p.started = true + p.last = table + return nil + } + changed := diffResolverTables(p.last, table) + p.last = table + return changed +} + +// dnsConfigDelayFn lets a test shorten the wait between two polls. +var dnsConfigDelayFn = (*dnsConfigPoller).nextDelay + +// startLoop runs the poll loop and returns a channel that closes when the loop +// returns. A caller joins that channel before it reads the log. +func (p *dnsConfigPoller) startLoop(stop <-chan struct{}, emit func([]dnsResolverEntry)) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) + p.loop(stop, emit) + }() + return done +} + +// loop polls until stop closes. The read starts a subprocess on macOS, so the +// delay follows the last activity and a quiet host pays little. The start of a +// run counts as activity, because the table of the host settles after the +// daemon comes up, and the read at entry is the baseline of the process. +func (p *dnsConfigPoller) loop(stop <-chan struct{}, emit func([]dnsResolverEntry)) { + now := p.now() + p.markActivity(now) + p.pollOnce(now) + delay := dnsConfigDelayFn(p, now) + deadline := now.Add(delay) + timer := time.NewTimer(delay) + defer timer.Stop() + for { + select { + case <-stop: + return + case <-p.kick: + // noteActivity set the time just now, so the delay is the fast one. + // A kick only moves a deadline closer; a storm of kicks must not + // push the read back again and again. + now := p.now() + wanted := now.Add(dnsConfigDelayFn(p, now)) + if !wanted.Before(deadline) { + continue + } + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timer.Reset(wanted.Sub(now)) + deadline = wanted + continue + case <-timer.C: + } + now := p.now() + if changed := p.pollOnce(now); len(changed) > 0 { + emit(changed) + } + delay := dnsConfigDelayFn(p, now) + deadline = now.Add(delay) + timer.Reset(delay) + } +} diff --git a/cmd/cli/dns_config_darwin.go b/cmd/cli/dns_config_darwin.go new file mode 100644 index 0000000..60d0c78 --- /dev/null +++ b/cmd/cli/dns_config_darwin.go @@ -0,0 +1,22 @@ +package cli + +import ( + "bytes" + "context" + "os/exec" + "time" +) + +// scutilDNSTimeout bounds the read. scutil talks to the configuration daemon, +// and that daemon stalls while the network moves. +const scutilDNSTimeout = 5 * time.Second + +func runSCUtilDNS() ([]dnsResolverEntry, error) { + ctx, cancel := context.WithTimeout(context.Background(), scutilDNSTimeout) + defer cancel() + out, err := exec.CommandContext(ctx, "scutil", "--dns").Output() + if err != nil { + return nil, err + } + return parseSCUtilDNS(bytes.NewReader(out)) +} diff --git a/cmd/cli/dns_config_others.go b/cmd/cli/dns_config_others.go new file mode 100644 index 0000000..9905dd5 --- /dev/null +++ b/cmd/cli/dns_config_others.go @@ -0,0 +1,9 @@ +//go:build !darwin + +package cli + +// runSCUtilDNS has no source of a resolver table outside macOS, so the poller +// finds no change and logs nothing. +func runSCUtilDNS() ([]dnsResolverEntry, error) { + return nil, nil +} diff --git a/cmd/cli/dns_config_test.go b/cmd/cli/dns_config_test.go new file mode 100644 index 0000000..b389ad0 --- /dev/null +++ b/cmd/cli/dns_config_test.go @@ -0,0 +1,598 @@ +package cli + +import ( + "errors" + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +const scutilDNSFixture = `DNS configuration + +resolver #1 + search domain[0] : corp.example + nameserver[0] : 192.168.50.200 + nameserver[1] : 192.168.50.1 + if_index : 16 (en0) + flags : Request A records, Request AAAA records + reach : 0x00020002 (Reachable,Directly Reachable Address) + order : 200000 + +resolver #2 + nameserver[0] : 100.100.100.100 + if_index : 28 (utun4) + flags : Supplemental, Request A records, Request AAAA records + reach : 0x00000003 (Reachable,Transient Connection) + order : 101600 + +resolver #3 + domain : local + options : mdns + timeout : 5 + flags : Request A records + reach : 0x00000000 (Not Reachable) + order : 300000 + +DNS configuration (for scoped queries) + +resolver #1 + nameserver[0] : 192.168.50.200 + if_index : 16 (en0) + flags : Scoped, Request A records + reach : 0x00020002 (Reachable,Directly Reachable Address) +` + +const scutilDNSScopedOnlyFixture = `DNS configuration + +DNS configuration (for scoped queries) + +resolver #1 + nameserver[0] : 192.168.50.200 + if_index : 16 (en0) + flags : Scoped, Request A records + reach : 0x00020002 (Reachable,Directly Reachable Address) +` + +var errStubResolverRun = errors.New("scutil failed") + +func wifiResolverEntry() dnsResolverEntry { + return dnsResolverEntry{ + Order: 200000, + Nameservers: []string{"192.168.50.200", "192.168.50.1"}, + IfIndex: 16, + Interface: "en0", + Flags: "Request A records, Request AAAA records", + SearchDomains: []string{"corp.example"}, + Reachable: "Reachable,Directly Reachable Address", + } +} + +func vpnResolverEntry() dnsResolverEntry { + return dnsResolverEntry{ + Order: 101600, + Nameservers: []string{"100.100.100.100"}, + IfIndex: 28, + Interface: "utun4", + Flags: "Supplemental, Request A records, Request AAAA records", + Reachable: "Reachable,Transient Connection", + } +} + +func mdnsResolverEntry() dnsResolverEntry { + return dnsResolverEntry{ + Order: 300000, + Flags: "Request A records", + Reachable: "Not Reachable", + } +} + +func Test_parseSCUtilDNS(t *testing.T) { + tests := []struct { + name string + input string + want []dnsResolverEntry + }{ + { + name: "both tables", + input: scutilDNSFixture, + want: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry(), mdnsResolverEntry(), { + Nameservers: []string{"192.168.50.200"}, + IfIndex: 16, + Interface: "en0", + Flags: "Scoped, Request A records", + Reachable: "Reachable,Directly Reachable Address", + Scoped: true, + }}, + }, + { + name: "scoped table when the first is empty", + input: scutilDNSScopedOnlyFixture, + want: []dnsResolverEntry{{ + Nameservers: []string{"192.168.50.200"}, + IfIndex: 16, + Interface: "en0", + Flags: "Scoped, Request A records", + Reachable: "Reachable,Directly Reachable Address", + Scoped: true, + }}, + }, + { + name: "no output", + input: "", + want: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := parseSCUtilDNS(strings.NewReader(tc.input)) + if err != nil { + t.Fatalf("parseSCUtilDNS() error = %v", err) + } + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("parseSCUtilDNS() = %+v, want %+v", got, tc.want) + } + }) + } +} + +func Test_diffResolverTables(t *testing.T) { + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + vanishedVPN := vpnResolverEntry() + vanishedVPN.Nameservers = nil + vanishedVPN.Action = resolverActionRemoved + appearedVPN := vpnResolverEntry() + appearedVPN.Action = resolverActionAdded + + tests := []struct { + name string + before []dnsResolverEntry + after []dnsResolverEntry + want []dnsResolverEntry + }{ + { + name: "no change", + before: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + after: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + want: nil, + }, + { + name: "one resolver changed", + before: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + after: []dnsResolverEntry{changedWiFi, vpnResolverEntry()}, + want: []dnsResolverEntry{changedWiFi}, + }, + { + name: "one resolver vanished", + before: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + after: []dnsResolverEntry{wifiResolverEntry()}, + want: []dnsResolverEntry{vanishedVPN}, + }, + { + name: "one resolver appeared", + before: []dnsResolverEntry{wifiResolverEntry()}, + after: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + want: []dnsResolverEntry{appearedVPN}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := diffResolverTables(tc.before, tc.after) + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("diffResolverTables() = %+v, want %+v", got, tc.want) + } + }) + } +} + +func Test_diffResolverTablesKeepsTheInput(t *testing.T) { + before := []dnsResolverEntry{vpnResolverEntry()} + diffResolverTables(before, nil) + if !reflect.DeepEqual(before[0], vpnResolverEntry()) { + t.Fatalf("before entry = %+v, want %+v", before[0], vpnResolverEntry()) + } +} + +// resolverRunResult is one answer of the stub runner. +type resolverRunResult struct { + table []dnsResolverEntry + err error +} + +// stubResolverRunner answers one result per call and repeats the last one. +type stubResolverRunner struct { + mu sync.Mutex + results []resolverRunResult + calls int +} + +func (s *stubResolverRunner) run() ([]dnsResolverEntry, error) { + s.mu.Lock() + defer s.mu.Unlock() + result := s.results[min(s.calls, len(s.results)-1)] + s.calls++ + return result.table, result.err +} + +func Test_dnsConfigPollerNextDelay(t *testing.T) { + base := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + tests := []struct { + name string + activity bool + elapsed time.Duration + want time.Duration + }{ + {name: "nine minutes after activity", activity: true, elapsed: 9 * time.Minute, want: dnsConfigFastInterval}, + {name: "eleven minutes after activity", activity: true, elapsed: 11 * time.Minute, want: dnsConfigSlowInterval}, + {name: "no activity yet", elapsed: time.Minute, want: dnsConfigSlowInterval}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + poller := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil }) + if tc.activity { + poller.noteActivity(base) + } + if got := poller.nextDelay(base.Add(tc.elapsed)); got != tc.want { + t.Fatalf("nextDelay() = %s, want %s", got, tc.want) + } + }) + } +} + +func Test_dnsConfigPollerPollOnce(t *testing.T) { + now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + baseline := []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()} + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + runner := &stubResolverRunner{results: []resolverRunResult{ + {table: baseline}, + {err: errStubResolverRun}, + {table: baseline}, + {table: []dnsResolverEntry{changedWiFi, vpnResolverEntry()}}, + }} + poller := newDNSConfigPoller(runner.run) + + if got := poller.pollOnce(now); got != nil { + t.Fatalf("baseline poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(time.Minute)); got != nil { + t.Fatalf("failed poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(2 * time.Minute)); got != nil { + t.Fatalf("poll after a failure = %+v, want nil", got) + } + want := []dnsResolverEntry{changedWiFi} + if got := poller.pollOnce(now.Add(3 * time.Minute)); !reflect.DeepEqual(got, want) { + t.Fatalf("changed poll = %+v, want %+v", got, want) + } +} + +func Test_dnsConfigPollerLoop(t *testing.T) { + previous := dnsConfigDelayFn + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return time.Millisecond } + t.Cleanup(func() { dnsConfigDelayFn = previous }) + + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + runner := &stubResolverRunner{results: []resolverRunResult{ + {table: []dnsResolverEntry{wifiResolverEntry()}}, + {table: []dnsResolverEntry{changedWiFi}}, + }} + poller := newDNSConfigPoller(runner.run) + + emitted := make(chan []dnsResolverEntry, 4) + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + poller.loop(stop, func(changed []dnsResolverEntry) { emitted <- changed }) + }() + + select { + case got := <-emitted: + want := []dnsResolverEntry{changedWiFi} + if !reflect.DeepEqual(got, want) { + t.Fatalf("emitted = %+v, want %+v", got, want) + } + case <-time.After(5 * time.Second): + t.Fatal("the loop emitted no change") + } + + close(stop) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the loop did not stop") + } +} + +// Test_dnsConfigPollerLoopWakesOnActivity proves that a network event does not +// wait for a sleeping timer. The delay is one hour until the event arrives, so +// a loop without the kick would poll after the test deadline. +func Test_dnsConfigPollerLoopWakesOnActivity(t *testing.T) { + var kicked atomic.Bool + parked := make(chan struct{}, 1) + previous := dnsConfigDelayFn + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { + if kicked.Load() { + return time.Millisecond + } + select { + case parked <- struct{}{}: + default: + } + return time.Hour + } + t.Cleanup(func() { dnsConfigDelayFn = previous }) + + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + runner := &stubResolverRunner{results: []resolverRunResult{ + {table: []dnsResolverEntry{wifiResolverEntry()}}, + {table: []dnsResolverEntry{changedWiFi}}, + }} + poller := newDNSConfigPoller(runner.run) + + emitted := make(chan []dnsResolverEntry, 4) + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + poller.loop(stop, func(changed []dnsResolverEntry) { emitted <- changed }) + }() + t.Cleanup(func() { + close(stop) + <-done + }) + + // The kick matters only once the loop waits on the long delay. + select { + case <-parked: + case <-time.After(5 * time.Second): + t.Fatal("the loop did not park on the long delay") + } + kicked.Store(true) + poller.noteActivity(time.Now()) + + select { + case got := <-emitted: + want := []dnsResolverEntry{changedWiFi} + if !reflect.DeepEqual(got, want) { + t.Fatalf("emitted = %+v, want %+v", got, want) + } + case <-time.After(5 * time.Second): + t.Fatal("the loop polled no table after the activity") + } +} + +// Test_dnsConfigPollerLoopTakesABaseline proves that the loop reads the table +// once at entry. Without that read the first change after the start reports +// the whole table. +func Test_dnsConfigPollerLoopTakesABaseline(t *testing.T) { + previous := dnsConfigDelayFn + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return time.Hour } + t.Cleanup(func() { dnsConfigDelayFn = previous }) + + runner := &stubResolverRunner{results: []resolverRunResult{{table: []dnsResolverEntry{wifiResolverEntry()}}}} + poller := newDNSConfigPoller(runner.run) + + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + poller.loop(stop, func([]dnsResolverEntry) { t.Error("the baseline read reported a change") }) + }() + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + runner.mu.Lock() + calls := runner.calls + runner.mu.Unlock() + if calls > 0 { + break + } + time.Sleep(time.Millisecond) + } + close(stop) + <-done + + runner.mu.Lock() + defer runner.mu.Unlock() + if runner.calls != 1 { + t.Fatalf("reads at the loop entry: got %d, want 1", runner.calls) + } + if !poller.started { + t.Fatal("the loop took no baseline") + } +} + +// oversizedSCUtilDNS is a table with one line that no scanner buffer holds. A +// stalled or hostile process produces output of this shape. +func oversizedSCUtilDNS() string { + return "DNS configuration\n\nresolver #1\n nameserver[0] : " + strings.Repeat("9", 70*1024) + "\n" +} + +// Test_parseSCUtilDNSRejectsAnOversizedLine proves that a line the scanner +// cannot hold ends the read with an error. A part of a table must not pass for +// the table of the host. +func Test_parseSCUtilDNSRejectsAnOversizedLine(t *testing.T) { + got, err := parseSCUtilDNS(strings.NewReader(oversizedSCUtilDNS())) + if err == nil { + t.Fatal("parseSCUtilDNS accepted a 70 KiB line") + } + if got != nil { + t.Fatalf("parseSCUtilDNS() = %+v, want nil", got) + } +} + +// Test_parseSCUtilDNSRejectsAnOversizedOutput proves that output past the +// bound ends the read with an error instead of a table cut in the middle. +func Test_parseSCUtilDNSRejectsAnOversizedOutput(t *testing.T) { + block := "resolver #1\n nameserver[0] : 192.168.50.200\n order : 200000\n" + repeats := scutilDNSOutputLimit/len(block) + 2 + got, err := parseSCUtilDNS(strings.NewReader("DNS configuration\n\n" + strings.Repeat(block, repeats))) + if err == nil { + t.Fatal("parseSCUtilDNS accepted output past the bound") + } + if got != nil { + t.Fatalf("parseSCUtilDNS() = %+v, want nil", got) + } +} + +// Test_dnsConfigPollerKeepsTheTableOnAnOversizedRead proves that a read the +// parser rejects reports nothing and keeps the table of the read before. +func Test_dnsConfigPollerKeepsTheTableOnAnOversizedRead(t *testing.T) { + now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + outputs := []string{scutilDNSFixture, oversizedSCUtilDNS(), scutilDNSFixture} + calls := 0 + poller := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { + output := outputs[min(calls, len(outputs)-1)] + calls++ + return parseSCUtilDNS(strings.NewReader(output)) + }) + + if got := poller.pollOnce(now); got != nil { + t.Fatalf("baseline poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(time.Minute)); got != nil { + t.Fatalf("oversized poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(2 * time.Minute)); got != nil { + t.Fatalf("poll after the oversized read = %+v, want nil", got) + } +} + +// scutilTwoTables renders a host with one unscoped resolver and one scoped +// resolver, the shape of a Mac with two active services. +func scutilTwoTables(unscoped, scoped string) string { + return "DNS configuration\n\nresolver #1\n nameserver[0] : " + unscoped + "\n if_index : 14 (en0)\n" + + " flags : Request A records\n reach : 0x00020002 (Reachable,Directly Reachable Address)\n\n" + + "DNS configuration (for scoped queries)\n\nresolver #1\n nameserver[0] : " + scoped + "\n if_index : 15 (en1)\n" + + " flags : Scoped, Request A records\n reach : 0x00020002 (Reachable,Directly Reachable Address)\n" +} + +// Test_diffResolverTablesSeesAScopedChange changes the scoped resolver of a +// second service while the unscoped resolver stays. The diff must report the +// scoped change, because the two tables are not copies of each other. +func Test_diffResolverTablesSeesAScopedChange(t *testing.T) { + before, err := parseSCUtilDNS(strings.NewReader(scutilTwoTables("192.168.1.1", "10.0.0.1"))) + if err != nil { + t.Fatal(err) + } + after, err := parseSCUtilDNS(strings.NewReader(scutilTwoTables("192.168.1.1", "9.9.9.9"))) + if err != nil { + t.Fatal(err) + } + changed := diffResolverTables(before, after) + if len(changed) != 1 || len(changed[0].Nameservers) != 1 || changed[0].Nameservers[0] != "9.9.9.9" { + t.Fatalf("changed resolvers = %+v, want the scoped resolver with 9.9.9.9", changed) + } + if !changed[0].Scoped || before[0].Scoped || len(before) != 2 { + t.Fatalf("scope marks: changed %v, before %+v; want the scoped resolver marked and the unscoped one not", changed[0].Scoped, before) + } +} + +// Test_dnsConfigPollerPollsUnderContinuousActivity kicks the poller faster +// than its fast interval. A kick must not push the read back again and again, +// or a host in a storm of transitions never reports a resolver change. +func Test_dnsConfigPollerPollsUnderContinuousActivity(t *testing.T) { + var reads atomic.Int64 + changed := wifiResolverEntry() + changed.Nameservers = []string{"9.9.9.9"} + p := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { + if reads.Add(1) == 1 { + return []dnsResolverEntry{wifiResolverEntry()}, nil + } + return []dnsResolverEntry{changed}, nil + }) + origDelay := dnsConfigDelayFn + t.Cleanup(func() { dnsConfigDelayFn = origDelay }) + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return 50 * time.Millisecond } + + var emits atomic.Int64 + stop := make(chan struct{}) + done := p.startLoop(stop, func([]dnsResolverEntry) { emits.Add(1) }) + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + p.noteActivity(time.Now()) + time.Sleep(5 * time.Millisecond) + } + close(stop) + <-done + + if emits.Load() == 0 { + t.Fatalf("no resolver event during 500 ms of activity (reads %d)", reads.Load()) + } +} + +// scutilScopedTable renders the scoped table the way macOS prints it: one +// resolver per interface and no order line. +func scutilScopedTable(resolvers ...[2]string) string { + var b strings.Builder + b.WriteString("DNS configuration\n\nresolver #1\n nameserver[0] : 100.100.100.100\n if_index : 28 (utun4)\n flags : Request A records, Request AAAA records\n reach : 0x00000003 (Reachable,Transient Connection)\n order : 200000\n\n") + b.WriteString("DNS configuration (for scoped queries)\n\n") + for i, resolver := range resolvers { + fmt.Fprintf(&b, "resolver #%d\n nameserver[0] : %s\n if_index : %s\n flags : Scoped, Request A records\n reach : 0x00020002 (Reachable,Directly Reachable Address)\n\n", i+1, resolver[1], resolver[0]) + } + return b.String() +} + +// Test_diffResolverTablesKeysScopedResolversByInterface inserts a scoped +// resolver between two others and removes it again. The scoped table has no +// order line, so the interface is the identity, and the diff must name the +// resolver that came and went, not its neighbors. +func Test_diffResolverTablesKeysScopedResolversByInterface(t *testing.T) { + en0 := [2]string{"16 (en0)", "192.168.50.200"} + en7 := [2]string{"30 (en7)", "10.9.9.1"} + utun := [2]string{"28 (utun4)", "100.100.100.100"} + before, err := parseSCUtilDNS(strings.NewReader(scutilScopedTable(en0, utun))) + if err != nil { + t.Fatal(err) + } + after, err := parseSCUtilDNS(strings.NewReader(scutilScopedTable(en0, en7, utun))) + if err != nil { + t.Fatal(err) + } + + inserted := diffResolverTables(before, after) + if len(inserted) != 1 || inserted[0].Interface != "en7" || inserted[0].Nameservers == nil { + t.Fatalf("insert of en7 reported %+v, want en7 alone with its nameserver", inserted) + } + removed := diffResolverTables(after, before) + if len(removed) != 1 || removed[0].Interface != "en7" || removed[0].Nameservers != nil { + t.Fatalf("removal of en7 reported %+v, want en7 alone without nameservers", removed) + } +} + +// Test_diffResolverTablesNamesTheAction tells a resolver that came from one +// that went. An mDNS resolver has no nameservers, so without the action the +// two events read the same. +func Test_diffResolverTablesNamesTheAction(t *testing.T) { + mdns := "DNS configuration\n\nresolver #1\n domain : local\n options : mdns\n timeout : 5\n flags : Request A records, Request AAAA records\n reach : 0x00000000 (Not Reachable)\n order : 300000\n" + table, err := parseSCUtilDNS(strings.NewReader(mdns)) + if err != nil { + t.Fatal(err) + } + appeared := diffResolverTables(nil, table) + vanished := diffResolverTables(table, nil) + if len(appeared) != 1 || appeared[0].Action != resolverActionAdded { + t.Fatalf("appeared = %+v, want one resolver with action added", appeared) + } + if len(vanished) != 1 || vanished[0].Action != resolverActionRemoved { + t.Fatalf("vanished = %+v, want one resolver with action removed", vanished) + } + moved := wifiResolverEntry() + moved.Nameservers = []string{"9.9.9.9"} + changed := diffResolverTables([]dnsResolverEntry{wifiResolverEntry()}, []dnsResolverEntry{moved}) + if len(changed) != 1 || changed[0].Action != resolverActionChanged { + t.Fatalf("changed = %+v, want one resolver with action changed", changed) + } +} diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go index ab72492..7336b84 100644 --- a/cmd/cli/dns_intercept_darwin.go +++ b/cmd/cli/dns_intercept_darwin.go @@ -17,6 +17,8 @@ import ( "syscall" "time" + "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" ) @@ -221,8 +223,9 @@ func (p *prog) startDNSIntercept() error { } // Store the initial set of tunnel interfaces so we can detect changes later. + tunnels := p.activeTunnelInterfaces() p.mu.Lock() - p.lastTunnelIfaces = discoverTunnelInterfacesForReconcile() + p.lastTunnelIfaces = tunnels p.pendingTunnelIfaces = nil p.hasPendingTunnelIfaces = false p.mu.Unlock() @@ -723,14 +726,29 @@ func discoverTunnelInterfaces() []string { } } - if len(tunnels) > 0 { - mainLog.Load().Debug().Msgf("DNS intercept: discovered active tunnel interfaces: %v", tunnels) - } return tunnels } var discoverTunnelInterfacesForReconcile = discoverTunnelInterfaces +// tunnelInterfacesKey holds the last discovered tunnel set, so the discovery +// line reports a change and not every reconcile. +const tunnelInterfacesKey = "tunnel_interfaces" + +// activeTunnelInterfaces returns the live tunnel interfaces and names them on +// change. Every pf reconcile discovers them again, and the set is the same in +// almost every one of those reads. +func (p *prog) activeTunnelInterfaces() []string { + tunnels := discoverTunnelInterfacesForReconcile() + changed, repeats := p.repeats.changed(tunnelInterfacesKey, fmt.Sprintf("%v", tunnels)) + if !changed || len(tunnels) == 0 { + return tunnels + } + mainLog.Load().Debug().Uint64("repeats", repeats). + Msgf("DNS intercept: discovered active tunnel interfaces: %v", tunnels) + return tunnels +} + // dnsInterceptSupported reports whether DNS intercept mode is supported on this platform. func dnsInterceptSupported() bool { _, err := exec.LookPath("pfctl") @@ -787,7 +805,7 @@ func (p *prog) validateDNSIntercept() error { // // pf requires strict rule ordering: translation (rdr) BEFORE filtering (pass). func (p *prog) buildPFAnchorRules(vpnExemptions []vpnDNSExemption) string { - return p.buildPFAnchorRulesForTunnels(vpnExemptions, discoverTunnelInterfacesForReconcile()) + return p.buildPFAnchorRulesForTunnels(vpnExemptions, p.activeTunnelInterfaces()) } func (p *prog) buildPFAnchorRulesForTunnels(vpnExemptions []vpnDNSExemption, tunnelIfaces []string) string { @@ -1169,11 +1187,12 @@ var ( restorePFAnchorForReconcile = func(p *prog, reason string) pfAnchorCheckResult { return p.restorePFAnchor(reason) } + verifyPFStateFn = (*prog).verifyPFState pfShutdownStateRevokedForTest = func() {} ) func (p *prog) rebuildPFAnchorRules(vpnExemptions []vpnDNSExemption) ([]string, error) { - tunnelIfaces := append([]string(nil), discoverTunnelInterfacesForReconcile()...) + tunnelIfaces := append([]string(nil), p.activeTunnelInterfaces()...) rulesStr := p.buildPFAnchorRulesForTunnels(vpnExemptions, tunnelIfaces) if err := os.WriteFile(pfAnchorFile, []byte(rulesStr), 0644); err != nil { return nil, fmt.Errorf("write anchor file: %w", err) @@ -1224,7 +1243,7 @@ func (p *prog) restorePFAnchorWithTransportReset(reason string, resetTransports if resetTransports { p.resetUpstreamTransports() } - if !p.verifyPFState() { + if !verifyPFStateFn(p) { mainLog.Load().Error().Str("reason", reason).Msg("DNS intercept: rebuilt anchor failed post-load verification") return pfAnchorCheckFailed } @@ -1234,6 +1253,7 @@ func (p *prog) restorePFAnchorWithTransportReset(reason string, resetTransports p.commitPFReconcileState(tunnelIfaces) p.pfLastRestoreTime.Store(time.Now().UnixMilli()) mainLog.Load().Info().Str("reason", reason).Msg("DNS intercept: pf anchor restored successfully") + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{}) return pfAnchorCheckRestored } @@ -1284,7 +1304,8 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return false } - current := append([]string(nil), discoverTunnelInterfacesForReconcile()...) + current := append([]string(nil), p.activeTunnelInterfaces()...) + p.noteTunnelInterfaceSet(current) p.mu.Lock() prev := append([]string(nil), p.lastTunnelIfaces...) @@ -1319,18 +1340,9 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return true } - // Detect NEW tunnel interfaces (not just any change). - prevSet := make(map[string]bool, len(prev)) - for _, iface := range prev { - prevSet[iface] = true - } - hasNewTunnel := false - for _, iface := range current { - if !prevSet[iface] { - hasNewTunnel = true - mainLog.Load().Info().Msgf("DNS intercept: new tunnel interface detected: %s", iface) - break - } + added, _ := tunnelSetDiff(prev, current) + if len(added) > 0 { + mainLog.Load().Info().Msgf("DNS intercept: new tunnel interface detected: %s", added[0]) } if stabilizing { @@ -1338,14 +1350,13 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return newObservation } - if hasNewTunnel { + if len(added) > 0 { // A new VPN tunnel appeared. The dedicated post-stabilization repair path // rebuilds the anchor if no successful exemption update applies it first. p.pfStartStabilization() return newObservation } - mainLog.Load().Info().Msgf("DNS intercept: tunnel interfaces changed (was %v, now %v) — rebuilding pf anchor rules", prev, current) result := p.reconcilePFAnchorForTunnelChange() if result != pfAnchorCheckRestored { // Keep the desired tunnel set pending. The next event, delayed check, or @@ -1355,6 +1366,70 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return newObservation || result == pfAnchorCheckRestored } +// tunnelChangedMessage names the tunnel event of the journal. +const tunnelChangedMessage = "Tunnel interface changed" + +// noteTunnelInterfaceSet logs one event for each change of the tunnel set. The +// reconcile retries a set until it succeeds, and a set that returns to the one +// before it is news as well, so the diff runs against the set of the last +// event and not against the applied set. +func (p *prog) noteTunnelInterfaceSet(current []string) { + p.mu.Lock() + // Before the first event the applied set is the baseline, because no event + // named a set yet. + previous := p.lastLoggedTunnelIfaces + if previous == nil { + previous = p.lastTunnelIfaces + } + previous = append([]string(nil), previous...) + if stringSlicesEqual(previous, current) { + p.mu.Unlock() + return + } + p.lastLoggedTunnelIfaces = append([]string{}, current...) + p.mu.Unlock() + + added, removed := tunnelSetDiff(previous, current) + p.logTunnelInterfaceChange(added, removed) +} + +// logTunnelInterfaceChange names the tunnels that appeared and the ones that +// went away. The owner names the program behind a new tunnel, because its pf +// rules compete with the ctrld anchor for the same DNS packets. +func (p *prog) logTunnelInterfaceChange(added, removed []string) { + owner := "" + if len(added) > 0 { + owner = tunnelOwner(added[0]) + } + journal(mainLog.Load().Info()). + Strs("added", added). + Strs("removed", removed). + Str("owner", owner). + Msg(tunnelChangedMessage) +} + +// tunnelSetDiff reports the tunnels that appeared and the ones that went away, +// each in the order of the set it comes from. +func tunnelSetDiff(prev, current []string) (added, removed []string) { + before := make(map[string]bool, len(prev)) + for _, iface := range prev { + before[iface] = true + } + live := make(map[string]bool, len(current)) + for _, iface := range current { + live[iface] = true + if !before[iface] { + added = append(added, iface) + } + } + for _, iface := range prev { + if !live[iface] { + removed = append(removed, iface) + } + } + return added, removed +} + // stringSlicesEqual reports whether two string slices have the same elements in the same order. func stringSlicesEqual(a, b []string) bool { if len(a) != len(b) { @@ -1383,7 +1458,8 @@ func (p *prog) pfStartStabilization() { stableRequired = 45 * time.Second } - mainLog.Load().Info().Msgf("DNS intercept: VPN connecting — entering stabilization mode (waiting %s for pf to settle)", stableRequired) + journal(mainLog.Load().Info()).Msgf("DNS intercept: VPN connecting — entering stabilization mode (waiting %s for pf to settle)", stableRequired) + p.logPFAnchorList(pfAnchorReasonStabilizationStart, mainLog.Load().Info(), pfRuleDump{}) ctx, cancel := context.WithCancel(context.Background()) p.mu.Lock() @@ -1463,7 +1539,8 @@ func (p *prog) pfStabilizationLoopWithMaxWait(ctx context.Context, stableRequire // held still for the required window. The active loop retains ownership throughout: it // never re-enters stabilization recursively. func (p *prog) finishPFStabilization(stableRequired time.Duration) { - mainLog.Load().Info().Msgf("DNS intercept: pf stable for %s — reconciling anchor rules", stableRequired) + journal(mainLog.Load().Info()).Msgf("DNS intercept: pf stable for %s — reconciling anchor rules", stableRequired) + p.logPFAnchorList(pfAnchorReasonStabilizationEnd, mainLog.Load().Info(), pfRuleDump{}) result := p.reconcilePFAnchorAfterStabilization() if result != pfAnchorCheckRestored && result != pfAnchorCheckIntact { p.scheduleDelayedRechecks() @@ -1630,6 +1707,11 @@ func (p *prog) verifyInterceptAfterWake(gap time.Duration) { return } mainLog.Load().Info().Msgf("DNS intercept: host resumed after a %s gap - verifying interception is still translating", gap.Round(time.Second)) + p.noteHostWoke("detector", gap, freshNetworkState()) + // A resume moves the resolvers of the host, so the poll goes fast again. + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(networkEventsNowFn()) + } repairs := 0 probed := 0 @@ -1702,7 +1784,7 @@ func (p *prog) waitBeforeWakeProbe(delay time.Duration) bool { // could not run does not count: nothing was changed, so the next attempt may try again. // The caller holds functional-probe ownership for the call. func (p *prog) runWakeProbeAttempt(attempt, repairs int) (restored, repaired bool) { - initializeOsResolver(true) + initializeOsResolver(true, osResolverReasonWakeProbe) result := probePFInterceptFn(p) if result.result == pfProbeIndeterminate { mainLog.Load().Debug().Msg("DNS intercept: post-wake probe indeterminate; repair deferred") @@ -1743,6 +1825,77 @@ var runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { return exec.Command("pfctl", args...).CombinedOutput() } +const ( + // pfAnchorListMessage names the anchor state event of the journal. + pfAnchorListMessage = "PF anchor list changed" + + pfAnchorReasonStabilizationStart = "stabilization_start" + pfAnchorReasonStabilizationEnd = "stabilization_end" + pfAnchorReasonMissing = "missing" + pfAnchorReasonRestored = "restored" + + // pfAnchorStateKey holds the last anchor state, so an anchor that stays + // missing for a whole storm reaches the journal once. + pfAnchorStateKey = "pf_anchor_state" + + // pfAnchorWatchdogKey holds the last state the watchdog saw, so the healthy + // line reports the recovery and not every tick between two of them. + pfAnchorWatchdogKey = "pf_anchor_intact" + + pfAnchorIntactState = "pf anchor intact" + pfAnchorMissingState = "pf anchor missing" + + pfAnchorIntactMessage = "DNS intercept watchdog: " + pfAnchorIntactState +) + +// logPFAnchorList reports which anchors the running ruleset refers to. A capture +// of an outage has to tell whether ctrld or another program owned pf, and the +// anchor names are the only evidence of that. +func (p *prog) logPFAnchorList(reason string, level *zerolog.Event, dump pfRuleDump) { + anchors := p.pfAnchorNames(dump) + enabled, since := p.pfStatus() + if changed, _ := p.repeats.changed(pfAnchorStateKey, pfAnchorStateValue(reason, anchors, enabled)); !changed { + return + } + event := journal(level). + Str("reason", reason). + Bool("pf_enabled", enabled). + Str("pf_since", since) + if anchors == nil { + event.Bool("anchors_known", false).Msg(pfAnchorListMessage) + return + } + event.Strs("anchors", anchors).Msg(pfAnchorListMessage) +} + +// pfAnchorStateValue leaves the pf uptime out, because it grows between two +// reads of the same state. +func pfAnchorStateValue(reason string, anchors []string, enabled bool) string { + if anchors == nil { + return fmt.Sprintf("%s|%t|unknown", reason, enabled) + } + return fmt.Sprintf("%s|%t|%s", reason, enabled, strings.Join(anchors, ",")) +} + +// logPFAnchorIntact reports a healthy anchor. The watchdog finds it healthy on +// almost every tick for the life of the daemon, so only a change belongs in the +// log. +func (p *prog) logPFAnchorIntact() { + changed, repeats := p.repeats.changed(pfAnchorWatchdogKey, pfAnchorIntactState) + if !changed { + return + } + mainLog.Load().Debug().Uint64("repeats", repeats).Msg(pfAnchorIntactMessage) +} + +// notePFAnchorMissing reports the wiped anchor with the anchors that replaced +// it. The record of the missing state makes the next healthy check reach the +// log, so a capture holds both ends of the outage. +func (p *prog) notePFAnchorMissing(dump pfRuleDump) { + p.repeats.changed(pfAnchorWatchdogKey, pfAnchorMissingState) + p.logPFAnchorList(pfAnchorReasonMissing, mainLog.Load().Warn(), dump) +} + // ensurePFAnchorActive checks live PF state and returns an explicit outcome for // callers that must distinguish intact, restored, deferred, skipped, and failed. func (p *prog) ensurePFAnchorActive() pfAnchorCheckResult { @@ -1784,6 +1937,9 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu rdrAnchorRef := fmt.Sprintf("rdr-anchor \"%s\"", pfAnchorName) anchorRef := fmt.Sprintf("anchor \"%s\"", pfAnchorName) needsRestore := false + // The anchor event parses the rules this check reads, so it starts no + // second pair of pfctl processes. + var dump pfRuleDump // Check 1: anchor references in the main ruleset. natOut, err := runPFAnchorCheckCommand("-sn") @@ -1792,6 +1948,7 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump NAT rules") return pfAnchorCheckFailed } + dump.nat = natOut natStr := string(natOut) if !strings.Contains(natStr, rdrAnchorRef) { mainLog.Load().Warn().Msg("DNS intercept watchdog: rdr-anchor reference missing from running ruleset") @@ -1805,6 +1962,7 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump filter rules") return pfAnchorCheckFailed } + dump.rules = filterOut if !strings.Contains(string(filterOut), anchorRef) { mainLog.Load().Warn().Msg("DNS intercept watchdog: anchor reference missing from running filter rules") needsRestore = true @@ -1864,6 +2022,7 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu p.pfBackoffMultiplier.Add(1) mainLog.Load().Warn().Msgf("DNS intercept: rules wiped %s after restore — entering stabilization (backoff multiplier: %d)", elapsed, p.pfBackoffMultiplier.Load()) + p.notePFAnchorMissing(dump) p.pfStartStabilization() return pfAnchorCheckDeferred } @@ -1873,7 +2032,7 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu } if !needsRestore && !forceRebuild { - mainLog.Load().Debug().Msg("DNS intercept watchdog: pf anchor intact") + p.logPFAnchorIntact() return pfAnchorCheckIntact } @@ -1881,6 +2040,9 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu if forceRebuild && !needsRestore { reason = "post-stabilization rebuild" } + if needsRestore { + p.notePFAnchorMissing(dump) + } return restorePFAnchorForReconcile(p, reason) } @@ -2007,7 +2169,7 @@ func (p *prog) scheduleDelayedRechecks() { // Refresh OS resolver — VPN may have finished DNS cleanup since the // immediate handler ran. This clears stale LAN nameservers (e.g., // a VPN's DNS IP (e.g., 10.255.255.3) lingering in scutil --dns). - ctrld.InitializeOsResolver(true) + ctrld.InitializeOsResolverWithReason(true, osResolverReasonDelayedRecheck) if p.vpnDNS != nil { p.vpnDNS.Refresh(true) } @@ -2145,7 +2307,7 @@ func (p *prog) exemptVPNDNSServers(exemptions []vpnDNSExemption) error { } defer p.pfEnsureRunning.Store(false) - tunnelIfaces := append([]string(nil), discoverTunnelInterfacesForReconcile()...) + tunnelIfaces := append([]string(nil), p.activeTunnelInterfaces()...) rulesStr := p.buildPFAnchorRulesForTunnels(exemptions, tunnelIfaces) if err := os.WriteFile(pfAnchorFile, []byte(rulesStr), 0644); err != nil { diff --git a/cmd/cli/dns_intercept_darwin_test.go b/cmd/cli/dns_intercept_darwin_test.go index 6c8b24e..fa9b45b 100644 --- a/cmd/cli/dns_intercept_darwin_test.go +++ b/cmd/cli/dns_intercept_darwin_test.go @@ -6,9 +6,12 @@ import ( "context" "errors" "fmt" + "net" "os" "path/filepath" + "slices" "strings" + "sync" "testing" "time" @@ -987,7 +990,7 @@ func TestFinishPFStabilizationRunsFunctionalVerification(t *testing.T) { "-a com.controld.ctrld -sn": "rdr on lo0", }) originalResolver := initializeOsResolver - initializeOsResolver = func(bool) []string { return nil } + initializeOsResolver = func(bool, string) []string { return nil } t.Cleanup(func() { initializeOsResolver = originalResolver }) probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) @@ -1092,7 +1095,7 @@ func stubWakeProbeSchedule(t *testing.T, attempts int) (refreshes *int) { pfWakeProbeOwnerWait = 10 * time.Millisecond calls := 0 - initializeOsResolver = func(bool) []string { + initializeOsResolver = func(bool, string) []string { calls++ return []string{"10.0.0.1:53"} } @@ -1275,7 +1278,7 @@ func TestWakeProbeStopsWhenServiceStops(t *testing.T) { pfWakeProbeDelays = []time.Duration{0, time.Hour} t.Cleanup(func() { pfWakeProbeDelays = originalDelays }) originalResolver := initializeOsResolver - initializeOsResolver = func(bool) []string { return []string{"10.0.0.1:53"} } + initializeOsResolver = func(bool, string) []string { return []string{"10.0.0.1:53"} } t.Cleanup(func() { initializeOsResolver = originalResolver }) probes, _ := stubStabilizationProbe(t, nil, false) // first probe fails, reload refuses @@ -1433,3 +1436,531 @@ func TestStartDNSInterceptSkipsBackgroundWorkWhenInstallFails(t *testing.T) { // wiring a moment to report rather than racing the test's end. time.Sleep(50 * time.Millisecond) } + +// ============================================================================= +// pf anchor state, tunnel, and wake event tests +// ============================================================================= + +// pfStateCommandsMu guards the answer of the pf and tunnel state reads, because +// a watchdog or stabilization goroutine can read while a test replaces it. +var ( + pfStateCommandsMu sync.Mutex + pfStateCommandFn func(string, ...string) ([]byte, error) +) + +// init keeps the whole test binary away from pfctl and ifconfig. A test that +// needs values stubs them with stubPFStateCommands. +func init() { + pfStateRunCommand = func(name string, args ...string) ([]byte, error) { + pfStateCommandsMu.Lock() + run := pfStateCommandFn + pfStateCommandsMu.Unlock() + if run == nil { + return nil, errors.New("pf state read is not stubbed") + } + return run(name, args...) + } +} + +// stubPFStateCommands answers the pf and tunnel state reads from fixture text, +// keyed by the whole command line. +func stubPFStateCommands(t *testing.T, outputs map[string]string) { + t.Helper() + stubPFStateCommandFn(t, func(name string, args ...string) ([]byte, error) { + key := strings.Join(append([]string{name}, args...), " ") + output, ok := outputs[key] + if !ok { + return nil, fmt.Errorf("no fixture for %q", key) + } + return []byte(output), nil + }) +} + +// stubPFStateCommandFn answers the pf and tunnel state reads from run. +func stubPFStateCommandFn(t *testing.T, run func(string, ...string) ([]byte, error)) { + t.Helper() + pfStateCommandsMu.Lock() + original := pfStateCommandFn + pfStateCommandFn = run + pfStateCommandsMu.Unlock() + t.Cleanup(func() { + pfStateCommandsMu.Lock() + pfStateCommandFn = original + pfStateCommandsMu.Unlock() + }) +} + +// pfIntactAnchorRules answers the anchor check of a healthy ruleset. +var pfIntactAnchorRules = map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", +} + +// pfStateFixtures answers the anchor list and status reads with a real ruleset. +var pfStateFixtures = map[string]string{ + "pfctl -sr": pfShowRules, + "pfctl -sn": pfShowNAT, + "pfctl -si": pfStatusEnabled, +} + +// waitForStabilizationExit waits for the stabilization loop to release pf. +func waitForStabilizationExit(t *testing.T, p *prog) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for p.pfStabilizing.Load() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if p.pfStabilizing.Load() { + t.Fatal("the stabilization loop did not observe the closed stop channel") + } +} + +// Test_logPFAnchorListLogsStabilizationStartAndEnd covers the anchor list at the +// two stabilization events. A capture of a VPN connect has to name the anchors +// that were loaded before ctrld waited, and the ones that were loaded after. +func Test_logPFAnchorListLogsStabilizationStartAndEnd(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + stubPFAnchorCheckCommand(t, pfIntactAnchorRules) + originalResolver := initializeOsResolver + initializeOsResolver = func(bool, string) []string { return nil } + t.Cleanup(func() { initializeOsResolver = originalResolver }) + stubStabilizationProbe(t, []bool{true}, true) + + stopCh := make(chan struct{}) + close(stopCh) + p := &prog{dnsInterceptState: &pfState{}, stopCh: stopCh} + p.pfStartStabilization() + waitForStabilizationExit(t, p) + p.pfStabilizing.Store(true) + p.finishPFStabilization(time.Millisecond) + stopPFTestTimers(p) + + events := jsonLogEvents(t, logs, "PF anchor list changed") + if len(events) != 2 { + t.Fatalf("got %d anchor list events for one stabilization, want 2", len(events)) + } + wantField(t, events[0], "reason", "stabilization_start") + wantField(t, events[0], "journal", true) + wantField(t, events[0], "pf_enabled", true) + wantField(t, events[0], "pf_since", "0 days 02:11:05") + wantAnchors(t, events[0], []string{"com.apple/*", "com.controld.ctrld"}) + wantField(t, events[1], "reason", "stabilization_end") + wantField(t, events[1], "journal", true) + + for _, prefix := range []string{ + "DNS intercept: VPN connecting", + "DNS intercept: pf stable for", + } { + lines := eventsWithPrefix(t, logs, prefix) + if len(lines) != 1 { + t.Fatalf("got %d lines with prefix %q, want 1", len(lines), prefix) + } + wantField(t, lines[0], "journal", true) + } +} + +// Test_logPFAnchorListReportsAMissingAnchorOnce covers the watchdog state. The +// anchor stays missing for as long as another program holds pf, and one line per +// tick would fill the journal with the same fact. +func Test_logPFAnchorListReportsAMissingAnchorOnce(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + outputs := pfIntactAnchorRules + original := runPFAnchorCheckCommand + runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { + return []byte(outputs[strings.Join(args, " ")]), nil + } + originalRestore := restorePFAnchorForReconcile + restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult { return pfAnchorCheckRestored } + t.Cleanup(func() { + runPFAnchorCheckCommand = original + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{dnsInterceptState: &pfState{}} + for range 3 { + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckIntact { + t.Fatalf("healthy check result = %v, want intact", result) + } + } + intact := eventsWithPrefix(t, logs, "DNS intercept watchdog: pf anchor intact") + if len(intact) != 1 { + t.Fatalf("got %d intact lines for three healthy checks, want 1", len(intact)) + } + wantField(t, intact[0], "repeats", float64(0)) + + outputs = map[string]string{} + for range 2 { + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckRestored { + t.Fatalf("wiped check result = %v, want restored", result) + } + } + missing := jsonLogEvents(t, logs, "PF anchor list changed") + if len(missing) != 1 { + t.Fatalf("got %d anchor list events for two wiped checks, want 1", len(missing)) + } + wantField(t, missing[0], "reason", "missing") + wantField(t, missing[0], "level", "warn") + wantField(t, missing[0], "journal", true) + + outputs = pfIntactAnchorRules + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckIntact { + t.Fatalf("recovered check result = %v, want intact", result) + } + intact = eventsWithPrefix(t, logs, "DNS intercept watchdog: pf anchor intact") + if len(intact) != 2 { + t.Fatalf("got %d intact lines after the anchor came back, want 2", len(intact)) + } + // The second wiped check repeated the state of the first one, and that held + // line is the count the recovered line carries. + wantField(t, intact[1], "repeats", float64(1)) +} + +// Test_logPFAnchorListReportsARestoredAnchor covers the restore report. The +// missing event alone leaves a capture without the anchors ctrld ended with. +func Test_logPFAnchorListReportsARestoredAnchor(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + originalReference := ensurePFAnchorReferenceForRestore + originalRebuild := rebuildPFAnchorRulesForReconcile + originalVerify := verifyPFStateFn + ensurePFAnchorReferenceForRestore = func(*prog) error { return nil } + rebuildPFAnchorRulesForReconcile = func(*prog, []vpnDNSExemption) ([]string, error) { return nil, nil } + verifyPFStateFn = func(*prog) bool { return true } + t.Cleanup(func() { + ensurePFAnchorReferenceForRestore = originalReference + rebuildPFAnchorRulesForReconcile = originalRebuild + verifyPFStateFn = originalVerify + }) + + p := &prog{dnsInterceptState: &pfState{}} + if result := p.restorePFAnchorWithTransportReset("test", false); result != pfAnchorCheckRestored { + t.Fatalf("restore result = %v, want restored", result) + } + + events := jsonLogEvents(t, logs, "PF anchor list changed") + if len(events) != 1 { + t.Fatalf("got %d anchor list events for one restore, want 1", len(events)) + } + wantField(t, events[0], "reason", "restored") + wantField(t, events[0], "level", "info") + wantField(t, events[0], "journal", true) + wantAnchors(t, events[0], []string{"com.apple/*", "com.controld.ctrld"}) +} + +// Test_logPFAnchorListMarksAnUnknownAnchorList covers a failed read. pfctl fails +// on a host that revoked the privileges of ctrld, and an empty list there reads +// as a ruleset that holds no anchor at all. +func Test_logPFAnchorListMarksAnUnknownAnchorList(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, nil) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{}) + + events := jsonLogEvents(t, logs, "PF anchor list changed") + if len(events) != 1 { + t.Fatalf("got %d anchor list events, want 1", len(events)) + } + wantField(t, events[0], "anchors_known", false) + if _, ok := events[0]["anchors"]; ok { + t.Fatalf("a failed read reported an anchor list: %v", events[0]) + } +} + +// TestTunnelInterfaceChangedNamesTheOwner covers the tunnel event. The #611 +// capture could not tell a VPN tunnel from a mesh client, and the owner is the +// field that names the program whose pf rules compete with the ctrld anchor. +func TestTunnelInterfaceChangedNamesTheOwner(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, map[string]string{"ifconfig -v utun9": ifconfigWindscribeTunnel}) + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return []string{"utun9"} } + t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover }) + + p := &prog{dnsInterceptState: &pfState{}, lastTunnelIfaces: []string{"utun7"}} + // Stabilization owns the rebuild, so the event is all this drives. + p.pfStabilizing.Store(true) + if !p.checkTunnelInterfaceChanges() { + t.Fatal("the tunnel set change was not observed") + } + + events := jsonLogEvents(t, logs, "Tunnel interface changed") + if len(events) != 1 { + t.Fatalf("got %d tunnel events for one set change, want 1", len(events)) + } + wantField(t, events[0], "owner", "Windscribe VPN") + wantField(t, events[0], "journal", true) + wantStrings(t, events[0], "added", []string{"utun9"}) + wantStrings(t, events[0], "removed", []string{"utun7"}) + + discovered := eventsWithPrefix(t, logs, "DNS intercept: discovered active tunnel interfaces") + if len(discovered) != 1 { + t.Fatalf("got %d discovery lines for one tunnel set, want 1", len(discovered)) + } + if p.checkTunnelInterfaceChanges() { + t.Fatal("the identical pending tunnel set was not coalesced") + } + discovered = eventsWithPrefix(t, logs, "DNS intercept: discovered active tunnel interfaces") + if len(discovered) != 1 { + t.Fatalf("got %d discovery lines for an unchanged tunnel set, want 1", len(discovered)) + } +} + +// TestVerifyInterceptAfterWakeReportsTheWake covers the detector source of the +// wake event. The 2026-09-14 capture has no wake line at all, and the resolver +// table after a resume is the state the investigation needs next. +func TestVerifyInterceptAfterWakeReportsTheWake(t *testing.T) { + logs := captureDebugMainLog(t) + stubWakeProbeSchedule(t, 4) + stubStabilizationProbe(t, []bool{true}, true) + now := time.Date(2026, 9, 14, 7, 14, 0, 0, time.UTC) + originalNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return now } + t.Cleanup(func() { networkEventsNowFn = originalNow }) + + p := &prog{ + dnsInterceptState: &pfState{}, + dnsConfig: newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil }), + } + p.verifyInterceptAfterWake(18 * time.Second) + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 { + t.Fatalf("got %d Host woke events for one resume, want 1", len(events)) + } + wantField(t, events[0], "source", "detector") + wantField(t, events[0], "gap_ms", float64(18000)) + wantField(t, events[0], "journal", true) + if got := p.dnsConfig.nextDelay(now); got != dnsConfigFastInterval { + t.Fatalf("DNS configuration poll delay after a wake = %s, want %s", got, dnsConfigFastInterval) + } +} + +func wantAnchors(t *testing.T, event map[string]any, want []string) { + t.Helper() + wantStrings(t, event, "anchors", want) +} + +func wantStrings(t *testing.T, event map[string]any, field string, want []string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("field %q: got %v, want a list", field, event[field]) + } + if len(values) != len(want) { + t.Fatalf("field %q: got %v, want %v", field, values, want) + } + for i, value := range values { + if value != want[i] { + t.Fatalf("field %q: got %v, want %v", field, values, want) + } + } +} + +// TestIgnoredNetworkChangeJournalsTheInterfaceChange proves that an interface +// that appears or disappears on the ignored path enters the journal with its +// class and its action, so a hypervisor adapter storm is readable after the fact. +func TestIgnoredNetworkChangeJournalsTheInterfaceChange(t *testing.T) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + stubSnapshotVirtualSet(t, "bridge101") + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return nil } + t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover }) + + p := &prog{dnsInterceptState: &pfState{}, vpnDNS: newVPNDNSManager(nil)} + p.pfStabilizing.Store(true) + t.Cleanup(func() { + p.pfDelayedRecheckMu.Lock() + defer p.pfDelayedRecheckMu.Unlock() + for _, timer := range p.pfDelayedRecheckTimers { + if timer != nil { + timer.Stop() + } + } + }) + stateWith := func(names ...string) *netmon.State { + state := &netmon.State{Interface: map[string]netmon.Interface{}} + for _, name := range names { + state.Interface[name] = netmon.Interface{Interface: &net.Interface{Name: name, Flags: net.FlagUp}} + } + return state + } + now := time.Unix(1_000_000, 0) + + p.handleDNSInterceptIgnoredNetworkChange(&netmon.ChangeDelta{Old: stateWith("en0"), New: stateWith("en0", "bridge101")}, now) + p.handleDNSInterceptIgnoredNetworkChange(&netmon.ChangeDelta{Old: stateWith("en0", "bridge101"), New: stateWith("en0")}, now.Add(time.Second)) + + events := jsonLogEvents(t, logs, "DNS intercept: interface appeared/disappeared — starting interception probe monitor") + if len(events) != 2 { + t.Fatalf("interface events: got %d, want 2: %s", len(events), logs.String()) + } + for i, action := range []string{"added", "removed"} { + wantField(t, events[i], "journal", true) + wantField(t, events[i], "interface", "bridge101") + wantField(t, events[i], "class", "virtual") + wantField(t, events[i], "action", action) + } +} + +// TestTunnelInterfaceChangedLogsOncePerChange covers the retry path and the +// flap back. The reconcile keeps a new tunnel set pending until it succeeds, +// so every retry repeated the same event, and a set that returned to the one +// before it reported nothing at all. +func TestTunnelInterfaceChangedLogsOncePerChange(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + originalDiscover := discoverTunnelInterfacesForReconcile + originalRestore := restorePFAnchorForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return []string{"utun7"} } + restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult { return pfAnchorCheckFailed } + t.Cleanup(func() { + discoverTunnelInterfacesForReconcile = originalDiscover + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{dnsInterceptState: &pfState{}, lastTunnelIfaces: []string{"utun7", "utun9"}} + for range 3 { + p.checkTunnelInterfaceChanges() + } + + events := jsonLogEvents(t, logs, tunnelChangedMessage) + if len(events) != 1 { + t.Fatalf("got %d tunnel events for three retries of one change, want 1", len(events)) + } + wantStrings(t, events[0], "removed", []string{"utun9"}) + + // The tunnel comes back before a reconcile applied its removal. + discoverTunnelInterfacesForReconcile = func() []string { return []string{"utun7", "utun9"} } + p.checkTunnelInterfaceChanges() + + events = jsonLogEvents(t, logs, tunnelChangedMessage) + if len(events) != 2 { + t.Fatalf("got %d tunnel events after the tunnel came back, want 2", len(events)) + } + wantStrings(t, events[1], "added", []string{"utun9"}) +} + +// TestPFAnchorWipeAfterRestoreReportsTheMissingAnchor covers the wipe that +// follows a restore within ten seconds. That path enters stabilization instead +// of a rebuild, and it left the journal without the wipe that caused it. +func TestPFAnchorWipeAfterRestoreReportsTheMissingAnchor(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + stubPFAnchorCheckCommand(t, map[string]string{"-sn": ""}) + stubStabilizationProbe(t, []bool{true}, true) + stopCh := make(chan struct{}) + close(stopCh) + + p := &prog{dnsInterceptState: &pfState{}, stopCh: stopCh} + p.pfLastRestoreTime.Store(time.Now().UnixMilli()) + + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckDeferred { + t.Fatalf("wipe after restore result = %v, want deferred", result) + } + waitForStabilizationExit(t, p) + stopPFTestTimers(p) + + events := jsonLogEvents(t, logs, pfAnchorListMessage) + if len(events) != 2 { + t.Fatalf("got %d anchor list events for one wipe, want 2", len(events)) + } + wantField(t, events[0], "reason", pfAnchorReasonMissing) + wantField(t, events[0], "level", "warn") + wantField(t, events[1], "reason", pfAnchorReasonStabilizationStart) +} + +// TestPFAnchorListParsesTheRulesTheCallerHolds covers the anchor read of the +// watchdog. The watchdog already read both rule sets, and one more pair of +// pfctl processes on the network change path is work the host does not need. +func TestPFAnchorListParsesTheRulesTheCallerHolds(t *testing.T) { + logs := captureDebugMainLog(t) + var reads []string + stubPFStateCommandFn(t, func(name string, args ...string) ([]byte, error) { + key := strings.Join(append([]string{name}, args...), " ") + reads = append(reads, key) + output, ok := pfStateFixtures[key] + if !ok { + return nil, fmt.Errorf("no fixture for %q", key) + } + return []byte(output), nil + }) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonMissing, mainLog.Load().Warn(), pfRuleDump{ + rules: []byte(pfShowRules), + nat: []byte(pfShowNAT), + }) + + if want := []string{"pfctl -si"}; !slices.Equal(reads, want) { + t.Fatalf("pf state reads = %v, want %v", reads, want) + } + events := jsonLogEvents(t, logs, pfAnchorListMessage) + if len(events) != 1 { + t.Fatalf("got %d anchor list events, want 1", len(events)) + } + wantAnchors(t, events[0], []string{"com.apple/*", "com.controld.ctrld"}) + wantField(t, events[0], "pf_enabled", true) +} + +// TestPFAnchorListBacksOffOnExhaustedResources covers a failed anchor read. +// Every other pfctl read feeds the backoff, so the anchor read kept starting +// processes on a host that had none left. +func TestPFAnchorListBacksOffOnExhaustedResources(t *testing.T) { + captureDebugMainLog(t) + stubPFStateCommandFn(t, func(string, ...string) ([]byte, error) { + return nil, errors.New("fork/exec /sbin/pfctl: resource temporarily unavailable") + }) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{}) + + if !p.pfExecBackoffActive() { + t.Fatal("a failed anchor read did not start the pfctl backoff") + } +} + +// TestPFAnchorListMarksATruncatedRuleListUnknown covers an oversized read. A +// reader stops at a line it cannot hold, and the anchors it collected before +// that line read as a ruleset that lost the rest of them. +func TestPFAnchorListMarksATruncatedRuleListUnknown(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{ + rules: []byte(pfShowRules + strings.Repeat("a", 70<<10) + "\n"), + nat: []byte(pfShowNAT), + }) + + events := jsonLogEvents(t, logs, pfAnchorListMessage) + if len(events) != 1 { + t.Fatalf("got %d anchor list events, want 1", len(events)) + } + wantField(t, events[0], "anchors_known", false) + if _, ok := events[0]["anchors"]; ok { + t.Fatalf("a truncated read reported an anchor list: %v", events[0]) + } +} + +// TestPFStateOutputIsBounded covers the memory a state read can take. A pfctl +// that prints without end must not grow the daemon. +func TestPFStateOutputIsBounded(t *testing.T) { + out := &boundedBuffer{limit: 8} + if _, err := out.Write([]byte("12345")); err != nil { + t.Fatalf("write below the limit: %v", err) + } + if _, err := out.Write([]byte("67890")); err != nil { + t.Fatalf("write above the limit: %v", err) + } + if !out.exceeded { + t.Fatal("the writer did not report the dropped bytes") + } + if got := out.Len(); got > 8 { + t.Fatalf("the writer kept %d bytes, want at most 8", got) + } +} diff --git a/cmd/cli/dns_intercept_settle.go b/cmd/cli/dns_intercept_settle.go index 972a2ab..2af934d 100644 --- a/cmd/cli/dns_intercept_settle.go +++ b/cmd/cli/dns_intercept_settle.go @@ -2,11 +2,20 @@ package cli import "github.com/Control-D-Inc/ctrld" -var initializeOsResolver = ctrld.InitializeOsResolver +var initializeOsResolver = ctrld.InitializeOsResolverWithReason + +// Reasons that the journal reports when the resolver set changes. Each one +// names the code path that read the resolvers of the host. +const ( + osResolverReasonStart = "start" + osResolverReasonWakeProbe = "wake_probe" + osResolverReasonVPNSettle = "vpn_settle" + osResolverReasonDelayedRecheck = "delayed_recheck" +) func (p *prog) refreshDNSAfterVPNSettle(reason string) (routes, domainlessServers, exemptions int) { mainLog.Load().Info().Msgf("DNS intercept: refreshing OS/VPN DNS route state after VPN settle (%s)", reason) - ns := initializeOsResolver(true) + ns := initializeOsResolver(true, osResolverReasonVPNSettle) mainLog.Load().Debug().Msgf("DNS intercept: post-settle OS resolver nameservers: %v", ns) if p.vpnDNS == nil { diff --git a/cmd/cli/dns_intercept_settle_test.go b/cmd/cli/dns_intercept_settle_test.go index 36cc26f..c2216d0 100644 --- a/cmd/cli/dns_intercept_settle_test.go +++ b/cmd/cli/dns_intercept_settle_test.go @@ -12,8 +12,10 @@ func TestRefreshDNSAfterVPNSettleRefreshesOSResolverAndVPNRoutes(t *testing.T) { defer func() { initializeOsResolver = oldInitialize }() var initialized []bool - initializeOsResolver = func(force bool) []string { + var reasons []string + initializeOsResolver = func(force bool, reason string) []string { initialized = append(initialized, force) + reasons = append(reasons, reason) return []string{"10.102.26.10:53"} } @@ -40,6 +42,9 @@ func TestRefreshDNSAfterVPNSettleRefreshesOSResolverAndVPNRoutes(t *testing.T) { if len(initialized) != 1 || !initialized[0] { t.Fatalf("expected forced OS resolver refresh once, got %v", initialized) } + if len(reasons) != 1 || reasons[0] != osResolverReasonVPNSettle { + t.Fatalf("OS resolver reasons = %v, want [%s]", reasons, osResolverReasonVPNSettle) + } if got := p.vpnDNS.UpstreamForDomain("jira.cc.bmwgroup.net."); len(got) != 1 || got[0] != "10.102.26.10" { t.Fatalf("expected refreshed VPN DNS route, got %v", got) } diff --git a/cmd/cli/dns_intercept_windows.go b/cmd/cli/dns_intercept_windows.go index 9573862..4dd87d2 100644 --- a/cmd/cli/dns_intercept_windows.go +++ b/cmd/cli/dns_intercept_windows.go @@ -2749,7 +2749,7 @@ func (p *prog) scheduleDelayedRechecks() { } // Refresh OS resolver — VPN may have finished DNS cleanup since the // immediate handler ran. - ctrld.InitializeOsResolver(true) + ctrld.InitializeOsResolverWithReason(true, osResolverReasonDelayedRecheck) if p.vpnDNS != nil { p.vpnDNS.Refresh(true) } diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index 3be5cfc..bb6b0f5 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -264,6 +264,8 @@ func (p *prog) serveDNS(listenerNum string) error { } labelValues = append(labelValues, dns.TypeToString[q.Qtype]) labelValues = append(labelValues, dns.RcodeToString[answer.Rcode]) + // The grade of the window must not depend on the goroutine below. + p.health.countQuery() go func() { p.WithLabelValuesInc(statsQueriesCount, labelValues...) p.WithLabelValuesInc(statsClientQueriesCount, []string{ci.IP, ci.Mac, ci.Hostname}...) @@ -689,6 +691,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { } else { ctrld.Log(ctx, mainLog.Load().Debug(), "hit cached response") } + p.health.countCacheHit() res.answer = answer res.cached = true return res @@ -748,6 +751,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { if gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, vpnServers) { ctrld.Log(ctx, mainLog.Load().Debug(), "All VPN DNS servers had transport failures for %s; returning SERVFAIL while retained VPN DNS state is active", domain) + p.countFailedClientQuery(upstreams) answer := new(dns.Msg) answer.SetRcode(req.msg, dns.RcodeServerFailure) return &proxyResponse{answer: answer} @@ -815,6 +819,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { if !gotDNSAnswer && gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, dlServers) { ctrld.Log(ctx, mainLog.Load().Debug(), "All domain-less VPN DNS servers had transport failures for %s; returning SERVFAIL while retained VPN DNS state is active", domain) + p.countFailedClientQuery(upstreams) answer := new(dns.Msg) answer.SetRcode(req.msg, dns.RcodeServerFailure) return &proxyResponse{answer: answer} @@ -857,7 +862,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { // keep the address-bearing error at debug. if isInternalDomainUpstream(upstream) { ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "failed to resolve query") - ctrld.Log(ctx, p.querySampler.event(sampleClassResolveFailed, upstream).Str("failure", internalDomainFailureReason(err)), + ctrld.Log(ctx, p.querySampler.event(sampleClassInternalDomain, upstream).Str("failure", internalDomainFailureReason(err)), "failed to resolve query using an Internal Domain resolver") } else { ctrld.Log(ctx, p.querySampler.event(sampleClassResolveFailed, upstream).Err(err), "failed to resolve query") @@ -899,6 +904,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { if answer == nil { if serveStaleCache && staleAnswer != nil { ctrld.Log(ctx, mainLog.Load().Debug(), "serving stale cached response") + p.health.countCacheHit() now := time.Now() setCachedAnswerTTL(staleAnswer, now, now.Add(staleTTL)) res.answer = staleAnswer @@ -1017,12 +1023,24 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { } } + p.countFailedClientQuery(upstreams) answer := new(dns.Msg) answer.SetRcode(req.msg, dns.RcodeServerFailure) res.answer = answer return res } +// countFailedClientQuery grades one client query that ended without an answer. +// A query that only Internal Domain resolvers serve stays out of the grade: an +// endpoint away from the organization network never reaches them, and that is +// not an outage of the query path. +func (p *prog) countFailedClientQuery(upstreams []string) { + if internalDomainExplicitUpstreams(upstreams) { + return + } + p.health.countFailedQuery() +} + func (p *prog) upstreamsAndUpstreamConfigForPtr(upstreams []string, upstreamConfigs []*ctrld.UpstreamConfig) ([]string, []*ctrld.UpstreamConfig) { if len(p.localUpstreams) > 0 { tmp := make([]string, 0, len(p.localUpstreams)+len(upstreams)) @@ -1678,12 +1696,66 @@ func (p *prog) handleNetworkChange(delta *netmon.ChangeDelta, isMajorChange bool // delta.Major flag after a time jump does not change the cache. p.networkSourceMu.Lock() currentState := networkChangeCurrentStateFn(delta) - if !networkSnapshotCurrent(delta, isMajorChange, currentState) { + current := networkSnapshotCurrent(delta, isMajorChange, currentState) + if delta.TimeJumped { + // A late callback still tells that the host woke, but the network of + // the wake is the one that netmon holds now. + wakeState := delta.New + if !current { + wakeState = currentState + } + p.noteHostWoke("netmon", 0, wakeState) + } + // Only a current callback becomes the baseline of the next diff. A late + // callback that a newer snapshot replaced would hide the changes between + // the callback before it and the callback after it. + before := p.deltaBeforeState(delta, currentState, current, isMajorChange) + changes := diffNetworkDelta(before, delta.New) + if current && hasAddOrRemove(changes) { + refreshInterfaceMeta() + } + // The noise test runs before any other work, because the point of the class + // is that a storm starts no pfctl and no scutil process. + if current && noiseDelta(before, delta.New, delta.TimeJumped, changes) { p.networkSourceMu.Unlock() - mainLog.Load().Debug().Msg("Network transition skipped: snapshot superseded") + p.noteNoiseDelta(changes) return } - transitionID := p.networkTransitionGen.Add(1) + // A superseded snapshot never becomes a transition, so it takes no ID and + // reports zero. + var transitionID uint64 + changed := false + changedInterface := "" + sourceSnapshotAvailable := false + before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + after4, after6 := before4, before6 + // The newer callback of this epoch carries the same interface changes, so + // this outcome reports at debug and stays out of the journal. + outcome := transitionOutcomeSnapshotSuperseded + defer func() { + stateBefore, stateAfter := networkStateOrEmpty(before), networkStateOrEmpty(delta.New) + networkTransitionEvent(outcome).Uint64("transition_id", transitionID). + Bool("is_major_change", isMajorChange).Bool("changed", changed). + Str("interface", changedInterface). + Strs("changed_interfaces", changedInterfaceNames(changes)). + Bool("time_jumped", delta.TimeJumped). + Str("default_route_before", stateBefore.DefaultRouteInterface). + Str("default_route_after", stateAfter.DefaultRouteInterface). + Bool("have_v4_before", stateBefore.HaveV4).Bool("have_v4_after", stateAfter.HaveV4). + Bool("have_v6_before", stateBefore.HaveV6).Bool("have_v6_after", stateAfter.HaveV6). + Str("source_ipv4_before", before4.String()).Str("source_ipv6_before", before6.String()). + Str("source_ipv4_after", after4.String()). + Str("source_ipv6_after", after6.String()). + Bool("source_snapshot_available", sourceSnapshotAvailable).Str("outcome", outcome). + Msg(networkTransitionMessage) + }() + if !current { + p.networkSourceMu.Unlock() + return + } + p.flushNoiseSummary() + outcome = transitionOutcomeIgnored + transitionID = p.networkTransitionGen.Add(1) sourceState := delta.New // A late major callback and reordered minor callbacks share one cache // epoch. For these cases, source validity comes from the OS, not ordering. @@ -1707,30 +1779,25 @@ func (p *prog) handleNetworkChange(delta *netmon.ChangeDelta, isMajorChange bool } } p.networkSourceState, p.networkSourceEpoch = sourceState, currentState - sourceSnapshotAvailable := sourceState != nil - before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + sourceSnapshotAvailable = sourceState != nil validateDefaultLocalIPsFromDelta(sourceState, transitionID) - after4, after6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + after4, after6 = ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + // The network of this callback belongs to the epoch that the lines above + // store. A store after the unlock can overwrite the network of a newer + // callback. The header render reads the hardware ports, so it runs after + // the unlock and starts no process under the lock. + p.lastNetworkState.Store(currentState) + // HasIPv6 reads the flag that this callback stores, so one monitor serves + // the whole process. + ctrld.SetIPv6Available(currentState.HaveV6) p.networkSourceMu.Unlock() - p.noteNetworkState(currentState) + p.refreshLogHeader() + describeInterfaceChanges(changes, interfaceMetaFor) + logInterfaceChanges(transitionID, changes) p.handleDNS64NetworkChange(delta, isMajorChange) validIfaces := networkChangeValidInterfacesFn() - changed := false - changedInterface := "" - outcome := "ignored" - defer func() { - mainLog.Load().Debug().Uint64("transition_id", transitionID). - Bool("is_major_change", isMajorChange).Bool("changed", changed). - Str("interface", changedInterface). - Str("default_route_before", delta.Old.DefaultRouteInterface). - Str("default_route_after", delta.New.DefaultRouteInterface). - Str("source_ipv4_before", before4.String()).Str("source_ipv6_before", before6.String()). - Str("source_ipv4_after", after4.String()). - Str("source_ipv6_after", after6.String()). - Bool("source_snapshot_available", sourceSnapshotAvailable).Str("outcome", outcome).Msg("Network transition") - }() activeInterfaceExists := false var changeIPs []netip.Prefix // Check each valid interface for changes @@ -1750,6 +1817,9 @@ func (p *prog) handleNetworkChange(delta *netmon.ChangeDelta, isMajorChange bool // The interface is new (was not present in the old state). usableNewIPs := filterUsableIPs(newIPs) if newIface.IsUp() && len(usableNewIPs) > 0 { + // The new interface is the active one, or the change ends as + // no_active_interface and nothing reconciles. + activeInterfaceExists = true changed = true changeIPs = usableNewIPs changedInterface = ifaceName @@ -1905,6 +1975,10 @@ func (p *prog) handleNetworkChange(delta *netmon.ChangeDelta, isMajorChange bool outcome = "accepted" networkChangeReconcileFn(p, transitionID) + p.logNetworkSnapshot("transition") + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(networkEventsNowFn()) + } } // reconcileNetworkChange is the first DNS/PF mutation boundary after source updates. @@ -1963,32 +2037,16 @@ func (p *prog) handleDNSInterceptIgnoredNetworkChange(delta *netmon.ChangeDelta, // Spawn an async monitor that probes pf interception with backoff and forces // a full pf reload if broken. if delta.Old != nil { - interfaceChanged := false - var changedIface string - for ifaceName := range delta.Old.Interface { - if ifaceName == "lo0" { - continue - } - if _, exists := delta.New.Interface[ifaceName]; !exists { - interfaceChanged = true - changedIface = ifaceName - break - } + changedAction := "removed" + changedIface := interfaceOnlyIn(delta.Old.Interface, delta.New.Interface) + if changedIface == "" { + changedAction = "added" + changedIface = interfaceOnlyIn(delta.New.Interface, delta.Old.Interface) } - if !interfaceChanged { - for ifaceName := range delta.New.Interface { - if ifaceName == "lo0" { - continue - } - if _, exists := delta.Old.Interface[ifaceName]; !exists { - interfaceChanged = true - changedIface = ifaceName - break - } - } - } - if interfaceChanged { - mainLog.Load().Info().Str("interface", changedIface). + if changedIface != "" { + journal(mainLog.Load().Info()).Str("interface", changedIface). + Str("class", interfaceMetaFor(changedIface).Class). + Str("action", changedAction). Msg("DNS intercept: interface appeared/disappeared — starting interception probe monitor") go p.pfInterceptMonitor() } @@ -2005,6 +2063,20 @@ func (p *prog) handleDNSInterceptIgnoredNetworkChange(delta *netmon.ChangeDelta, } } +// interfaceOnlyIn names one interface that have holds and lack does not. The +// loopback stays out, because it never appears and never goes away. +func interfaceOnlyIn(have, lack map[string]netmon.Interface) string { + for name := range have { + if name == "lo0" { + continue + } + if _, exists := lack[name]; !exists { + return name + } + } + return "" +} + // interfaceStatesEqual compares two interface states func interfaceStatesEqual(a, b *netmon.Interface) bool { if a == nil || b == nil { @@ -2133,6 +2205,10 @@ func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig, fail // leaves DoH transports in a stale state. const recoveryDebounceWindow = 500 * time.Millisecond +// recoveryResolverReason names this flow in the OS resolver events, so a +// reader tells a resolver read of the recovery from every other read. +const recoveryResolverReason = "recovery" + // debounceRecovery schedules a handleRecovery(NetworkChange) call after a debounce // window. If called again before the window expires, the timer is reset so that // recovery runs once with the final network state. All other state updates (IP, @@ -2166,6 +2242,48 @@ func (p *prog) debounceRecovery(transitionID uint64) { mainLog.Load().Debug().Msg("Recovery debounce: scheduled (500ms window)") } +// endRecovery closes one recovery pass. Support reads the end event and the +// snapshot that follows it to learn what the recovery left behind. +func (p *prog) endRecovery(diagnostic *recoveryDiagnostic, outcome, targetBefore string) { + // A superseded pass changed nothing that it owns any more. Its successor + // owns the loopback target and the bypass flag, so reading them here would + // report the state of the successor. + if outcome == recoveryOutcomeSuperseded { + diagnostic.interceptTargetAction = interceptTargetActionUnchanged + diagnostic.bypassActive = false + } else { + diagnostic.interceptTargetAction = interceptTargetAction(targetBefore, p.interceptTargetSnapshot()) + diagnostic.bypassActive = p.recoveryBypass.Load() + } + diagnostic.end(outcome) + p.logNetworkSnapshot("recovery_end") + // The resolver table of the host settles after the recovery, so the poll + // stays fast until it does. + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(networkEventsNowFn()) + } +} + +// The actions that one recovery takes on the loopback DNS target. +const ( + interceptTargetActionUnchanged = "unchanged" + interceptTargetActionRemoved = "removed" + interceptTargetActionSet = "set" +) + +// interceptTargetAction names what one recovery did to the loopback DNS +// target. The target value alone does not say whether this pass wrote it. +func interceptTargetAction(before, after string) string { + switch { + case before == after: + return interceptTargetActionUnchanged + case after == "": + return interceptTargetActionRemoved + default: + return interceptTargetActionSet + } +} + // handleRecovery performs a unified recovery by removing DNS settings, // canceling existing recovery checks for network changes, but coalescing duplicate // upstream failure recoveries, waiting for recovery to complete (using a cancellable context without timeout), @@ -2190,9 +2308,11 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u return } diagnostic := recoveryDiagnostic{transitionID: transitionID, generation: gen, reason: reason, started: time.Now()} - diagnostic.event(mainLog.Load().Debug()).Bool("intercept", interceptRecovery).Msg("Recovery begin") - outcome := "superseded" - defer func() { diagnostic.end(outcome) }() + journal(diagnostic.event(mainLog.Load().Info())).Bool("intercept", interceptRecovery).Msg("Recovery begin") + p.logNetworkSnapshot("recovery_begin") + targetBefore := p.interceptTargetSnapshot() + outcome := recoveryOutcomeSuperseded + defer func() { p.endRecovery(&diagnostic, outcome, targetBefore) }() if reason == RecoveryReasonNetworkChange { mainLog.Load().Debug().Msg("Network change recovery now owns shared recovery state") } @@ -2217,15 +2337,19 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u // the OS/DHCP resolver. This handles captive portal authentication // without the overhead of filter teardown/rebuild. if interceptRecovery { - mainLog.Load().Info().Msg("DNS intercept recovery: enabling DHCP bypass (filters stay active)") + journal(mainLog.Load().Info()).Msg("DNS intercept recovery: enabling DHCP bypass (filters stay active)") // Reinitialize OS resolver to discover DHCP servers on the new network. mainLog.Load().Debug().Msg("DNS intercept recovery: discovering DHCP nameservers") - dhcpServers, systemNameservers := ctrld.InitializeOsResolverWithSystemNameservers(true) - if len(dhcpServers) == 0 { - mainLog.Load().Warn().Msg("DNS intercept recovery: no DHCP nameservers found") + // The effective list adds a synthetic public resolver when the network + // gave none, so the journal reports the discovered list instead. + resolverNameservers, systemNameservers := initializeOsResolverWithSystemNameserversFn(true, recoveryResolverReason) + diagnostic.dhcpServers = systemNameservers + if len(systemNameservers) == 0 { + journal(mainLog.Load().Warn()).Msg("DNS intercept recovery: no DHCP nameservers found") } else { - mainLog.Load().Info().Msgf("DNS intercept recovery: found DHCP nameservers: %v", dhcpServers) + journal(mainLog.Load().Info()).Strs("dhcp_servers", systemNameservers). + Msgf("DNS intercept recovery: found DHCP nameservers: %v", systemNameservers) } // If the new network provides no usable IPv4 DNS (e.g. IPv6-only @@ -2236,11 +2360,12 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u // Exempt DHCP nameservers from intercept filters so the OS resolver // can actually reach them on port 53. - if len(dhcpServers) > 0 { + // The OS resolver queries the effective list, so the exemptions follow it. + if len(resolverNameservers) > 0 { // Build exemptions without an Interface — DHCP servers are not VPN-specific, // so they only generate group-scoped pf rules (ctrld process only). - exemptions := make([]vpnDNSExemption, 0, len(dhcpServers)) - for _, s := range dhcpServers { + exemptions := make([]vpnDNSExemption, 0, len(resolverNameservers)) + for _, s := range resolverNameservers { host := s if h, _, err := net.SplitHostPort(s); err == nil { host = h @@ -2259,11 +2384,11 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u // For an OS failure, reinitialize OS resolver nameservers immediately. if reason == RecoveryReasonOSFailure { mainLog.Load().Debug().Msg("OS resolver failure detected; reinitializing OS resolver nameservers") - ns := ctrld.InitializeOsResolver(true) + ns := ctrld.InitializeOsResolverWithReason(true, recoveryResolverReason) if len(ns) == 0 { mainLog.Load().Warn().Msg("No nameservers found for OS resolver; using existing values") } else { - mainLog.Load().Info().Msgf("Reinitialized OS resolver with nameservers: %v", ns) + journal(mainLog.Load().Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns) } } } @@ -2272,19 +2397,25 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u upstreams := p.buildRecoveryUpstreams(reason) // Wait indefinitely until one of the upstreams recovers. - recovered, err := p.waitForUpstreamRecovery(recoveryCtx, upstreams, &diagnostic) + recovered, err := waitForUpstreamRecoveryFn(p, recoveryCtx, upstreams, &diagnostic) if err != nil { if p.recoveryOwnsState(gen) { - outcome = "canceled" + outcome = recoveryOutcomeCanceled } p.recoveryCanceledCleanup(gen) return } + diagnostic.recoveredUpstream = recovered if !p.recoveryOwnsState(gen) { mainLog.Load().Debug().Msgf("Recovery generation %d was superseded after upstream success; skipping stale completion", gen) return } - mainLog.Load().Info().Msgf("Upstream %q recovered; re-applying DNS settings", recovered) + // The reset ends the outage, so the down time is read before it. The name + // of the line is bounded, because an upstream key is operator text. + journalName := journalUpstreamName(recovered) + journal(mainLog.Load().Info()).Str("upstream", journalName). + Int64("down_for_ms", p.um.downFor(recovered).Milliseconds()). + Msgf("Upstream %q recovered; re-applying DNS settings", journalName) // Reset the upstream failure count and down state while this generation // still owns recovery completion. @@ -2298,11 +2429,11 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u // Reinitialize OS resolver for the recovered state. if reason == RecoveryReasonNetworkChange { - ns := ctrld.InitializeOsResolver(true) + ns := ctrld.InitializeOsResolverWithReason(true, recoveryResolverReason) if len(ns) == 0 { mainLog.Load().Warn().Msg("No nameservers found for OS resolver during network-change recovery; using existing values") } else { - mainLog.Load().Info().Msgf("Reinitialized OS resolver with nameservers: %v", ns) + journal(mainLog.Load().Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns) } } } else { @@ -2314,29 +2445,33 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u // on a healthy DHCP network. systemNameservers = systemNameserversForInterceptRetry() } else if reason == RecoveryReasonNetworkChange { - ns := ctrld.InitializeOsResolver(true) + ns := ctrld.InitializeOsResolverWithReason(true, recoveryResolverReason) if len(ns) == 0 { mainLog.Load().Warn().Msg("No nameservers found for OS resolver during network-change recovery; using existing values") } else { - mainLog.Load().Info().Msgf("Reinitialized OS resolver with nameservers: %v", ns) + journal(mainLog.Load().Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns) } } - // Apply our DNS settings back and log the interface state. + // Apply our DNS settings back. The snapshot that follows the end event + // reports the interfaces. p.setDNS(systemNameservers) - p.logInterfacesState() } if !p.completeRecovery(gen) { mainLog.Load().Debug().Msgf("Recovery generation %d was superseded during completion; preserving successor state", gen) return } - outcome = "completed" + outcome = recoveryOutcomeCompleted if interceptRecovery { - mainLog.Load().Info().Msg("DNS intercept recovery complete: disabling DHCP bypass, resuming normal flow") + journal(mainLog.Load().Info()).Msg("DNS intercept recovery complete: disabling DHCP bypass, resuming normal flow") } } +// waitForUpstreamRecoveryFn is the seam of the upstream probe. A test drives +// the whole recovery flow without a query to a real upstream. +var waitForUpstreamRecoveryFn = (*prog).waitForUpstreamRecovery + // waitForUpstreamRecovery checks the provided upstreams concurrently until one recovers. // It returns the name of the recovered upstream or an error if the check times out. func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string]*ctrld.UpstreamConfig, diagnostic *recoveryDiagnostic) (string, error) { @@ -2400,11 +2535,11 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string // we should try to reinit the OS resolver to ensure we can recover if name == upstreamOS && attempts%3 == 0 { mainLog.Load().Debug().Msgf("UpstreamOS check failed on attempt %d, reinitializing OS resolver", attempts) - ns := ctrld.InitializeOsResolver(true) + ns := ctrld.InitializeOsResolverWithReason(true, recoveryResolverReason) if len(ns) == 0 { mainLog.Load().Warn().Msg("No nameservers found for OS resolver; using existing values") } else { - mainLog.Load().Info().Msgf("Reinitialized OS resolver with nameservers: %v", ns) + journal(mainLog.Load().Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns) } } } diff --git a/cmd/cli/dns_target_darwin.go b/cmd/cli/dns_target_darwin.go index e39292c..ca8b00e 100644 --- a/cmd/cli/dns_target_darwin.go +++ b/cmd/cli/dns_target_darwin.go @@ -52,8 +52,7 @@ func (p *prog) loadInterceptDNSTargetStateLocked() { if err := json.Unmarshal(data, &st); err != nil || st.Service == "" || st.Value == "" { return } - p.interceptDNSTargetService = st.Service - p.interceptDNSTargetSetValue = st.Value + p.setInterceptDNSTargetLocked(st.Service, st.Value) mainLog.Load().Debug().Msgf("intercept DNS target: restored tracking of %s on %q from previous run", st.Value, st.Service) } @@ -177,10 +176,11 @@ func (p *prog) ensureInterceptDNSTarget(systemDiscovery []string) { mainLog.Load().Warn().Err(err).Msgf("intercept DNS target: could not set %s on %q", target, iface.Name) return } - p.interceptDNSTargetService = iface.Name - p.interceptDNSTargetSetValue = target + p.setInterceptDNSTargetLocked(iface.Name, target) p.persistInterceptDNSTargetStateLocked() - mainLog.Load().Warn().Msgf("intercept DNS target: service %q provides no usable IPv4 DNS; set %s so macOS can emit DNS queries (removed automatically when the network provides IPv4 DNS)", iface.Name, target) + journal(mainLog.Load().Warn()).Str("service", iface.Name).Str("target", target). + Str("reason", "dns_less_network"). + Msgf("intercept DNS target: service %q provides no usable IPv4 DNS; set %s so macOS can emit DNS queries (removed automatically when the network provides IPv4 DNS)", iface.Name, target) } // removeInterceptDNSTarget removes a previously set intercept DNS target, @@ -210,8 +210,12 @@ func (p *prog) removeInterceptDNSTargetLocked(reason string) { return } if !isInterceptDNSTargetOnly(cur, val) { - mainLog.Load().Debug().Msgf("intercept DNS target: %q DNS changed externally; not removing (%s)", svc, reason) p.clearInterceptDNSTargetStateLocked() + // ctrld owns the DNS of the service no longer, and a later outage + // report needs the moment that ownership ended. + journal(mainLog.Load().Info()).Str("service", svc).Str("target", val). + Str("reason", "external_change"). + Msgf("intercept DNS target: %q DNS changed externally; not removing (%s)", svc, reason) return } if saved := interceptSavedStaticNameserversFn(iface); len(saved) > 0 { @@ -224,11 +228,20 @@ func (p *prog) removeInterceptDNSTargetLocked(reason string) { return } p.clearInterceptDNSTargetStateLocked() - mainLog.Load().Info().Msgf("intercept DNS target: removed %s from %q (%s)", val, svc, reason) + journal(mainLog.Load().Info()).Str("service", svc).Str("target", val).Str("reason", reason). + Msgf("intercept DNS target: removed %s from %q (%s)", val, svc, reason) } func (p *prog) clearInterceptDNSTargetStateLocked() { - p.interceptDNSTargetService = "" - p.interceptDNSTargetSetValue = "" + p.setInterceptDNSTargetLocked("", "") p.persistInterceptDNSTargetStateLocked() } + +// setInterceptDNSTargetLocked stores the service that ctrld owns and the value +// it wrote, and publishes the value for the readers that take no lock. Callers +// must hold interceptDNSTargetMu. +func (p *prog) setInterceptDNSTargetLocked(service, value string) { + p.interceptDNSTargetService = service + p.interceptDNSTargetSetValue = value + p.publishInterceptTarget(value) +} diff --git a/cmd/cli/dns_target_darwin_test.go b/cmd/cli/dns_target_darwin_test.go index 6f37a80..a04f785 100644 --- a/cmd/cli/dns_target_darwin_test.go +++ b/cmd/cli/dns_target_darwin_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" "github.com/Control-D-Inc/ctrld" @@ -125,8 +126,7 @@ func persistInterceptTargetForTest(t *testing.T, p *prog, service, value string) p.interceptDNSTargetMu.Lock() defer p.interceptDNSTargetMu.Unlock() p.interceptDNSTargetLoaded = true - p.interceptDNSTargetService = service - p.interceptDNSTargetSetValue = value + p.setInterceptDNSTargetLocked(service, value) p.persistInterceptDNSTargetStateLocked() } @@ -246,3 +246,104 @@ func TestRemoveInterceptDNSTargetRetainsStateOnFailure(t *testing.T) { }) } } + +// interceptTargetEvent returns the single event whose message starts with the +// given text. The two target events name the service and the value in their +// message, so no test can match a whole message. +func interceptTargetEvent(t *testing.T, logs *syncBuffer, prefix string) map[string]any { + t.Helper() + var found []map[string]any + for _, event := range jsonLogEvents(t, logs, "") { + message, _ := event["message"].(string) + if strings.HasPrefix(message, prefix) { + found = append(found, event) + } + } + if len(found) != 1 { + t.Fatalf("events with prefix %q: got %d, want 1", prefix, len(found)) + } + return found[0] +} + +func TestEnsureInterceptDNSTargetJournalsTheSetTarget(t *testing.T) { + logs := captureDebugMainLog(t) + newInterceptTargetHarness(t) + p := newInterceptTargetProg() + + p.ensureInterceptDNSTarget([]string{}) + + event := interceptTargetEvent(t, logs, "intercept DNS target: service ") + wantField(t, event, "journal", true) + wantField(t, event, "service", "Wi-Fi") + wantField(t, event, "target", "127.0.0.53") + wantField(t, event, "reason", "dns_less_network") +} + +func TestRemoveInterceptDNSTargetJournalsTheRemovedTarget(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"127.0.0.53"} + + p.removeInterceptDNSTarget("network has usable static IPv4 DNS") + + event := interceptTargetEvent(t, logs, "intercept DNS target: removed ") + wantField(t, event, "journal", true) + wantField(t, event, "service", "Wi-Fi") + wantField(t, event, "target", "127.0.0.53") + wantField(t, event, "reason", "network has usable static IPv4 DNS") +} + +// TestInterceptDNSTargetSnapshotFollowsTheWritesWithoutTheLock covers the read +// that a network report uses. The target mutex is held across networksetup +// calls that take seconds, so a report that takes it waits for them. +func TestInterceptDNSTargetSnapshotFollowsTheWritesWithoutTheLock(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + h.dns["Wi-Fi"] = nil + + p.ensureInterceptDNSTarget([]string{}) + + p.interceptDNSTargetMu.Lock() + written := p.interceptDNSTargetSetValue + got := p.interceptTargetSnapshot() + p.interceptDNSTargetMu.Unlock() + if written == "" { + t.Fatal("the pass wrote no target") + } + if got != written { + t.Fatalf("target snapshot = %q, want %q", got, written) + } + + p.removeInterceptDNSTarget("test") + + p.interceptDNSTargetMu.Lock() + got = p.interceptTargetSnapshot() + p.interceptDNSTargetMu.Unlock() + if got != "" { + t.Fatalf("target snapshot after the removal = %q, want an empty value", got) + } +} + +// TestRemoveInterceptDNSTargetJournalsAnExternalChange covers the branch that +// drops the ownership record. The user or another program owns the DNS of the +// service from that moment, and the journal held nothing about it. +func TestRemoveInterceptDNSTargetJournalsAnExternalChange(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"9.9.9.9"} + + p.removeInterceptDNSTarget("intercept shutdown") + + event := oneRecoveryEvent(t, logs, `intercept DNS target: "Wi-Fi" DNS changed externally; not removing (intercept shutdown)`) + wantField(t, event, "journal", true) + wantField(t, event, "service", "Wi-Fi") + wantField(t, event, "target", "127.0.0.53") + wantField(t, event, "reason", "external_change") + if p.interceptTargetSnapshot() != "" { + t.Fatal("the ownership record survived an external change") + } +} diff --git a/cmd/cli/intercept_target.go b/cmd/cli/intercept_target.go new file mode 100644 index 0000000..7fd3498 --- /dev/null +++ b/cmd/cli/intercept_target.go @@ -0,0 +1,15 @@ +package cli + +// publishInterceptTarget mirrors the loopback DNS target that ctrld wrote into +// a network service. Callers hold interceptDNSTargetMu. +func (p *prog) publishInterceptTarget(target string) { + p.interceptTargetMirror.Store(target) +} + +// interceptTargetSnapshot reads the mirrored loopback DNS target. It takes no +// lock, so a report of the network never waits on the networksetup calls that +// the target mutex covers. The value is empty while ctrld holds no service. +func (p *prog) interceptTargetSnapshot() string { + target, _ := p.interceptTargetMirror.Load().(string) + return target +} diff --git a/cmd/cli/log_header.go b/cmd/cli/log_header.go index f9a58f6..317da3d 100644 --- a/cmd/cli/log_header.go +++ b/cmd/cli/log_header.go @@ -2,7 +2,6 @@ package cli import ( "bytes" - "cmp" "net" "net/netip" "os" @@ -13,8 +12,8 @@ import ( "time" "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" - "tailscale.com/net/netmon" ) // processStartTime is the time this process started. A reader compares it with @@ -42,21 +41,6 @@ const ( // macOS, and a header refresh follows every network change. var headerOSVersion = sync.OnceValue(osVersion) -// logHeaderInterface is one interface of the header network snapshot. -type logHeaderInterface struct { - Name string - Up bool - IPs []string -} - -// logHeaderNetwork is the minimal network snapshot of a header line. -type logHeaderNetwork struct { - DefaultRouteInterface string - HaveV4 bool - HaveV6 bool - Interfaces []logHeaderInterface -} - // logHeaderInput holds every value of a header line. The caller collects the // values, so the renderer stays free of locks and of system calls. type logHeaderInput struct { @@ -72,7 +56,7 @@ type logHeaderInput struct { UpstreamTypes []string ResolverUID string LogFiles []string - Network logHeaderNetwork + Network networkSnapshot Trigger string } @@ -95,7 +79,7 @@ func renderLogHeader(in logHeaderInput) []byte { Strs("upstream_types", in.UpstreamTypes). Str("resolver_uid", in.ResolverUID). Strs("log_files", in.LogFiles). - Dict("network", logHeaderNetworkDict(in.Network)) + Dict("network", snapshotDict(in.Network)) if in.Trigger != "" { event = event.Str("trigger", in.Trigger) } @@ -113,86 +97,13 @@ func logHeaderEvent(logger *zerolog.Logger) *zerolog.Event { return logger.Log().Str(zerolog.LevelFieldName, zerolog.LevelInfoValue) } -func logHeaderNetworkDict(network logHeaderNetwork) *zerolog.Event { - interfaces := zerolog.Arr() - for _, iface := range network.Interfaces { - interfaces = interfaces.Dict(zerolog.Dict(). - Str("name", iface.Name). - Bool("up", iface.Up). - Strs("ips", iface.IPs)) - } - return zerolog.Dict(). - Str("default_route_interface", network.DefaultRouteInterface). - Bool("have_v4", network.HaveV4). - Bool("have_v6", network.HaveV6). - Array("interfaces", interfaces) -} - -// logHeaderNetworkFromState makes the header snapshot from a netmon state. -func logHeaderNetworkFromState(st *netmon.State) logHeaderNetwork { - if st == nil { - return logHeaderNetwork{} - } - network := logHeaderNetwork{ - DefaultRouteInterface: st.DefaultRouteInterface, - HaveV4: st.HaveV4, - HaveV6: st.HaveV6, - } - for name, iface := range st.Interface { - network.Interfaces = append(network.Interfaces, logHeaderInterface{ - Name: name, - Up: iface.Interface != nil && iface.IsUp(), - IPs: prefixStrings(st.InterfaceIPs[name]), - }) - } - slices.SortFunc(network.Interfaces, func(a, b logHeaderInterface) int { - return cmp.Compare(a.Name, b.Name) - }) - return network -} - -// logHeaderNetworkFresh reads the network now. A header that no network -// callback reached yet, and the header of an upload, need the live picture. -func logHeaderNetworkFresh() logHeaderNetwork { - state, err := readNetworkSourceStateFn() - if err != nil { - state = nil - } - network := logHeaderNetworkFromState(state) - if route, err := netmon.DefaultRouteInterface(); err == nil { - network.DefaultRouteInterface = route - } - network.HaveV4, network.HaveV6 = globalAddressFamilies(state) - return network -} - -// globalAddressFamilies reports which families the host can reach. A loopback -// or a link-local address reaches no other host, so it proves nothing. -func globalAddressFamilies(st *netmon.State) (haveV4, haveV6 bool) { - if st == nil { - return false, false - } - for name, iface := range st.Interface { - if iface.Interface == nil || !iface.IsUp() { - continue - } - for _, prefix := range st.InterfaceIPs[name] { - addr := prefix.Addr().Unmap() - if !addr.IsGlobalUnicast() || addr.IsLoopback() || addr.IsLinkLocalUnicast() { - continue - } - if addr.Is4() { - haveV4 = true - continue - } - haveV6 = true - } - } - return haveV4, haveV6 -} - // logHeaderInput collects every header value of the running program. func (p *prog) logHeaderInput() logHeaderInput { + return p.logHeaderInputWith(p.networkSnapshot()) +} + +// logHeaderInputWith collects every header value around one network snapshot. +func (p *prog) logHeaderInputWith(network networkSnapshot) logHeaderInput { in := logHeaderInput{ Version: curVersion(), Commit: commit, @@ -204,7 +115,7 @@ func (p *prog) logHeaderInput() logHeaderInput { // every artifact, so the header keeps its first characters only. ResolverUID: redactToken(cdUID), LogFiles: p.logHeaderFiles(), - Network: p.logHeaderNetwork(), + Network: network, } p.headerConfigInto(&in) return in @@ -218,35 +129,15 @@ func (p *prog) headerConfigInto(in *logHeaderInput) { logHeaderFromConfig(p.cfg, in) } -// sendLogHeader renders the header that leads one upload. +// sendLogHeader renders the header that leads one upload. The stored state can +// be older than the request, and support reads this line to learn the network +// of the moment the user asked. func (p *prog) sendLogHeader() []byte { - in := p.logHeaderInput() - // The stored state can be older than the request, and support reads this - // line to learn the network of the moment the user asked. - in.Network = logHeaderNetworkFresh() + in := p.logHeaderInputWith(buildNetworkSnapshot(p.freshSnapshotInputs())) in.Trigger = logHeaderTriggerSend return renderLogHeader(in) } -// noteNetworkState keeps the newest network that ctrld saw and puts it in the -// header bytes of every open log file. The caller must hold no lock, because -// the render reads the config and the network. -func (p *prog) noteNetworkState(st *netmon.State) { - if st != nil { - p.lastNetworkState.Store(st) - } - p.refreshLogHeader() -} - -// logHeaderNetwork takes the network that the last monitor callback reported. -func (p *prog) logHeaderNetwork() logHeaderNetwork { - state := p.lastNetworkState.Load() - if state == nil { - return logHeaderNetworkFresh() - } - return logHeaderNetworkFromState(state) -} - // logHeaderFiles names the files that this run writes, debug first. func (p *prog) logHeaderFiles() []string { files := make([]string, 0, 3) diff --git a/cmd/cli/log_header_test.go b/cmd/cli/log_header_test.go index 48d4968..867962d 100644 --- a/cmd/cli/log_header_test.go +++ b/cmd/cli/log_header_test.go @@ -40,14 +40,26 @@ type logHeaderTestLine struct { ResolverUID string `json:"resolver_uid"` LogFiles []string `json:"log_files"` Network struct { - DefaultRouteInterface string `json:"default_route_interface"` - HaveV4 bool `json:"have_v4"` - HaveV6 bool `json:"have_v6"` - Interfaces []struct { - Name string `json:"name"` - Up bool `json:"up"` - IPs []string `json:"ips"` + DefaultRouteV4 string `json:"default_route_v4"` + DefaultRouteV6 string `json:"default_route_v6"` + GatewayV4 string `json:"gateway_v4"` + GatewayV6 string `json:"gateway_v6"` + HaveV4 bool `json:"have_v4"` + HaveV6 bool `json:"have_v6"` + Interfaces []struct { + Name string `json:"name"` + Class string `json:"class"` + Up bool `json:"up"` + IPs []string `json:"ips"` + HardwarePort string `json:"hardware_port"` + Service string `json:"service"` } `json:"interfaces"` + Resolvers []string `json:"resolvers"` + SourceIPv4 string `json:"source_ipv4"` + SourceIPv6 string `json:"source_ipv6"` + InterceptTarget string `json:"intercept_target"` + LinkType string `json:"link_type"` + Tethered bool `json:"tethered"` } `json:"network"` Trigger string `json:"trigger"` } @@ -66,12 +78,15 @@ func logHeaderTestInput() logHeaderInput { UpstreamTypes: []string{"doh", "os"}, ResolverUID: redactToken("abcd1234"), LogFiles: []string{"/tmp/ctrld.log", "/tmp/ctrld-journal.log"}, - Network: logHeaderNetwork{ - DefaultRouteInterface: "en0", - HaveV4: true, - Interfaces: []logHeaderInterface{ - {Name: "en0", Up: true, IPs: []string{"192.0.2.10/24"}}, + Network: networkSnapshot{ + DefaultRouteV4: "en0", + GatewayV4: "192.0.2.1", + HaveV4: true, + Interfaces: []snapshotInterface{ + {Name: "en0", Class: "hardware", Up: true, IPs: []string{"192.0.2.10/24"}, HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, }, + Resolvers: []string{"192.0.2.1:53"}, + LinkType: "wifi", }, } } @@ -135,9 +150,14 @@ func Test_renderLogHeaderHoldsEveryField(t *testing.T) { // characters only. {"resolver_uid", parsed.ResolverUID, "abcd***"}, {"log_files", parsed.LogFiles, in.LogFiles}, - {"network.default_route_interface", parsed.Network.DefaultRouteInterface, in.Network.DefaultRouteInterface}, + {"network.default_route_v4", parsed.Network.DefaultRouteV4, in.Network.DefaultRouteV4}, + {"network.default_route_v6", parsed.Network.DefaultRouteV6, in.Network.DefaultRouteV6}, + {"network.gateway_v4", parsed.Network.GatewayV4, in.Network.GatewayV4}, {"network.have_v4", parsed.Network.HaveV4, in.Network.HaveV4}, {"network.have_v6", parsed.Network.HaveV6, in.Network.HaveV6}, + {"network.resolvers", parsed.Network.Resolvers, in.Network.Resolvers}, + {"network.link_type", parsed.Network.LinkType, in.Network.LinkType}, + {"network.tethered", parsed.Network.Tethered, in.Network.Tethered}, } for _, c := range checks { if !reflect.DeepEqual(c.got, c.want) { @@ -149,8 +169,11 @@ func Test_renderLogHeaderHoldsEveryField(t *testing.T) { t.Fatalf("network.interfaces holds %d entries, want 1", len(parsed.Network.Interfaces)) } iface := parsed.Network.Interfaces[0] - if iface.Name != "en0" || !iface.Up || !reflect.DeepEqual(iface.IPs, []string{"192.0.2.10/24"}) { - t.Errorf("network.interfaces[0] = %+v, want en0 up with 192.0.2.10/24", iface) + if iface.Name != "en0" || iface.Class != "hardware" || !iface.Up || iface.HardwarePort != "Wi-Fi" { + t.Errorf("network.interfaces[0] = %+v, want the hardware port en0 up on Wi-Fi", iface) + } + if !reflect.DeepEqual(iface.IPs, []string{"192.0.2.10/24"}) { + t.Errorf("network.interfaces[0].ips = %v, want 192.0.2.10/24", iface.IPs) } if parsed.Time == "" { t.Error("time is empty") @@ -245,8 +268,12 @@ func Test_logHeaderFromConfigReportsOffWhenUnset(t *testing.T) { } } -func Test_logHeaderNetworkFromState(t *testing.T) { +// Test_renderLogHeaderShowsTheSnapshotOfAState renders the header of one +// network state, so a reader of the file sees every interface of the host in +// one order. +func Test_renderLogHeaderShowsTheSnapshotOfAState(t *testing.T) { setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + stubHeaderSnapshotSources(t) state := &netmon.State{ DefaultRouteInterface: "en0", HaveV4: true, @@ -255,52 +282,40 @@ func Test_logHeaderNetworkFromState(t *testing.T) { "awdl0": {Interface: &net.Interface{Name: "awdl0"}}, }, InterfaceIPs: map[string][]netip.Prefix{ - "en0": {netip.MustParsePrefix("192.0.2.10/24"), netip.MustParsePrefix("2001:db8::1/64")}, + "en0": {netip.MustParsePrefix("192.0.2.10/24"), netip.MustParsePrefix("2001:db8::1/64")}, + "awdl0": {netip.MustParsePrefix("fe80::2/64")}, }, } - network := logHeaderNetworkFromState(state) - - want := logHeaderNetwork{ - DefaultRouteInterface: "en0", - HaveV4: true, - Interfaces: []logHeaderInterface{ - {Name: "awdl0", Up: false, IPs: []string{}}, - {Name: "en0", Up: true, IPs: []string{"192.0.2.10/24", "2001:db8::1/64"}}, - }, - } - if !reflect.DeepEqual(network, want) { - t.Fatalf("network = %+v, want %+v", network, want) - } - in := logHeaderTestInput() - in.Network = network + in.Network = buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "192.0.2.1", Interface: "en0"}, + RouteV6: defaultRoute{Gateway: "fe80::1%en0", Interface: "en0"}, + Meta: interfaceMetaFor, + }) parsed, _ := parseLogHeader(t, renderLogHeader(in)) - if parsed.Network.DefaultRouteInterface != "en0" { - t.Errorf("network.default_route_interface = %q, want %q", parsed.Network.DefaultRouteInterface, "en0") + + if parsed.Network.DefaultRouteV4 != "en0" || parsed.Network.DefaultRouteV6 != "en0" { + t.Errorf("default route = %q, %q, want en0 for both families", parsed.Network.DefaultRouteV4, parsed.Network.DefaultRouteV6) } if len(parsed.Network.Interfaces) != 2 { t.Fatalf("network.interfaces holds %d entries, want 2", len(parsed.Network.Interfaces)) } - if parsed.Network.Interfaces[0].Name != "awdl0" || parsed.Network.Interfaces[0].Up { - t.Errorf("network.interfaces[0] = %+v, want awdl0 down", parsed.Network.Interfaces[0]) + if parsed.Network.Interfaces[0].Name != "awdl0" || parsed.Network.Interfaces[0].Class != "airdrop" || parsed.Network.Interfaces[0].Up { + t.Errorf("network.interfaces[0] = %+v, want awdl0 down in the airdrop class", parsed.Network.Interfaces[0]) } if got := parsed.Network.Interfaces[1].IPs; !reflect.DeepEqual(got, []string{"192.0.2.10/24", "2001:db8::1/64"}) { t.Errorf("network.interfaces[1].ips = %v", got) } } -func Test_logHeaderNetworkFromStateWithoutState(t *testing.T) { - if network := logHeaderNetworkFromState(nil); !reflect.DeepEqual(network, logHeaderNetwork{}) { - t.Errorf("network = %+v, want the zero value", network) - } -} - // setupLogHeaderWiringTest puts ctrld in cd mode with a temporary home // directory, so initInternalLogging writes its files there. Every global that // the logging setup touches goes back when the test ends. func setupLogHeaderWiringTest(t *testing.T) string { t.Helper() + stubHeaderSnapshotSources(t) origSilent, origCdUID, origHomedir, origVerbose := silent, cdUID, homedir, verbose origMainLog, origProxyLogger := mainLog.Load(), ctrld.ProxyLogger.Load() origLevel := zerolog.GlobalLevel() @@ -492,14 +507,22 @@ func Test_logHeaderWiring_restartAppendsHeader(t *testing.T) { func Test_logHeaderWiring_usesLastNetworkState(t *testing.T) { setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + stubHeaderSnapshotSources(t) origRead := readNetworkSourceStateFn t.Cleanup(func() { readNetworkSourceStateFn = origRead }) p := &prog{cfg: &ctrld.Config{}} - p.lastNetworkState.Store(&netmon.State{DefaultRouteInterface: "en7"}) + p.lastNetworkState.Store(&netmon.State{ + DefaultRouteInterface: "en7", + Interface: map[string]netmon.Interface{"en7": {Interface: &net.Interface{Name: "en7", Flags: net.FlagUp}}}, + InterfaceIPs: map[string][]netip.Prefix{"en7": {netip.MustParsePrefix("192.0.2.7/24")}}, + }) - if got := p.logHeaderInput().Network.DefaultRouteInterface; got != "en7" { - t.Errorf("default_route_interface = %q, want %q", got, "en7") + if got := p.logHeaderInput().Network.DefaultRouteV4; got != "en7" { + t.Errorf("default_route_v4 = %q, want %q", got, "en7") + } + if got := p.logHeaderInput().Network.GatewayV4; got != "192.0.2.1" { + t.Errorf("gateway_v4 = %q, want %q", got, "192.0.2.1") } p.lastNetworkState.Store(nil) @@ -522,10 +545,22 @@ func Test_logHeaderWiring_usesLastNetworkState(t *testing.T) { } } +// stubHeaderSnapshotSources keeps the header snapshot away from the host. One +// render reads the hardware ports, the route table, and the NAT64 class, and no +// test may run the commands behind them. +func stubHeaderSnapshotSources(t *testing.T) { + t.Helper() + stubSnapshotDNS64(t) + stubSnapshotGateways(t, "192.0.2.1", "") + stubSnapshotPlatformMeta(t, nil) + stubSnapshotVirtualSet(t) +} + // stubLogHeaderNetworkRead makes the fresh network read return one up // interface, so a header test never depends on the host network. func stubLogHeaderNetworkRead(t *testing.T, name string) { t.Helper() + stubHeaderSnapshotSources(t) origRead := readNetworkSourceStateFn t.Cleanup(func() { readNetworkSourceStateFn = origRead }) readNetworkSourceStateFn = func() (*netmon.State, error) { @@ -538,6 +573,7 @@ func stubLogHeaderNetworkRead(t *testing.T, name string) { func Test_sendLogHeader_usesFreshNetworkAndTrigger(t *testing.T) { setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + stubHeaderSnapshotSources(t) stubLogHeaderNetworkRead(t, "en9") p := &prog{cfg: &ctrld.Config{}} p.lastNetworkState.Store(&netmon.State{ diff --git a/cmd/cli/log_journal.go b/cmd/cli/log_journal.go index 0f851be..995de48 100644 --- a/cmd/cli/log_journal.go +++ b/cmd/cli/log_journal.go @@ -6,10 +6,12 @@ import ( "regexp" "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" ) -// journalField marks an event for the retained journal stream. -const journalField = "journal" +// journalField marks an event for the retained journal stream. Both packages +// share the name, so one writer keeps the lines of both. +const journalField = ctrld.JournalField // journalMarker is the serialized form of the journal field. The journal // writer keeps a line below warn level only when the line contains it. @@ -30,7 +32,7 @@ func redactRetainedLine(p []byte) []byte { // journal marks an event for the retained journal stream. The event keeps // its own level, so an info event stays info on the console. func journal(e *zerolog.Event) *zerolog.Event { - return e.Bool(journalField, true) + return ctrld.Journal(e) } // journalLevelWriter passes on the lines that the journal keeps and drops the diff --git a/cmd/cli/log_journal_test.go b/cmd/cli/log_journal_test.go index ff6fd49..abd8f7e 100644 --- a/cmd/cli/log_journal_test.go +++ b/cmd/cli/log_journal_test.go @@ -195,6 +195,7 @@ func startInternalLogging(t *testing.T) (*prog, string) { cdUID = "test-uid" silent = false verbose = 0 + stubHeaderSnapshotSources(t) p := &prog{cfg: &ctrld.Config{}} p.initInternalLogging(nil) diff --git a/cmd/cli/log_writer_silent_test.go b/cmd/cli/log_writer_silent_test.go index 70eddb3..15bca70 100644 --- a/cmd/cli/log_writer_silent_test.go +++ b/cmd/cli/log_writer_silent_test.go @@ -59,6 +59,7 @@ func Test_initInternalLogging_silentCreatesNoFile(t *testing.T) { homedir = dir cdUID = "test-uid" // cd mode, which would otherwise enable internal logging silent = true + stubHeaderSnapshotSources(t) p := &prog{cfg: &ctrld.Config{}} p.initInternalLogging(nil) @@ -94,6 +95,7 @@ func Test_initInternalLogging_createsJournalFile(t *testing.T) { cdUID = "test-uid" silent = false verbose = 0 + stubHeaderSnapshotSources(t) p := &prog{cfg: &ctrld.Config{}} p.initInternalLogging(nil) diff --git a/cmd/cli/net_darwin.go b/cmd/cli/net_darwin.go index 6233161..8d9c4ab 100644 --- a/cmd/cli/net_darwin.go +++ b/cmd/cli/net_darwin.go @@ -3,14 +3,52 @@ package cli import ( "bufio" "bytes" + "context" + "errors" "io" "net" "os/exec" "strings" + "sync" + "time" +) + +// hardwarePortInfo holds the two names that macOS gives a network device: the +// hardware port of the adapter and the network service that uses it. +type hardwarePortInfo struct { + Port string + Service string +} + +// A network event reads the ports of every interface, so the two networksetup +// calls run at most once per window. +const hardwarePortsTTL = time.Minute + +// networksetupTimeout bounds one command. A stuck read must not hold the +// network callback that waits for the port names. +const networksetupTimeout = 5 * time.Second + +// maxNetworksetupOutput bounds one read. A host with many ports stays far +// below it, so more than this is a broken command. +const maxNetworksetupOutput = 1 << 20 + +// maxNetworksetupLine bounds one line, so one token fills no memory. +const maxNetworksetupLine = 64 * 1024 + +var errNetworksetupOutputTooLarge = errors.New("networksetup output above the bound") + +var ( + hardwarePortsMu sync.Mutex + hardwarePortsCache map[string]hardwarePortInfo + hardwarePortsReadAt time.Time + + // Tests replace these seams to avoid a networksetup call. + readHardwarePortsFn = readHardwarePorts + hardwarePortsNowFn = time.Now ) func patchNetIfaceName(iface *net.Interface) (bool, error) { - b, err := exec.Command("networksetup", "-listnetworkserviceorder").Output() + b, err := networksetupOutput("-listnetworkserviceorder") if err != nil { return false, err } @@ -24,7 +62,14 @@ func patchNetIfaceName(iface *net.Interface) (bool, error) { } func networkServiceName(ifaceName string, r io.Reader) string { - scanner := bufio.NewScanner(r) + return serviceNamesByDevice(r)[ifaceName] +} + +// serviceNamesByDevice maps each device of "networksetup +// -listnetworkserviceorder" to the name of the service that owns it. +func serviceNamesByDevice(r io.Reader) map[string]string { + names := make(map[string]string) + scanner := scanNetworksetup(r) prevLine := "" for scanner.Scan() { line := scanner.Text() @@ -32,16 +77,198 @@ func networkServiceName(ifaceName string, r io.Reader) string { // Network services is disabled. continue } - if !strings.Contains(line, "Device: "+ifaceName) { + device := serviceOrderDevice(line) + if device == "" { prevLine = line continue } - parts := strings.SplitN(prevLine, " ", 2) - if len(parts) == 2 { - return strings.TrimSpace(parts[1]) + if _, seen := names[device]; !seen { + names[device] = serviceOrderName(prevLine) } + prevLine = line } - return "" + if scanner.Err() != nil { + // A truncated read names fewer services than the host has. + return nil + } + return names +} + +// scanNetworksetup reads the output line by line under a bounded buffer. +func scanNetworksetup(r io.Reader) *bufio.Scanner { + scanner := bufio.NewScanner(r) + scanner.Buffer(nil, maxNetworksetupLine) + return scanner +} + +// serviceOrderDevice reads the device out of a line like +// "(Hardware Port: Wi-Fi, Device: en0)". +func serviceOrderDevice(line string) string { + _, after, found := strings.Cut(line, "Device:") + if !found { + return "" + } + return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(after), ")")) +} + +// serviceOrderName reads the service name out of a line like "(1) Wi-Fi". +func serviceOrderName(line string) string { + parts := strings.SplitN(line, " ", 2) + if len(parts) != 2 { + return "" + } + return strings.TrimSpace(parts[1]) +} + +// parseHardwarePorts maps each device of "networksetup -listallhardwareports" +// to its hardware port. Each block names the port before the device. +func parseHardwarePorts(r io.Reader) map[string]hardwarePortInfo { + ports := make(map[string]hardwarePortInfo) + scanner := scanNetworksetup(r) + port := "" + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if after, ok := strings.CutPrefix(line, "Hardware Port:"); ok { + port = strings.TrimSpace(after) + continue + } + after, ok := strings.CutPrefix(line, "Device:") + if !ok { + continue + } + device := strings.TrimSpace(after) + if device == "" { + continue + } + ports[device] = hardwarePortInfo{Port: port} + port = "" + } + if scanner.Err() != nil { + // A truncated read names fewer ports than the host has. + return nil + } + return ports +} + +// addServiceNames fills the service of every known device. A device that no +// hardware port lists is not a port of this host. +func addServiceNames(m map[string]hardwarePortInfo, r io.Reader) { + for device, service := range serviceNamesByDevice(r) { + info, known := m[device] + if !known { + continue + } + info.Service = service + m[device] = info + } +} + +// hardwarePortsByDevice returns the port and service of every device. The +// commands run outside the mutex, so a slow read blocks no other caller. +func hardwarePortsByDevice() map[string]hardwarePortInfo { + if ports, fresh := cachedHardwarePorts(); fresh { + return ports + } + return storeHardwarePorts(readHardwarePortsFn()) +} + +// cachedHardwarePorts reports whether the read window still holds. +func cachedHardwarePorts() (map[string]hardwarePortInfo, bool) { + hardwarePortsMu.Lock() + defer hardwarePortsMu.Unlock() + if hardwarePortsReadAt.IsZero() { + return nil, false + } + return hardwarePortsCache, hardwarePortsNowFn().Sub(hardwarePortsReadAt) < hardwarePortsTTL +} + +// storeHardwarePorts starts the window again and keeps the last good map. A +// failing command must run once per window, not once per lookup. +func storeHardwarePorts(ports map[string]hardwarePortInfo) map[string]hardwarePortInfo { + hardwarePortsMu.Lock() + defer hardwarePortsMu.Unlock() + hardwarePortsReadAt = hardwarePortsNowFn() + if ports != nil { + hardwarePortsCache = ports + } + return hardwarePortsCache +} + +// readHardwarePorts runs the two networksetup commands. +func readHardwarePorts() map[string]hardwarePortInfo { + b, err := networksetupOutput("-listallhardwareports") + if err != nil { + return nil + } + ports := parseHardwarePorts(bytes.NewReader(b)) + if ports == nil { + // A truncated list must keep the last good map. + return nil + } + b, err = networksetupOutput("-listnetworkserviceorder") + if err != nil { + return ports + } + addServiceNames(ports, bytes.NewReader(b)) + return ports +} + +// networksetupOutput runs one networksetup list under a time bound and a size +// bound. +func networksetupOutput(arg string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), networksetupTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "networksetup", arg) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + out, readErr := readBoundedOutput(stdout) + if readErr != nil { + // A command ends only when something reads the rest of its output. + _, _ = io.Copy(io.Discard, stdout) + } + if err := cmd.Wait(); err != nil { + return nil, err + } + return out, readErr +} + +// readBoundedOutput reads at most maxNetworksetupOutput bytes and reports an +// error above that. +func readBoundedOutput(r io.Reader) ([]byte, error) { + out, err := io.ReadAll(io.LimitReader(r, maxNetworksetupOutput+1)) + if err != nil { + return nil, err + } + if len(out) > maxNetworksetupOutput { + return nil, errNetworksetupOutputTooLarge + } + return out, nil +} + +// platformInterfaceMeta returns the macOS names of an interface. The class +// comes from the interface name, so it stays empty here. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + info := hardwarePortsByDevice()[name] + return "", info.Port, info.Service +} + +// platformVirtualInterfaces returns no set. The name rules already class the +// AirDrop and the link-local interfaces of macOS. +func platformVirtualInterfaces() map[string]struct{} { + return nil +} + +// refreshInterfaceMeta drops the read window. The cache has no port name for +// an adapter that appeared, so the next lookup reads the ports again. +func refreshInterfaceMeta() { + hardwarePortsMu.Lock() + defer hardwarePortsMu.Unlock() + hardwarePortsReadAt = time.Time{} } // validInterface reports whether the *net.Interface is a valid one. @@ -50,27 +277,16 @@ func validInterface(iface *net.Interface, validIfacesMap map[string]struct{}) bo return ok } -// validInterfacesMap returns a set of all valid hardware ports. +// validInterfacesMap returns a set of all valid hardware ports. The devices +// come from the port cache, so a network event starts no second command. func validInterfacesMap() map[string]struct{} { - b, err := exec.Command("networksetup", "-listallhardwareports").Output() - if err != nil { + ports := hardwarePortsByDevice() + if ports == nil { return nil } - return parseListAllHardwarePorts(bytes.NewReader(b)) -} - -// parseListAllHardwarePorts parses output of "networksetup -listallhardwareports" -// and returns map presents all hardware ports. -func parseListAllHardwarePorts(r io.Reader) map[string]struct{} { - m := make(map[string]struct{}) - scanner := bufio.NewScanner(r) - for scanner.Scan() { - line := scanner.Text() - after, ok := strings.CutPrefix(line, "Device: ") - if !ok { - continue - } - m[after] = struct{}{} + devices := make(map[string]struct{}, len(ports)) + for device := range ports { + devices[device] = struct{}{} } - return m + return devices } diff --git a/cmd/cli/net_darwin_test.go b/cmd/cli/net_darwin_test.go new file mode 100644 index 0000000..280e4d5 --- /dev/null +++ b/cmd/cli/net_darwin_test.go @@ -0,0 +1,315 @@ +package cli + +import ( + "maps" + "strings" + "testing" + "time" +) + +const listAllHardwarePortsFixture = ` +Hardware Port: Wi-Fi +Device: en0 +Ethernet Address: a0:78:17:68:56:3f + +Hardware Port: iPhone USB +Device: en7 +Ethernet Address: 6e:5a:6b:12:34:56 + +Hardware Port: Thunderbolt Bridge +Device: bridge0 +Ethernet Address: 36:21:bb:3a:7a:40 + +Hardware Port: Ethernet Adapter (en6) +Device: en6 +Ethernet Address: 3a:3e:fc:1e:ab:41 + +Hardware Port: com.wireguard.macos +Device: + +VLAN Configurations +=================== +` + +const listNetworkServiceOrderFixture = `An asterisk (*) denotes that a network service is disabled. +(1) Wi-Fi +(Hardware Port: Wi-Fi, Device: en0) + +(2) iPhone USB +(Hardware Port: iPhone USB, Device: en7) + +(3) Thunderbolt Bridge +(Hardware Port: Thunderbolt Bridge, Device: bridge0) + +(*) Ethernet Adapter (en6) +(Hardware Port: Ethernet Adapter (en6), Device: en6) + +(4) ca-001-stg +(Hardware Port: com.wireguard.macos, Device: ) + +` + +func Test_parseHardwarePorts(t *testing.T) { + ports := parseHardwarePorts(strings.NewReader(listAllHardwarePortsFixture)) + addServiceNames(ports, strings.NewReader(listNetworkServiceOrderFixture)) + + tests := []struct { + device string + wantPort string + wantService string + }{ + {"en0", "Wi-Fi", "Wi-Fi"}, + {"en7", "iPhone USB", "iPhone USB"}, + {"bridge0", "Thunderbolt Bridge", "Thunderbolt Bridge"}, + {"en6", "Ethernet Adapter (en6)", ""}, + } + if len(ports) != len(tests) { + t.Fatalf("parseHardwarePorts returned %d devices, want %d: %v", len(ports), len(tests), ports) + } + for _, tc := range tests { + t.Run(tc.device, func(t *testing.T) { + info := ports[tc.device] + if info.Port != tc.wantPort { + t.Errorf("port of %s = %q, want %q", tc.device, info.Port, tc.wantPort) + } + if info.Service != tc.wantService { + t.Errorf("service of %s = %q, want %q", tc.device, info.Service, tc.wantService) + } + }) + } +} + +func Test_networkServiceName(t *testing.T) { + tests := []struct { + device string + want string + }{ + {"en0", "Wi-Fi"}, + {"en7", "iPhone USB"}, + {"bridge0", "Thunderbolt Bridge"}, + {"en6", ""}, + {"en9", ""}, + } + for _, tc := range tests { + t.Run(tc.device, func(t *testing.T) { + got := networkServiceName(tc.device, strings.NewReader(listNetworkServiceOrderFixture)) + if got != tc.want { + t.Errorf("networkServiceName(%q) = %q, want %q", tc.device, got, tc.want) + } + }) + } +} + +func Test_parseHardwarePorts_dropsATruncatedRead(t *testing.T) { + longLine := strings.Repeat("x", maxNetworksetupLine+1) + + if got := parseHardwarePorts(strings.NewReader(listAllHardwarePortsFixture + longLine)); got != nil { + t.Errorf("parseHardwarePorts of a truncated read = %v, want nil", got) + } + if got := serviceNamesByDevice(strings.NewReader(listNetworkServiceOrderFixture + longLine)); got != nil { + t.Errorf("serviceNamesByDevice of a truncated read = %v, want nil", got) + } +} + +func Test_readBoundedOutput(t *testing.T) { + out, err := readBoundedOutput(strings.NewReader(listAllHardwarePortsFixture)) + if err != nil || string(out) != listAllHardwarePortsFixture { + t.Fatalf("readBoundedOutput of the fixture = %q, %v, want the fixture and no error", out, err) + } + if _, err := readBoundedOutput(strings.NewReader(strings.Repeat("x", maxNetworksetupOutput+1))); err == nil { + t.Fatal("readBoundedOutput of an output above the bound gave no error") + } +} + +// stubHardwarePorts replaces the command and clock seams of the hardware port +// cache, and restores them with the cache itself. +func stubHardwarePorts(t *testing.T, read func() map[string]hardwarePortInfo, now func() time.Time) { + t.Helper() + readBefore, nowBefore := readHardwarePortsFn, hardwarePortsNowFn + hardwarePortsMu.Lock() + cacheBefore, readAtBefore := hardwarePortsCache, hardwarePortsReadAt + hardwarePortsCache, hardwarePortsReadAt = nil, time.Time{} + hardwarePortsMu.Unlock() + t.Cleanup(func() { + readHardwarePortsFn, hardwarePortsNowFn = readBefore, nowBefore + hardwarePortsMu.Lock() + hardwarePortsCache, hardwarePortsReadAt = cacheBefore, readAtBefore + hardwarePortsMu.Unlock() + }) + readHardwarePortsFn, hardwarePortsNowFn = read, now +} + +func Test_hardwarePortsByDevice_caches(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi", Service: "Wi-Fi"}} + }, func() time.Time { return now }) + + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("first read port = %q, want Wi-Fi", got) + } + now = now.Add(59 * time.Second) + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads within the cache window = %d, want 1", reads) + } + now = now.Add(2 * time.Second) + hardwarePortsByDevice() + if reads != 2 { + t.Fatalf("reads after the cache window = %d, want 2", reads) + } +} + +func Test_hardwarePortsByDevice_keepsLastGoodRead(t *testing.T) { + now := time.Unix(1700000000, 0) + fail := false + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + if fail { + return nil + } + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi"}} + }, func() time.Time { return now }) + + hardwarePortsByDevice() + fail = true + now = now.Add(2 * time.Minute) + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("port after a failed read = %q, want Wi-Fi", got) + } +} + +func Test_hardwarePortsByDevice_keepsTheWindowAfterAFailedRead(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return nil + }, func() time.Time { return now }) + + hardwarePortsByDevice() + now = now.Add(time.Second) + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads of a failing command within the window = %d, want 1", reads) + } + now = now.Add(2 * time.Minute) + hardwarePortsByDevice() + if reads != 2 { + t.Fatalf("reads after the window = %d, want 2", reads) + } +} + +func Test_refreshInterfaceMeta_dropsTheReadWindow(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi"}} + }, func() time.Time { return now }) + + hardwarePortsByDevice() + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads within the window = %d, want 1", reads) + } + refreshInterfaceMeta() + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("port after a refresh = %q, want Wi-Fi", got) + } + if reads != 2 { + t.Fatalf("reads after a refresh = %d, want 2", reads) + } +} + +func Test_hardwarePortsByDevice_readsOutsideTheLock(t *testing.T) { + now := time.Unix(1700000000, 0) + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + // A command runs outside the mutex, so this read can take it. + if !hardwarePortsMu.TryLock() { + t.Error("the command ran under the hardware ports mutex") + } else { + hardwarePortsMu.Unlock() + } + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi"}} + }, func() time.Time { return now }) + + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("port = %q, want Wi-Fi", got) + } +} + +// fixtureHardwarePorts returns the port map of the two command fixtures. +func fixtureHardwarePorts() map[string]hardwarePortInfo { + ports := parseHardwarePorts(strings.NewReader(listAllHardwarePortsFixture)) + addServiceNames(ports, strings.NewReader(listNetworkServiceOrderFixture)) + return ports +} + +func Test_validInterfacesMap_comesFromThePortCache(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return fixtureHardwarePorts() + }, func() time.Time { return now }) + + want := map[string]struct{}{"en0": {}, "en7": {}, "bridge0": {}, "en6": {}} + if got := validInterfacesMap(); !maps.Equal(got, want) { + t.Fatalf("validInterfacesMap = %v, want %v", got, want) + } + validInterfacesMap() + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads for two maps and one lookup = %d, want 1", reads) + } +} + +func Test_validInterfacesMap_namesNoInterfaceWithoutARead(t *testing.T) { + now := time.Unix(1700000000, 0) + stubHardwarePorts(t, func() map[string]hardwarePortInfo { return nil }, func() time.Time { return now }) + + if got := validInterfacesMap(); len(got) != 0 { + t.Fatalf("validInterfacesMap after a failed read = %v, want an empty set", got) + } +} + +// Test_transitionTestGlobalsReadNoPorts counts the port reads of the tests +// that take the transition globals. A test must read the ports of no host. +func Test_transitionTestGlobalsReadNoPorts(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return fixtureHardwarePorts() + }, func() time.Time { return now }) + + TestNetworkChangeDoesNotRestoreDownDefaultRouteSources(t) + TestNetworkChangeSourceMovedToUpInterface(t) + + if reads != 0 { + t.Fatalf("port reads of two transition tests = %d, want 0", reads) + } +} + +func Test_platformInterfaceMeta(t *testing.T) { + now := time.Unix(1700000000, 0) + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + return map[string]hardwarePortInfo{"en7": {Port: "iPhone USB", Service: "iPhone USB"}} + }, func() time.Time { return now }) + + class, hardwarePort, service := platformInterfaceMeta("en7") + if class != "" { + t.Errorf("class = %q, want empty", class) + } + if hardwarePort != "iPhone USB" || service != "iPhone USB" { + t.Errorf("port, service = %q, %q, want iPhone USB twice", hardwarePort, service) + } + if class, hardwarePort, service = platformInterfaceMeta("utun3"); class != "" || hardwarePort != "" || service != "" { + t.Errorf("unknown device gave %q, %q, %q, want empty strings", class, hardwarePort, service) + } + if platformVirtualInterfaces() != nil { + t.Error("platformVirtualInterfaces is not nil on darwin") + } +} diff --git a/cmd/cli/net_linux.go b/cmd/cli/net_linux.go index ea17d3d..71e7601 100644 --- a/cmd/cli/net_linux.go +++ b/cmd/cli/net_linux.go @@ -4,11 +4,39 @@ import ( "net" "net/netip" "os" + "path/filepath" + "slices" + "strconv" "strings" "tailscale.com/net/netmon" ) +// The kernel gives an Ethernet-like link the type ARPHRD_ETHER. +const arphrdEther = 1 + +// Tests point the readers at a temporary directory. +var ( + sysfsNetRoot = "/sys/class/net" + sysfsVirtualNetRoot = "/sys/devices/virtual/net" +) + +// noiseInterfacePrefixes name the interfaces of a container host. They change +// every few seconds, and no traffic of the host goes through them. +var noiseInterfacePrefixes = []string{"docker", "veth", "virbr", "br-"} + +// usbTetherDrivers name the kernel modules of a phone that shares its data +// connection over the cable. +var usbTetherDrivers = []string{"ipheth", "rndis_host"} + +// sysfsInfo holds what sysfs knows about one interface. +type sysfsInfo struct { + Type int + DevType string + Driver string + Wireless bool +} + func patchNetIfaceName(iface *net.Interface) (bool, error) { return true, nil } // validInterface reports whether the *net.Interface is a valid one. @@ -42,7 +70,7 @@ func validInterfacesMap() map[string]struct{} { // virtualInterfaces returns a map of virtual interfaces on current machine. func virtualInterfaces() map[string]struct{} { s := make(map[string]struct{}) - entries, _ := os.ReadDir("/sys/devices/virtual/net") + entries, _ := os.ReadDir(sysfsVirtualNetRoot) for _, entry := range entries { if entry.IsDir() { s[strings.TrimSpace(entry.Name())] = struct{}{} @@ -50,3 +78,104 @@ func virtualInterfaces() map[string]struct{} { } return s } + +// sysfsInterfaceInfo reads the sysfs entry of an interface. A missing file +// leaves its field empty. +func sysfsInterfaceInfo(name string) sysfsInfo { + dir := filepath.Join(sysfsNetRoot, name) + info := sysfsInfo{DevType: sysfsUeventValue(dir, "DEVTYPE"), Driver: sysfsDriver(dir)} + info.Type, _ = strconv.Atoi(readSysfsAttribute(dir, "type")) + if entry, err := os.Stat(filepath.Join(dir, "wireless")); err == nil && entry.IsDir() { + info.Wireless = true + } + return info +} + +// readSysfsAttribute reads one attribute without its trailing newline. +func readSysfsAttribute(dir, file string) string { + b, err := os.ReadFile(filepath.Join(dir, file)) + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} + +// sysfsUeventValue reads one entry of the uevent file. +func sysfsUeventValue(dir, key string) string { + for _, line := range strings.Split(readSysfsAttribute(dir, "uevent"), "\n") { + after, ok := strings.CutPrefix(strings.TrimSpace(line), key+"=") + if !ok { + continue + } + return strings.TrimSpace(after) + } + return "" +} + +// sysfsDriver names the kernel module behind an interface. The driver link of +// the device points at the module, and the uevent of the device names it too. +func sysfsDriver(dir string) string { + device := filepath.Join(dir, "device") + if target, err := os.Readlink(filepath.Join(device, "driver")); err == nil { + return filepath.Base(target) + } + if driver := sysfsUeventValue(device, "DRIVER"); driver != "" { + return driver + } + return sysfsUeventValue(dir, "DRIVER") +} + +// sysfsHardwarePort names the link in the words that the link type mapping +// reads. The port names of macOS use the same words. +func sysfsHardwarePort(info sysfsInfo) string { + switch { + case info.Wireless: + return "Wi-Fi" + case slices.Contains(usbTetherDrivers, info.Driver): + return hardwarePortIPhoneUSB + case info.DevType != "": + return info.DevType + case info.Type == arphrdEther: + return "Ethernet" + } + return "" +} + +// sysfsClass names the kind of an interface. The name rule comes before the +// device type, so an interface that went away keeps its class. A bridge, a +// bond, and a VLAN carry the traffic of the host, so they stay hardware. +func sysfsClass(name string, info sysfsInfo) string { + switch { + case info.DevType == "tun" || info.DevType == "tap": + return "tunnel" + case hasInterfacePrefix(name, noiseInterfacePrefixes...): + return "virtual" + case info.Wireless || info.Type == arphrdEther: + return "hardware" + } + return "" +} + +// platformInterfaceMeta returns the Linux class and link name of an interface. +// Linux has no service name. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + info := sysfsInterfaceInfo(name) + return sysfsClass(name, info), sysfsHardwarePort(info), "" +} + +// platformVirtualInterfaces returns the container interfaces of the host. The +// kernel lists the uplink bridges and the bonds beside them, and those carry +// the traffic of the host. +func platformVirtualInterfaces() map[string]struct{} { + noise := make(map[string]struct{}) + for name := range virtualInterfaces() { + if hasInterfacePrefix(name, noiseInterfacePrefixes...) { + noise[name] = struct{}{} + } + } + return noise +} + +// refreshInterfaceMeta does no work. Each sysfs read answers from the kernel +// of this moment, so there is no cache to refresh. +func refreshInterfaceMeta() {} diff --git a/cmd/cli/net_linux_test.go b/cmd/cli/net_linux_test.go new file mode 100644 index 0000000..db2ba0f --- /dev/null +++ b/cmd/cli/net_linux_test.go @@ -0,0 +1,201 @@ +package cli + +import ( + "maps" + "os" + "path/filepath" + "testing" +) + +// writeSysfsInterface builds one interface entry under a fake sysfs root. +func writeSysfsInterface(t *testing.T, root, name string, files map[string]string, wireless bool) { + t.Helper() + dir := filepath.Join(root, name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("create %s: %v", dir, err) + } + for file, content := range files { + if err := os.WriteFile(filepath.Join(dir, file), []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", file, err) + } + } + if !wireless { + return + } + if err := os.MkdirAll(filepath.Join(dir, "wireless"), 0o755); err != nil { + t.Fatalf("create the wireless directory: %v", err) + } +} + +// writeSysfsDriverLink points the device entry of an interface at a driver. +func writeSysfsDriverLink(t *testing.T, root, name, driver string) { + t.Helper() + device := filepath.Join(root, name, "device") + if err := os.MkdirAll(device, 0o755); err != nil { + t.Fatalf("create %s: %v", device, err) + } + if err := os.Symlink(filepath.Join(root, "drivers", driver), filepath.Join(device, "driver")); err != nil { + t.Fatalf("link the driver of %s: %v", name, err) + } +} + +// writeSysfsDriverUevent names the driver in the uevent of the device. +func writeSysfsDriverUevent(t *testing.T, root, name, driver string) { + t.Helper() + device := filepath.Join(root, name, "device") + if err := os.MkdirAll(device, 0o755); err != nil { + t.Fatalf("create %s: %v", device, err) + } + if err := os.WriteFile(filepath.Join(device, "uevent"), []byte("DRIVER="+driver+"\n"), 0o644); err != nil { + t.Fatalf("write the uevent of %s: %v", name, err) + } +} + +// stubSysfsNetRoot points the sysfs reader at a temporary directory. +func stubSysfsNetRoot(t *testing.T) string { + t.Helper() + before := sysfsNetRoot + t.Cleanup(func() { sysfsNetRoot = before }) + sysfsNetRoot = t.TempDir() + return sysfsNetRoot +} + +func Test_sysfsInterfaceInfo(t *testing.T) { + root := stubSysfsNetRoot(t) + writeSysfsInterface(t, root, "wlan0", map[string]string{ + "type": "1\n", + "uevent": "INTERFACE=wlan0\nDEVTYPE=wlan\nIFINDEX=3\n", + }, true) + writeSysfsInterface(t, root, "eth0", map[string]string{ + "type": "1\n", + "uevent": "INTERFACE=eth0\nIFINDEX=2\n", + }, false) + writeSysfsInterface(t, root, "tun0", map[string]string{ + "type": "65534\n", + "uevent": "DEVTYPE=tun\n", + }, false) + + tests := []struct { + name string + want sysfsInfo + }{ + {"wlan0", sysfsInfo{Type: 1, DevType: "wlan", Wireless: true}}, + {"eth0", sysfsInfo{Type: 1}}, + {"tun0", sysfsInfo{Type: 65534, DevType: "tun"}}, + {"absent0", sysfsInfo{}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sysfsInterfaceInfo(tc.name); got != tc.want { + t.Errorf("sysfsInterfaceInfo(%q) = %+v, want %+v", tc.name, got, tc.want) + } + }) + } +} + +func Test_sysfsClass(t *testing.T) { + tests := []struct { + name string + iface string + info sysfsInfo + want string + }{ + {"wireless", "wlan0", sysfsInfo{Wireless: true}, "hardware"}, + {"ethernet", "eth0", sysfsInfo{Type: 1}, "hardware"}, + {"tun devtype", "tun0", sysfsInfo{Type: 65534, DevType: "tun"}, "tunnel"}, + {"tap devtype", "tap0", sysfsInfo{Type: 1, DevType: "tap"}, "tunnel"}, + {"unknown", "sit0", sysfsInfo{Type: 65534}, ""}, + {"docker bridge", "docker0", sysfsInfo{Type: 1, DevType: "bridge"}, "virtual"}, + {"container link", "veth1a2b3c", sysfsInfo{Type: 1}, "virtual"}, + {"libvirt bridge", "virbr0", sysfsInfo{Type: 1, DevType: "bridge"}, "virtual"}, + {"compose bridge", "br-1a2b3c", sysfsInfo{Type: 1, DevType: "bridge"}, "virtual"}, + {"uplink bridge", "br0", sysfsInfo{Type: 1, DevType: "bridge"}, "hardware"}, + {"bond", "bond0", sysfsInfo{Type: 1, DevType: "bond"}, "hardware"}, + {"vlan uplink", "eth0.100", sysfsInfo{Type: 1, DevType: "vlan"}, "hardware"}, + {"removed container link", "veth9f8e7d", sysfsInfo{}, "virtual"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sysfsClass(tc.iface, tc.info); got != tc.want { + t.Errorf("sysfsClass(%q, %+v) = %q, want %q", tc.iface, tc.info, got, tc.want) + } + }) + } +} + +func Test_platformVirtualInterfaces_namesTheContainerInterfaces(t *testing.T) { + root := t.TempDir() + before := sysfsVirtualNetRoot + t.Cleanup(func() { sysfsVirtualNetRoot = before }) + sysfsVirtualNetRoot = root + for _, name := range []string{"lo", "docker0", "veth1a2b3c", "virbr0", "br-1a2b3c", "br0", "bond0"} { + if err := os.MkdirAll(filepath.Join(root, name), 0o755); err != nil { + t.Fatalf("create %s: %v", name, err) + } + } + + want := map[string]struct{}{"docker0": {}, "veth1a2b3c": {}, "virbr0": {}, "br-1a2b3c": {}} + if got := platformVirtualInterfaces(); !maps.Equal(got, want) { + t.Errorf("platformVirtualInterfaces = %v, want %v", got, want) + } + if got := virtualInterfaces(); len(got) != 7 { + t.Errorf("virtualInterfaces = %v, want every entry of the kernel directory", got) + } +} + +func Test_platformInterfaceMeta(t *testing.T) { + root := stubSysfsNetRoot(t) + writeSysfsInterface(t, root, "ctrld-test-eth0", map[string]string{ + "type": "1\n", + "uevent": "DEVTYPE=bridge\n", + }, false) + + class, hardwarePort, service := platformInterfaceMeta("ctrld-test-eth0") + if class != "hardware" { + t.Errorf("class = %q, want hardware", class) + } + if hardwarePort != "bridge" { + t.Errorf("hardware port = %q, want bridge", hardwarePort) + } + if service != "" { + t.Errorf("service = %q, want empty", service) + } +} + +func Test_platformInterfaceMeta_namesTheLink(t *testing.T) { + root := stubSysfsNetRoot(t) + writeSysfsInterface(t, root, "wlan0", map[string]string{"type": "1\n", "uevent": "DEVTYPE=wlan\n"}, true) + writeSysfsInterface(t, root, "eth0", map[string]string{"type": "1\n", "uevent": "INTERFACE=eth0\n"}, false) + writeSysfsInterface(t, root, "br0", map[string]string{"type": "1\n", "uevent": "DEVTYPE=bridge\n"}, false) + writeSysfsInterface(t, root, "tun0", map[string]string{"type": "65534\n", "uevent": "DEVTYPE=tun\n"}, false) + writeSysfsInterface(t, root, "usb0", map[string]string{"type": "1\n"}, false) + writeSysfsDriverLink(t, root, "usb0", "ipheth") + writeSysfsInterface(t, root, "usb1", map[string]string{"type": "1\n"}, false) + writeSysfsDriverUevent(t, root, "usb1", "rndis_host") + writeSysfsInterface(t, root, "eth1", map[string]string{"type": "1\n"}, false) + writeSysfsDriverUevent(t, root, "eth1", "e1000e") + + tests := []struct { + iface, class, hardwarePort, linkType string + }{ + {"wlan0", "hardware", "Wi-Fi", "wifi"}, + {"eth0", "hardware", "Ethernet", "ethernet"}, + {"eth1", "hardware", "Ethernet", "ethernet"}, + {"br0", "hardware", "bridge", "unknown"}, + {"tun0", "tunnel", "tun", "tunnel"}, + {"usb0", "hardware", hardwarePortIPhoneUSB, "usb_tether"}, + {"usb1", "hardware", hardwarePortIPhoneUSB, "usb_tether"}, + } + for _, tc := range tests { + t.Run(tc.iface, func(t *testing.T) { + class, hardwarePort, service := platformInterfaceMeta(tc.iface) + if class != tc.class || hardwarePort != tc.hardwarePort || service != "" { + t.Errorf("platformInterfaceMeta(%q) = %q, %q, %q, want %q, %q, empty", + tc.iface, class, hardwarePort, service, tc.class, tc.hardwarePort) + } + if got := linkTypeFor(hardwarePort, class); got != tc.linkType { + t.Errorf("linkTypeFor(%q, %q) = %q, want %q", hardwarePort, class, got, tc.linkType) + } + }) + } +} diff --git a/cmd/cli/net_others.go b/cmd/cli/net_others.go index f347278..1e95b93 100644 --- a/cmd/cli/net_others.go +++ b/cmd/cli/net_others.go @@ -20,3 +20,18 @@ func validInterfacesMap() map[string]struct{} { } return map[string]struct{}{defaultRoute.InterfaceName: {}} } + +// platformInterfaceMeta returns no names. These platforms have no source for +// them, so the interface name decides the class. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + return "", "", "" +} + +// platformVirtualInterfaces returns no set. +func platformVirtualInterfaces() map[string]struct{} { + return nil +} + +// refreshInterfaceMeta does no work. These platforms keep no interface names, +// so an adapter that appears leaves nothing stale. +func refreshInterfaceMeta() {} diff --git a/cmd/cli/net_windows.go b/cmd/cli/net_windows.go index bed06b5..226e327 100644 --- a/cmd/cli/net_windows.go +++ b/cmd/cli/net_windows.go @@ -5,6 +5,9 @@ import ( "log" "net" "os" + "strings" + "sync" + "time" "github.com/microsoft/wmi/pkg/base/host" "github.com/microsoft/wmi/pkg/base/instance" @@ -13,6 +16,31 @@ import ( "github.com/microsoft/wmi/pkg/hardware/network/netadapter" ) +// adapterInfo holds what one WMI query knows about a network adapter. +type adapterInfo struct { + Name string + Description string + Virtual bool + ConnectorPresent bool + Hardware bool +} + +// Every network change reads the adapter set, so one query serves a window. +const adapterCacheTTL = time.Minute + +var ( + adapterCacheMu sync.Mutex + adapterCache []adapterInfo + // The set before the last read still names the adapter that went away, + // so a removed adapter keeps its class in the journal. + adapterCachePrevious []adapterInfo + adapterCacheReadAt time.Time + + // Tests replace these seams to avoid a WMI query. + readAdaptersFn = readAdapters + adapterCacheNowFn = time.Now +) + func patchNetIfaceName(iface *net.Interface) (bool, error) { return true, nil } @@ -35,6 +63,123 @@ func validInterfacesMap() map[string]struct{} { // validInterfaces returns a list of all physical interfaces. func validInterfaces() []string { + var names []string + for _, adapter := range adapters() { + if !adapter.ConnectorPresent { + mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", adapter.Name).Msg("skipping non-physical adapter") + continue + } + if !adapter.Hardware { + mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", adapter.Name).Msg("skipping non-hardware interface") + continue + } + names = append(names, adapter.Name) + } + return names +} + +// adapters returns the adapter set of the host, from the cache while it is +// younger than the window. The query runs under the mutex, so two callers +// start one query. +func adapters() []adapterInfo { + adapterCacheMu.Lock() + defer adapterCacheMu.Unlock() + now := adapterCacheNowFn() + if !adapterCacheReadAt.IsZero() && now.Sub(adapterCacheReadAt) < adapterCacheTTL { + return adapterCache + } + read := readAdaptersFn() + // A failing query waits for the window too, so it starts no query storm. + adapterCacheReadAt = now + if read == nil { + // Keep the last good read, so a failed query loses no adapter. + return adapterCache + } + adapterCachePrevious, adapterCache = adapterCache, read + return read +} + +// refreshInterfaceMeta drops the read window. The cache still describes the +// adapter that appeared or went away, and the next lookup reads the set. +func refreshInterfaceMeta() { + adapterCacheMu.Lock() + defer adapterCacheMu.Unlock() + adapterCacheReadAt = time.Time{} +} + +// tunnelAdapterWords name the VPN drivers of Windows. The description is the +// only signal, because WMI describes a tunnel adapter as an Ethernet adapter. +var tunnelAdapterWords = []string{"tap", "wintun", "wireguard", "vpn"} + +// adapterClass names the kind of an adapter. WMI marks a software adapter +// virtual and gives it no connector. +func adapterClass(adapter adapterInfo) string { + switch { + case tunnelAdapter(adapter.Description): + return "tunnel" + case adapter.Virtual && !adapter.ConnectorPresent: + return "virtual" + } + return "hardware" +} + +// tunnelAdapter reports a description that names a VPN driver. +func tunnelAdapter(description string) bool { + lower := strings.ToLower(description) + for _, word := range tunnelAdapterWords { + if strings.Contains(lower, word) { + return true + } + } + return false +} + +// platformInterfaceMeta returns the Windows class and adapter description of +// an interface. Windows has no service name. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + adapter, known := adapterByName(name) + if !known { + return "", "", "" + } + return adapterClass(adapter), adapter.Description, "" +} + +// adapterByName finds an adapter in the current set, or in the set before the +// last read. An adapter that went away keeps its names that way. +func adapterByName(name string) (adapterInfo, bool) { + if adapter, known := findAdapter(adapters(), name); known { + return adapter, true + } + adapterCacheMu.Lock() + defer adapterCacheMu.Unlock() + return findAdapter(adapterCachePrevious, name) +} + +func findAdapter(set []adapterInfo, name string) (adapterInfo, bool) { + for _, adapter := range set { + if adapter.Name == name { + return adapter, true + } + } + return adapterInfo{}, false +} + +// platformVirtualInterfaces returns the adapters that carry no traffic of the +// host network. +func platformVirtualInterfaces() map[string]struct{} { + virtual := make(map[string]struct{}) + for _, adapter := range adapters() { + if adapterClass(adapter) != "virtual" { + continue + } + virtual[adapter.Name] = struct{}{} + } + return virtual +} + +// readAdapters runs one WMI query for every network adapter. It returns nil +// when the query fails. +func readAdapters() []adapterInfo { log.SetOutput(io.Discard) defer log.SetOutput(os.Stderr) whost := host.NewWmiLocalHost() @@ -47,47 +192,53 @@ func validInterfaces() []string { mainLog.Load().Warn().Err(err).Msg("failed to get wmi network adapter") return nil } - var adapters []string + var read []adapterInfo for _, i := range instances { adapter, err := netadapter.NewNetworkAdapter(i) if err != nil { mainLog.Load().Warn().Err(err).Msg("failed to get network adapter") continue } - - name, err := adapter.GetPropertyName() - if err != nil { - mainLog.Load().Warn().Err(err).Msg("failed to get interface name") + info, ok := readAdapterInfo(adapter) + if !ok { continue } - - // From: https://learn.microsoft.com/en-us/previous-versions/windows/desktop/legacy/hh968170(v=vs.85) - // - // "Indicates if a connector is present on the network adapter. This value is set to TRUE - // if this is a physical adapter or FALSE if this is not a physical adapter." - physical, err := adapter.GetPropertyConnectorPresent() - if err != nil { - mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", name).Msg("failed to get network adapter connector present property") - continue - } - if !physical { - mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", name).Msg("skipping non-physical adapter") - continue - } - - // Check if it's a hardware interface. Checking only for connector present is not enough - // because some interfaces are not physical but have a connector. - hardware, err := adapter.GetPropertyHardwareInterface() - if err != nil { - mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", name).Msg("failed to get network adapter hardware interface property") - continue - } - if !hardware { - mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", name).Msg("skipping non-hardware interface") - continue - } - - adapters = append(adapters, name) + read = append(read, info) } - return adapters + return read +} + +// readAdapterInfo reads the properties of one adapter. It reports false when a +// property that decides validity is unreadable. +func readAdapterInfo(adapter *netadapter.NetworkAdapter) (adapterInfo, bool) { + name, err := adapter.GetPropertyName() + if err != nil { + mainLog.Load().Warn().Err(err).Msg("failed to get interface name") + return adapterInfo{}, false + } + + // From: https://learn.microsoft.com/en-us/previous-versions/windows/desktop/legacy/hh968170(v=vs.85) + // + // "Indicates if a connector is present on the network adapter. This value is set to TRUE + // if this is a physical adapter or FALSE if this is not a physical adapter." + connectorPresent, err := adapter.GetPropertyConnectorPresent() + if err != nil { + mainLog.Load().Debug().Str("method", "readAdapterInfo").Str("interface", name).Msg("failed to get network adapter connector present property") + return adapterInfo{}, false + } + + // Check if it's a hardware interface. Checking only for connector present is not enough + // because some interfaces are not physical but have a connector. + hardware, err := adapter.GetPropertyHardwareInterface() + if err != nil { + mainLog.Load().Debug().Str("method", "readAdapterInfo").Str("interface", name).Msg("failed to get network adapter hardware interface property") + return adapterInfo{}, false + } + + info := adapterInfo{Name: name, ConnectorPresent: connectorPresent, Hardware: hardware} + // The class and the description describe the adapter only, so an + // unreadable one costs no validity. + info.Description, _ = adapter.GetPropertyInterfaceDescription() + info.Virtual, _ = adapter.GetPropertyVirtual() + return info, true } diff --git a/cmd/cli/net_windows_test.go b/cmd/cli/net_windows_test.go index a15f119..2f7395b 100644 --- a/cmd/cli/net_windows_test.go +++ b/cmd/cli/net_windows_test.go @@ -5,6 +5,7 @@ import ( "bytes" "slices" "strings" + "sync" "testing" "time" ) @@ -40,3 +41,143 @@ func validInterfacesPowershell() []string { } return res } + +func Test_adapterClass(t *testing.T) { + tests := []struct { + name string + adapter adapterInfo + want string + }{ + {"virtual without a connector", adapterInfo{Virtual: true}, "virtual"}, + {"virtual with a connector", adapterInfo{Virtual: true, ConnectorPresent: true}, "hardware"}, + {"physical adapter", adapterInfo{ConnectorPresent: true, Hardware: true}, "hardware"}, + {"unknown adapter", adapterInfo{}, "hardware"}, + {"tap driver", adapterInfo{Description: "TAP-Windows Adapter V9", Virtual: true}, "tunnel"}, + {"wintun driver", adapterInfo{Description: "Wintun Userspace Tunnel", Virtual: true}, "tunnel"}, + {"wireguard driver", adapterInfo{Description: "WireGuard Tunnel #1", Virtual: true}, "tunnel"}, + {"vpn description", adapterInfo{Description: "Cisco AnyConnect VPN Adapter", ConnectorPresent: true}, "tunnel"}, + {"lower case description", adapterInfo{Description: "openvpn tap-windows6"}, "tunnel"}, + {"ethernet adapter", adapterInfo{Description: "Intel(R) Ethernet Connection I219-V", ConnectorPresent: true, Hardware: true}, "hardware"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := adapterClass(tc.adapter); got != tc.want { + t.Errorf("adapterClass(%+v) = %q, want %q", tc.adapter, got, tc.want) + } + }) + } +} + +// stubAdapters replaces the WMI and clock seams of the adapter cache, and +// restores them with the cache itself. +func stubAdapters(t *testing.T, read func() []adapterInfo, now func() time.Time) { + t.Helper() + readBefore, nowBefore := readAdaptersFn, adapterCacheNowFn + adapterCacheMu.Lock() + cacheBefore, readAtBefore := adapterCache, adapterCacheReadAt + adapterCache, adapterCacheReadAt = nil, time.Time{} + adapterCacheMu.Unlock() + t.Cleanup(func() { + readAdaptersFn, adapterCacheNowFn = readBefore, nowBefore + adapterCacheMu.Lock() + adapterCache, adapterCacheReadAt = cacheBefore, readAtBefore + adapterCacheMu.Unlock() + }) + readAdaptersFn, adapterCacheNowFn = read, now +} + +func Test_adapterCache(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubAdapters(t, func() []adapterInfo { + reads++ + return []adapterInfo{ + {Name: "Ethernet", Description: "Intel I219-V", ConnectorPresent: true, Hardware: true}, + {Name: "Hyper-V Switch", Description: "Hyper-V Virtual Switch", Virtual: true}, + } + }, func() time.Time { return now }) + + if got := validInterfaces(); !slices.Equal(got, []string{"Ethernet"}) { + t.Fatalf("validInterfaces = %v, want [Ethernet]", got) + } + now = now.Add(59 * time.Second) + validInterfacesMap() + if reads != 1 { + t.Fatalf("reads within the cache window = %d, want 1", reads) + } + now = now.Add(2 * time.Second) + adapters() + if reads != 2 { + t.Fatalf("reads after the cache window = %d, want 2", reads) + } + refreshInterfaceMeta() + if reads != 2 { + t.Fatalf("reads for a refresh alone = %d, want 2", reads) + } + adapters() + if reads != 3 { + t.Fatalf("reads after a refresh = %d, want 3", reads) + } + + virtual := platformVirtualInterfaces() + if _, ok := virtual["Hyper-V Switch"]; !ok || len(virtual) != 1 { + t.Fatalf("platformVirtualInterfaces = %v, want the virtual switch only", virtual) + } + class, hardwarePort, service := platformInterfaceMeta("Ethernet") + if class != "hardware" || hardwarePort != "Intel I219-V" || service != "" { + t.Fatalf("platformInterfaceMeta = %q, %q, %q, want hardware, Intel I219-V, empty", class, hardwarePort, service) + } +} + +func Test_adapterCacheReadsOnceForTwoCallers(t *testing.T) { + now := time.Unix(1700000000, 0) + var mu sync.Mutex + reads := 0 + stubAdapters(t, func() []adapterInfo { + mu.Lock() + reads++ + mu.Unlock() + time.Sleep(10 * time.Millisecond) + return []adapterInfo{{Name: "Ethernet", Description: "Intel I219-V", ConnectorPresent: true, Hardware: true}} + }, func() time.Time { return now }) + + var wg sync.WaitGroup + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + adapters() + }() + } + wg.Wait() + + mu.Lock() + defer mu.Unlock() + if reads != 1 { + t.Fatalf("queries for four callers = %d, want 1", reads) + } +} + +func Test_adapterCacheKeepsTheRemovedAdapter(t *testing.T) { + now := time.Unix(1700000000, 0) + removed := false + stubAdapters(t, func() []adapterInfo { + read := []adapterInfo{{Name: "Ethernet", Description: "Intel I219-V", ConnectorPresent: true, Hardware: true}} + if removed { + return read + } + return append(read, adapterInfo{Name: "WireGuard", Description: "WireGuard Tunnel #1", Virtual: true}) + }, func() time.Time { return now }) + + adapters() + removed = true + refreshInterfaceMeta() + + class, hardwarePort, _ := platformInterfaceMeta("WireGuard") + if class != "tunnel" || hardwarePort != "WireGuard Tunnel #1" { + t.Fatalf("meta of the removed adapter = %q, %q, want tunnel, WireGuard Tunnel #1", class, hardwarePort) + } + if class, _, _ := platformInterfaceMeta("Absent"); class != "" { + t.Fatalf("meta of an adapter that no read named = %q, want empty", class) + } +} diff --git a/cmd/cli/network_delta.go b/cmd/cli/network_delta.go new file mode 100644 index 0000000..18ea6bd --- /dev/null +++ b/cmd/cli/network_delta.go @@ -0,0 +1,301 @@ +package cli + +import ( + "net" + "net/netip" + "slices" + "strings" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// interfaceChange describes one interface that a network delta changed. +type interfaceChange struct { + Name string + Action string + Class string + HardwarePort string + Service string + IPsBefore []string + IPsAfter []string + Flags string + MTU int + IsDefaultRoute bool +} + +// interfaceMeta holds the platform description of one interface. An empty +// Class asks the caller to read the class from the interface name. +type interfaceMeta struct { + Class string + HardwarePort string + Service string + LinkType string +} + +// interfaceMetaFunc reads the platform description of one interface. +type interfaceMetaFunc func(name string) interfaceMeta + +// virtualInterfaceSetFn names the virtual adapters of the host. Each platform +// file fills it, because a name alone does not tell a container bridge from a +// physical port. +var virtualInterfaceSetFn = func() map[string]struct{} { return nil } + +// interfaceSnapshot holds the part of one interface state that the diff +// compares. +type interfaceSnapshot struct { + present bool + up bool + mac string + mtu int + flags string + ips []string +} + +// interfaceClass groups an interface by the traffic it carries. The class +// decides which changes are noise. +func interfaceClass(name string, virtual bool) string { + switch { + case strings.HasPrefix(name, "lo"): + return "loopback" + case hasInterfacePrefix(name, "awdl", "llw"): + return "airdrop" + case hasInterfacePrefix(name, "utun", "tun", "tap", "wg", "ipsec", "ppp"): + return "tunnel" + case virtual: + return "virtual" + default: + return "hardware" + } +} + +// deltaBeforeState picks the state that the diff of one callback compares +// against, and makes a current callback the baseline of the next one. netmon +// caches major snapshots only, so every minor callback of an epoch carries the +// same Old, and a difference that lasts would look new in each of them. A +// callback that a newer snapshot replaced reads the baseline and leaves it. +// The caller holds networkSourceMu. +// +// netmon runs its callbacks in their own goroutines, so a minor callback of a +// new epoch can run before the major callback that opened the epoch. That +// major then reports its own changes against its Old and leaves the newer +// baseline in place. A major whose Old is the cache of the epoch before +// continues from the last reported state of that epoch, so a change that a +// minor callback reported does not appear again. +func (p *prog) deltaBeforeState(delta *netmon.ChangeDelta, epoch *netmon.State, current, major bool) *netmon.State { + before := p.networkDeltaState + switch { + case before == nil: + before = delta.Old + case current && major && p.networkDeltaEpoch == epoch: + return delta.Old + case p.networkDeltaEpoch != epoch && delta.Old != p.networkDeltaEpoch: + before = delta.Old + } + if current { + p.networkDeltaState, p.networkDeltaEpoch = delta.New, epoch + } + return before +} + +// diffNetworkDelta describes every interface that changed between two states. +// It walks every interface, not only the valid hardware ports, because the +// journal must show the port that went away too. +func diffNetworkDelta(beforeState, afterState *netmon.State) []interfaceChange { + before, after := networkStateOrEmpty(beforeState), networkStateOrEmpty(afterState) + virtual := virtualInterfaceSetFn() + var changes []interfaceChange + for _, name := range interfaceNameUnion(before, after) { + snapBefore, snapAfter := readInterfaceSnapshot(before, name), readInterfaceSnapshot(after, name) + action := interfaceAction(snapBefore, snapAfter) + if action == "" { + continue + } + known := snapAfter + if !snapAfter.present { + known = snapBefore + } + _, isVirtual := virtual[name] + changes = append(changes, interfaceChange{ + Name: name, + Action: action, + Class: interfaceClass(name, isVirtual), + IPsBefore: snapBefore.ips, + IPsAfter: snapAfter.ips, + Flags: known.flags, + MTU: known.mtu, + IsDefaultRoute: name == after.DefaultRouteInterface, + }) + } + return changes +} + +// noiseDelta reports a delta that no part of the daemon must act on. AirDrop +// and the virtual adapters of a container host change every few seconds, and +// each change would otherwise start the whole reconcile chain. A time jump +// means the host woke, so the daemon acts on the delta whatever it holds. +func noiseDelta(beforeState, afterState *netmon.State, timeJumped bool, changes []interfaceChange) bool { + if len(changes) == 0 || timeJumped { + return false + } + before, after := networkStateOrEmpty(beforeState), networkStateOrEmpty(afterState) + if before.DefaultRouteInterface != after.DefaultRouteInterface { + return false + } + if before.HaveV4 != after.HaveV4 || before.HaveV6 != after.HaveV6 { + return false + } + source4, source6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + for _, change := range changes { + if !noiseClass(change.Class) || change.IsDefaultRoute { + return false + } + if carriesHostTraffic(change.IPsBefore, source4, source6) { + return false + } + if carriesHostTraffic(change.IPsAfter, source4, source6) { + return false + } + } + return true +} + +// noiseClass names the classes that change on their own. The kernel moves the +// link-local address of an AirDrop or a container adapter every few seconds. +func noiseClass(class string) bool { + return class == "airdrop" || class == "virtual" +} + +// carriesHostTraffic reports an address list that reaches another host. A +// bridge, a bond, or a tunnel adapter can own the traffic of the host, so a +// routable address or the resolver source address makes the interface real +// whatever its name says. +func carriesHostTraffic(ips []string, source4, source6 net.IP) bool { + for _, ip := range ips { + prefix, err := netip.ParsePrefix(ip) + if err != nil { + continue + } + addr := prefix.Addr() + if addr.IsGlobalUnicast() || sameAddress(addr, source4) || sameAddress(addr, source6) { + return true + } + } + return false +} + +// sameAddress compares one interface address with a resolver source address. +func sameAddress(addr netip.Addr, ip net.IP) bool { + source, ok := netip.AddrFromSlice(ip) + return ok && source.Unmap() == addr +} + +// describeInterfaceChanges fills the platform fields of the changes that reach +// the journal. macOS reads them with two subprocesses, so only a delta that +// the daemon acts on pays for them. The platform class wins where it knows +// one, because a name rule cannot see a virtual adapter on Windows. +func describeInterfaceChanges(changes []interfaceChange, meta interfaceMetaFunc) { + if meta == nil { + return + } + for i := range changes { + info := meta(changes[i].Name) + changes[i].HardwarePort, changes[i].Service = info.HardwarePort, info.Service + if info.Class != "" { + changes[i].Class = info.Class + } + } +} + +// interfaceAction names the first difference that the journal reports. The +// order runs from the coarse change to the fine one, so one interface gives +// one action. +func interfaceAction(before, after interfaceSnapshot) string { + switch { + case before.present && !after.present: + return "removed" + case !before.present && after.present: + return "added" + case before.up && !after.up: + return "down" + case !before.up && after.up: + return "up" + case hasExtraEntry(before.ips, after.ips): + return "ip_removed" + case hasExtraEntry(after.ips, before.ips): + return "ip_added" + case before.mac != after.mac: + return "mac_changed" + case before.mtu != after.mtu: + return "mtu_changed" + default: + return "" + } +} + +func readInterfaceSnapshot(state *netmon.State, name string) interfaceSnapshot { + snapshot := interfaceSnapshot{ips: sortedPrefixStrings(state.InterfaceIPs[name])} + iface, ok := state.Interface[name] + if !ok { + return snapshot + } + snapshot.present = true + if iface.Interface == nil { + return snapshot + } + snapshot.up = iface.IsUp() + snapshot.mtu = iface.MTU + snapshot.flags = iface.Flags.String() + if len(iface.HardwareAddr) > 0 { + snapshot.mac = iface.HardwareAddr.String() + } + return snapshot +} + +func interfaceNameUnion(before, after *netmon.State) []string { + names := make([]string, 0, len(before.Interface)+len(after.Interface)) + for name := range before.Interface { + names = append(names, name) + } + for name := range after.Interface { + if _, both := before.Interface[name]; !both { + names = append(names, name) + } + } + slices.Sort(names) + return names +} + +func sortedPrefixStrings(prefixes []netip.Prefix) []string { + ips := prefixStrings(prefixes) + slices.Sort(ips) + return ips +} + +// networkStateOrEmpty keeps the diff free of nil checks. The first delta after +// start carries no old state. +func networkStateOrEmpty(state *netmon.State) *netmon.State { + if state != nil { + return state + } + return &netmon.State{} +} + +func hasInterfacePrefix(name string, prefixes ...string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} + +// hasExtraEntry reports an entry of list that other does not hold. +func hasExtraEntry(list, other []string) bool { + for _, entry := range list { + if !slices.Contains(other, entry) { + return true + } + } + return false +} diff --git a/cmd/cli/network_delta_test.go b/cmd/cli/network_delta_test.go new file mode 100644 index 0000000..1c06ff5 --- /dev/null +++ b/cmd/cli/network_delta_test.go @@ -0,0 +1,488 @@ +package cli + +import ( + "net" + "net/netip" + "slices" + "testing" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// deltaTestInterface describes one interface of a test network state. +type deltaTestInterface struct { + name string + up bool + mac string + mtu int + ips []string +} + +var ( + deltaTestUpFlags = net.FlagUp | net.FlagBroadcast | net.FlagMulticast + deltaTestDownFlags = net.Flags(0) +) + +func deltaTestState(t *testing.T, route string, ifaces ...deltaTestInterface) *netmon.State { + t.Helper() + state := &netmon.State{ + DefaultRouteInterface: route, + Interface: map[string]netmon.Interface{}, + InterfaceIPs: map[string][]netip.Prefix{}, + } + for _, iface := range ifaces { + flags := deltaTestDownFlags + if iface.up { + flags = deltaTestUpFlags + } + var mac net.HardwareAddr + if iface.mac != "" { + parsed, err := net.ParseMAC(iface.mac) + if err != nil { + t.Fatalf("ParseMAC(%q) error: %v", iface.mac, err) + } + mac = parsed + } + state.Interface[iface.name] = netmon.Interface{Interface: &net.Interface{ + Name: iface.name, + Flags: flags, + MTU: iface.mtu, + HardwareAddr: mac, + }} + for _, ip := range iface.ips { + state.InterfaceIPs[iface.name] = append(state.InterfaceIPs[iface.name], netip.MustParsePrefix(ip)) + } + } + return state +} + +func sameInterfaceChange(got, want interfaceChange) bool { + return got.Name == want.Name && + got.Action == want.Action && + got.Class == want.Class && + got.HardwarePort == want.HardwarePort && + got.Service == want.Service && + slices.Equal(got.IPsBefore, want.IPsBefore) && + slices.Equal(got.IPsAfter, want.IPsAfter) && + got.Flags == want.Flags && + got.MTU == want.MTU && + got.IsDefaultRoute == want.IsDefaultRoute +} + +func assertInterfaceChanges(t *testing.T, got, want []interfaceChange) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("changes = %+v, want %+v", got, want) + } + for i := range want { + if !sameInterfaceChange(got[i], want[i]) { + t.Errorf("change %d = %+v, want %+v", i, got[i], want[i]) + } + } +} + +func Test_diffNetworkDelta(t *testing.T) { + en0 := deltaTestInterface{name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}} + awdl0 := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:02", mtu: 1484, ips: []string{"fe80::1/64"}} + awdl0NewMAC := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:03", mtu: 1484, ips: []string{"fe80::1/64"}} + en5 := deltaTestInterface{name: "en5", up: true, mac: "aa:bb:cc:dd:ee:05", mtu: 1500, ips: []string{"192.0.2.50/24"}} + + for _, tt := range []struct { + name string + routeOld string + routeNew string + nilBefore bool + before []deltaTestInterface + after []deltaTestInterface + meta interfaceMetaFunc + want []interfaceChange + }{ + { + name: "airdrop_mac_change_with_link_local_only", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + want: []interfaceChange{{ + Name: "awdl0", Action: "mac_changed", Class: "airdrop", + IPsBefore: []string{"fe80::1/64"}, IPsAfter: []string{"fe80::1/64"}, + Flags: deltaTestUpFlags.String(), MTU: 1484, + }}, + }, + { + name: "airdrop_mac_and_link_local_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, { + name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:03", mtu: 1484, ips: []string{"fe80::2/64"}, + }}, + want: []interfaceChange{{ + Name: "awdl0", Action: "ip_removed", Class: "airdrop", + IPsBefore: []string{"fe80::1/64"}, IPsAfter: []string{"fe80::2/64"}, + Flags: deltaTestUpFlags.String(), MTU: 1484, + }}, + }, + { + name: "hardware_address_added_with_airdrop_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "192.0.2.11/24"}, + }, awdl0NewMAC}, + want: []interfaceChange{ + { + Name: "awdl0", Action: "mac_changed", Class: "airdrop", + IPsBefore: []string{"fe80::1/64"}, IPsAfter: []string{"fe80::1/64"}, + Flags: deltaTestUpFlags.String(), MTU: 1484, + }, + { + Name: "en0", Action: "ip_added", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24", "192.0.2.11/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }, + }, + }, + { + name: "interface_removed", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, en5}, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en5", Action: "removed", Class: "hardware", + IPsBefore: []string{"192.0.2.50/24"}, IPsAfter: nil, + Flags: deltaTestUpFlags.String(), MTU: 1500, + }}, + }, + { + name: "interface_added", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{en0, en5}, + want: []interfaceChange{{ + Name: "en5", Action: "added", Class: "hardware", + IPsBefore: nil, IPsAfter: []string{"192.0.2.50/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, + }}, + }, + { + name: "address_removed", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "192.0.2.11/24"}, + }}, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en0", Action: "ip_removed", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24", "192.0.2.11/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "interface_down", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{{ + name: "en0", up: false, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, + }}, + want: []interfaceChange{{ + Name: "en0", Action: "down", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestDownFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "interface_up", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{{ + name: "en0", up: false, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, + }}, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en0", Action: "up", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "mtu_changed", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1400, ips: []string{"192.0.2.10/24"}, + }}, + want: []interfaceChange{{ + Name: "en0", Action: "mtu_changed", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1400, IsDefaultRoute: true, + }}, + }, + { + name: "default_route_moves_to_the_new_interface", + routeOld: "en0", routeNew: "en5", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{{ + name: "en0", up: false, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, + }, en5}, + want: []interfaceChange{ + { + Name: "en0", Action: "down", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestDownFlags.String(), MTU: 1500, + }, + { + Name: "en5", Action: "added", Class: "hardware", + IPsBefore: nil, IPsAfter: []string{"192.0.2.50/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }, + }, + }, + { + name: "nil_old_state_adds_every_interface", + routeNew: "en0", + nilBefore: true, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en0", Action: "added", Class: "hardware", + IPsBefore: nil, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "no_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0}, + want: nil, + }, + { + name: "meta_fills_the_platform_fields", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, {name: "utun3", up: true, mtu: 1400}}, + after: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1400, ips: []string{"192.0.2.10/24"}, + }, {name: "utun3", up: true, mtu: 1380}}, + meta: func(name string) interfaceMeta { + if name == "utun3" { + return interfaceMeta{Class: "virtual"} + } + return interfaceMeta{HardwarePort: "Wi-Fi", Service: "Wi-Fi"} + }, + want: []interfaceChange{ + { + Name: "en0", Action: "mtu_changed", Class: "hardware", + HardwarePort: "Wi-Fi", Service: "Wi-Fi", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1400, IsDefaultRoute: true, + }, + { + Name: "utun3", Action: "mtu_changed", Class: "virtual", + Flags: deltaTestUpFlags.String(), MTU: 1380, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + var before *netmon.State + if !tt.nilBefore { + before = deltaTestState(t, tt.routeOld, tt.before...) + } + changes := diffNetworkDelta(before, deltaTestState(t, tt.routeNew, tt.after...)) + describeInterfaceChanges(changes, tt.meta) + assertInterfaceChanges(t, changes, tt.want) + }) + } +} + +func Test_diffNetworkDeltaVirtualSet(t *testing.T) { + virtual := virtualInterfaceSetFn + t.Cleanup(func() { virtualInterfaceSetFn = virtual }) + virtualInterfaceSetFn = func() map[string]struct{} { return map[string]struct{}{"docker0": {}} } + + before := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1500, ips: []string{"192.0.2.10/24"}}, + deltaTestInterface{name: "docker0", up: true, mtu: 1500}, + ) + after := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1400, ips: []string{"192.0.2.10/24"}}, + deltaTestInterface{name: "docker0", up: true, mtu: 1400}, + ) + assertInterfaceChanges(t, diffNetworkDelta(before, after), []interfaceChange{ + { + Name: "docker0", Action: "mtu_changed", Class: "virtual", + Flags: deltaTestUpFlags.String(), MTU: 1400, + }, + { + Name: "en0", Action: "mtu_changed", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1400, IsDefaultRoute: true, + }, + }) +} + +func Test_interfaceClass(t *testing.T) { + for _, tt := range []struct { + name string + virtual bool + want string + }{ + {name: "lo0", want: "loopback"}, + {name: "awdl0", want: "airdrop"}, + {name: "llw0", want: "airdrop"}, + {name: "utun3", want: "tunnel"}, + {name: "tun0", want: "tunnel"}, + {name: "tap0", want: "tunnel"}, + {name: "wg0", want: "tunnel"}, + {name: "ipsec0", want: "tunnel"}, + {name: "ppp0", want: "tunnel"}, + {name: "docker0", virtual: true, want: "virtual"}, + {name: "docker0", want: "hardware"}, + {name: "en0", want: "hardware"}, + } { + t.Run(tt.name+"_"+tt.want, func(t *testing.T) { + if got := interfaceClass(tt.name, tt.virtual); got != tt.want { + t.Errorf("interfaceClass(%q, %v) = %q, want %q", tt.name, tt.virtual, got, tt.want) + } + }) + } +} + +func Test_noiseDelta(t *testing.T) { + en0 := deltaTestInterface{name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}} + en5 := deltaTestInterface{name: "en5", up: true, mac: "aa:bb:cc:dd:ee:05", mtu: 1500, ips: []string{"192.0.2.50/24"}} + awdl0 := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:02", mtu: 1484, ips: []string{"fe80::1/64"}} + awdl0NewMAC := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:03", mtu: 1484, ips: []string{"fe80::1/64"}} + en0Extra := deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "192.0.2.11/24"}, + } + + docker0 := deltaTestInterface{name: "docker0", up: true, mtu: 1500} + docker0NewMTU := deltaTestInterface{name: "docker0", up: true, mtu: 1400} + + for _, tt := range []struct { + name string + routeOld string + routeNew string + before []deltaTestInterface + after []deltaTestInterface + virtual []string + timeJumped bool + beforeV4, beforeV6 bool + afterV4, afterV6 bool + source4 string + want bool + }{ + { + name: "airdrop_only", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + want: true, + }, + { + name: "hardware_and_airdrop", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0Extra, awdl0NewMAC}, + want: false, + }, + { + name: "no_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0}, + want: false, + }, + { + name: "default_route_moved", + routeOld: "en0", routeNew: "en5", + before: []deltaTestInterface{en0, en5, awdl0}, + after: []deltaTestInterface{en0, en5, awdl0NewMAC}, + want: false, + }, + { + name: "virtual_only", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, docker0}, + after: []deltaTestInterface{en0, docker0NewMTU}, + virtual: []string{"docker0"}, + want: true, + }, + { + name: "a_time_jump_is_never_noise", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + timeJumped: true, + want: false, + }, + { + name: "the_default_route_runs_over_a_virtual_interface", + routeOld: "docker0", routeNew: "docker0", + before: []deltaTestInterface{en0, docker0}, + after: []deltaTestInterface{en0, docker0NewMTU}, + virtual: []string{"docker0"}, + want: false, + }, + { + name: "a_virtual_interface_holds_a_routable_address", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, {name: "docker0", up: true, mtu: 1500, ips: []string{"172.17.0.1/16"}}}, + after: []deltaTestInterface{en0, {name: "docker0", up: true, mtu: 1400, ips: []string{"172.17.0.1/16"}}}, + virtual: []string{"docker0"}, + want: false, + }, + { + name: "an_airdrop_interface_holds_the_resolver_source", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, {name: "awdl0", up: true, mtu: 1484, ips: []string{"169.254.10.5/16"}}}, + after: []deltaTestInterface{en0, {name: "awdl0", up: true, mtu: 1400, ips: []string{"169.254.10.5/16"}}}, + source4: "169.254.10.5", + want: false, + }, + { + name: "have_v4_flips", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + beforeV4: true, + want: false, + }, + { + name: "have_v6_flips", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + afterV6: true, + want: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + stubSnapshotVirtualSet(t, tt.virtual...) + stubResolverSource(t, tt.source4) + before := deltaTestState(t, tt.routeOld, tt.before...) + after := deltaTestState(t, tt.routeNew, tt.after...) + before.HaveV4, before.HaveV6 = tt.beforeV4, tt.beforeV6 + after.HaveV4, after.HaveV6 = tt.afterV4, tt.afterV6 + changes := diffNetworkDelta(before, after) + if got := noiseDelta(before, after, tt.timeJumped, changes); got != tt.want { + t.Errorf("noiseDelta = %v, want %v, changes %+v", got, tt.want, changes) + } + }) + } +} + +// stubResolverSource sets the address that the resolver sends its queries +// from. The tests own that global, so each case starts from a known source. +func stubResolverSource(t *testing.T, source4 string) { + t.Helper() + original4, original6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + t.Cleanup(func() { + ctrld.SetDefaultLocalIPv4(original4) + ctrld.SetDefaultLocalIPv6(original6) + }) + ctrld.SetDefaultLocalIPv4(net.ParseIP(source4)) + ctrld.SetDefaultLocalIPv6(nil) +} diff --git a/cmd/cli/network_events.go b/cmd/cli/network_events.go new file mode 100644 index 0000000..959fb29 --- /dev/null +++ b/cmd/cli/network_events.go @@ -0,0 +1,167 @@ +package cli + +import ( + "time" + + "github.com/rs/zerolog" + "tailscale.com/net/netmon" +) + +// networkEventsNowFn is the clock of the network events. Tests replace it. +var networkEventsNowFn = time.Now + +const ( + hostWokeMessage = "Host woke" + + // noiseDeltaMessage names the deltas that no part of the daemon acts on. + noiseDeltaMessage = "Network delta noise" + + // interfaceChangedMessage names one interface that a delta changed. + interfaceChangedMessage = "Network interface changed" + + // networkTransitionMessage names the outcome of one delta. + networkTransitionMessage = "Network transition" + + // transitionOutcomeIgnored names a delta that left every usable interface + // as it was. + transitionOutcomeIgnored = "ignored" + + // transitionOutcomeSnapshotSuperseded names a delta whose snapshot netmon + // already replaced. A newer callback reports the same interfaces. + transitionOutcomeSnapshotSuperseded = "snapshot_superseded" +) + +// noteHostWoke reports one wake to the wake reporter. The caller passes the +// network of the wake, because the state that ctrld holds still describes the +// network before the sleep. netmon reports a time jump and measures no gap, so +// its report waits for the detector, and the journal gets one event per wake. +func (p *prog) noteHostWoke(source string, gap time.Duration, state *netmon.State) { + now := networkEventsNowFn() + // A resume moves the resolvers of the host, so every wake source starts the + // fast poll. + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(now) + } + p.wake.note(now, wakeReport{source: source, gap: gap, state: state}, logHostWoke) +} + +// logHostWoke writes the Host woke journal event of one report. +func logHostWoke(report wakeReport) { + gapKnown := report.gap > 0 + event := journal(mainLog.Load().Info()).Str("source", report.source).Bool("gap_known", gapKnown) + if gapKnown { + event.Int64("gap_ms", report.gap.Milliseconds()) + } + event.Str("default_route", defaultRouteOf(report.state)). + Int("interfaces_up", upInterfaceCount(report.state)). + Msg(hostWokeMessage) +} + +// noteNoiseDelta records one delta of a noise storm. AirDrop and the virtual +// adapters of a container host change every few seconds, so the journal keeps +// one line per window while the debug stream keeps them all. +func (p *prog) noteNoiseDelta(changes []interfaceChange) { + names := changedInterfaceNames(changes) + mainLog.Load().Debug().Strs("interfaces", names).Msg(noiseDeltaMessage) + if summary, emit := p.networkNoise.add(names, networkEventsNowFn()); emit { + logNoiseSummary(summary) + } +} + +// flushNoiseSummary closes the open noise window, because the delta that the +// daemon acts on ends the storm. +func (p *prog) flushNoiseSummary() { + if summary, emit := p.networkNoise.flush(networkEventsNowFn()); emit { + logNoiseSummary(summary) + } +} + +// logNoiseSummary writes one journal line for a window of noise deltas. +func logNoiseSummary(summary noiseSummary) { + journal(mainLog.Load().Info()). + Int("count", summary.Count). + Strs("interfaces", summary.Interfaces). + Str("first_at", summary.First.Format(time.RFC3339)). + Str("last_at", summary.Last.Format(time.RFC3339)). + Msg(noiseDeltaMessage) +} + +// logInterfaceChanges puts the interfaces of one transition in the journal. +// Support reads back the port that lost its address at the time of an outage. +// An AirDrop or a container adapter changes every few seconds, so the +// transition names it and the journal keeps the interfaces that carry the +// traffic of the host. +func logInterfaceChanges(transitionID uint64, changes []interfaceChange) { + for _, change := range changes { + if noiseClass(change.Class) { + continue + } + journal(mainLog.Load().Info()). + Uint64("transition_id", transitionID). + Str("interface", change.Name). + Str("action", change.Action). + Str("class", change.Class). + Str("hardware_port", change.HardwarePort). + Str("service", change.Service). + Strs("ips_before", change.IPsBefore). + Strs("ips_after", change.IPsAfter). + Str("flags", change.Flags). + Int("mtu", change.MTU). + Bool("is_default_route", change.IsDefaultRoute). + Msg(interfaceChangedMessage) + } +} + +// networkTransitionEvent picks the level of one transition line. An ignored +// delta and a superseded snapshot arrive many times per minute, so the journal +// keeps the outcomes that can change the resolver state only. +func networkTransitionEvent(outcome string) *zerolog.Event { + if outcome == transitionOutcomeIgnored || outcome == transitionOutcomeSnapshotSuperseded { + return mainLog.Load().Debug() + } + return journal(mainLog.Load().Info()) +} + +// changedInterfaceNames names the interfaces of one delta, in the order of the +// diff. +func changedInterfaceNames(changes []interfaceChange) []string { + names := make([]string, 0, len(changes)) + for _, change := range changes { + names = append(names, change.Name) + } + return names +} + +// hasAddOrRemove reports a delta that changed the adapter set of the host. The +// platform names of an adapter come from a cache, and that cache holds nothing +// for an adapter that just appeared. +func hasAddOrRemove(changes []interfaceChange) bool { + for _, change := range changes { + if change.Action == "added" || change.Action == "removed" { + return true + } + } + return false +} + +func defaultRouteOf(state *netmon.State) string { + if state == nil { + return "" + } + return state.DefaultRouteInterface +} + +// upInterfaceCount counts the interfaces with the up flag. A netmon entry +// without an embedded interface counts as down. +func upInterfaceCount(state *netmon.State) int { + if state == nil { + return 0 + } + up := 0 + for _, iface := range state.Interface { + if iface.Interface != nil && iface.IsUp() { + up++ + } + } + return up +} diff --git a/cmd/cli/network_events_test.go b/cmd/cli/network_events_test.go new file mode 100644 index 0000000..c8f47c1 --- /dev/null +++ b/cmd/cli/network_events_test.go @@ -0,0 +1,804 @@ +package cli + +import ( + "fmt" + "net" + "net/netip" + "slices" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// hostWokeTestState names the network that a wake event reports. +func hostWokeTestState() *netmon.State { + return &netmon.State{ + DefaultRouteInterface: "en0", + Interface: map[string]netmon.Interface{ + "en0": {Interface: &net.Interface{Name: "en0", Flags: net.FlagUp}}, + "en5": {Interface: &net.Interface{Name: "en5"}}, + "utun": {Interface: &net.Interface{Name: "utun", Flags: net.FlagUp}}, + }, + InterfaceIPs: map[string][]netip.Prefix{}, + } +} + +// hostWokeTestClock hands the wake events a clock that the test moves. +func hostWokeTestClock(t *testing.T, clock *time.Time) { + t.Helper() + original := networkEventsNowFn + t.Cleanup(func() { networkEventsNowFn = original }) + networkEventsNowFn = func() time.Time { return *clock } +} + +// hostWokeTestHold captures the hold timer of the wake reporter, so a test +// decides when the hold ends. +func hostWokeTestHold(p *prog) *[]func() { + var armed []func() + p.wake.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + return &armed +} + +// Test_noteHostWokeNetmonFirstWaitsForTheDetectorGap covers the order of the +// 2026-09-14 capture. netmon sees the time jump first and measures no gap, so +// its report waits for the detector, and the journal gets one event with the +// gap. +func Test_noteHostWokeNetmonFirstWaitsForTheDetectorGap(t *testing.T) { + logs := captureDebugMainLog(t) + clock := time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC) + hostWokeTestClock(t, &clock) + p := &prog{dnsConfig: newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil })} + armed := hostWokeTestHold(p) + + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if events := jsonLogEvents(t, logs, hostWokeMessage); len(events) != 0 { + t.Fatalf("got %d Host woke events before the hold ended, want 0", len(events)) + } + if got := p.dnsConfig.nextDelay(clock); got != dnsConfigFastInterval { + t.Fatalf("DNS configuration poll delay after a wake = %s, want %s", got, dnsConfigFastInterval) + } + clock = clock.Add(2 * time.Second) + p.noteHostWoke("detector", 90*time.Second, hostWokeTestState()) + for _, fire := range *armed { + fire() + } + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 { + t.Fatalf("got %d Host woke events, want one report with the gap of the detector", len(events)) + } + wantField(t, events[0], "source", "detector") + wantField(t, events[0], "gap_known", true) + wantField(t, events[0], "gap_ms", float64(90000)) + wantField(t, events[0], "default_route", "en0") + wantField(t, events[0], "interfaces_up", float64(2)) + wantField(t, events[0], "journal", true) + + clock = clock.Add(5 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if events = jsonLogEvents(t, logs, hostWokeMessage); len(events) != 1 { + t.Fatalf("got %d Host woke events, want no second report inside the window", len(events)) + } +} + +// Test_noteHostWokeNetmonAloneReportsAfterTheHold covers a host without the +// detector. The netmon report goes out without a gap when the hold ends. +func Test_noteHostWokeNetmonAloneReportsAfterTheHold(t *testing.T) { + logs := captureDebugMainLog(t) + clock := time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC) + hostWokeTestClock(t, &clock) + p := &prog{} + armed := hostWokeTestHold(p) + + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if len(*armed) != 1 { + t.Fatalf("holds armed = %d, want 1", len(*armed)) + } + (*armed)[0]() + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 { + t.Fatalf("got %d Host woke events after the hold, want 1", len(events)) + } + wantField(t, events[0], "source", "netmon") + wantField(t, events[0], "gap_known", false) + if _, measured := events[0]["gap_ms"]; measured { + t.Fatalf("the netmon report carries a gap that it cannot measure: %v", events[0]) + } +} + +// Test_noteHostWokeDetectorFirstReportsOneEvent covers the other order. The +// gap is known with the first report, so netmon adds nothing. +func Test_noteHostWokeDetectorFirstReportsOneEvent(t *testing.T) { + logs := captureDebugMainLog(t) + resume := time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC) + clock := resume + hostWokeTestClock(t, &clock) + p := &prog{} + armed := hostWokeTestHold(p) + + p.noteHostWoke("detector", 90*time.Second, hostWokeTestState()) + clock = resume.Add(2 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 || len(*armed) != 0 { + t.Fatalf("got %d Host woke events and %d holds inside the window, want 1 and 0", len(events), len(*armed)) + } + wantField(t, events[0], "source", "detector") + wantField(t, events[0], "gap_ms", float64(90000)) + + // netmon polls every 15 s, so the window has to outlast one poll. + clock = resume.Add(29 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if events = jsonLogEvents(t, logs, hostWokeMessage); len(events) != 1 || len(*armed) != 0 { + t.Fatalf("got %d Host woke events and %d holds, want one report for a wake that netmon reports late", len(events), len(*armed)) + } + + clock = resume.Add(31 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + for _, fire := range *armed { + fire() + } + if events = jsonLogEvents(t, logs, hostWokeMessage); len(events) != 2 { + t.Fatalf("got %d Host woke events after the window, want 2", len(events)) + } +} + +// networkEventsHarness drives the network callback on a clock that the test +// owns. Every host read stays behind a stub, so no test starts networksetup or +// reads the route table of the machine. +type networkEventsHarness struct { + prog *prog + logs *syncBuffer + now time.Time + reconciled int + ignored int +} + +func newNetworkEventsHarness(t *testing.T, valid ...string) *networkEventsHarness { + t.Helper() + sourceTestGlobals(t) + stubHeaderSnapshotSources(t) + stubSnapshotVirtualSet(t) + h := &networkEventsHarness{ + prog: &prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}, + logs: captureDebugMainLog(t), + now: time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC), + } + oldNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return h.now } + t.Cleanup(func() { networkEventsNowFn = oldNow }) + validSet := make(map[string]struct{}, len(valid)) + for _, name := range valid { + validSet[name] = struct{}{} + } + networkChangeValidInterfacesFn = func() map[string]struct{} { return validSet } + networkChangeReconcileFn = func(_ *prog, _ uint64) { h.reconciled++ } + networkChangeIgnoredInterceptFn = func(_ *prog, _ *netmon.ChangeDelta, _ time.Time) { h.ignored++ } + return h +} + +func (h *networkEventsHarness) handle(before, after *netmon.State, timeJumped bool) { + h.prog.handleNetworkChange(&netmon.ChangeDelta{Old: before, New: after, TimeJumped: timeJumped}, true) +} + +// stubNetmonCache gives every callback of the harness one cache snapshot. +// netmon caches major snapshots only, so the epoch of a minor callback stays +// at that snapshot. +func (h *networkEventsHarness) stubNetmonCache(t *testing.T, cache *netmon.State) { + t.Helper() + oldCurrent, oldRead := networkChangeCurrentStateFn, readNetworkSourceStateFn + t.Cleanup(func() { networkChangeCurrentStateFn, readNetworkSourceStateFn = oldCurrent, oldRead }) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return cache } + readNetworkSourceStateFn = func() (*netmon.State, error) { return cache, nil } +} + +// handleMinor drives one minor callback. netmon puts the cached major snapshot +// in Old, whatever the callback before it reported. +func (h *networkEventsHarness) handleMinor(cache, after *netmon.State) { + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: netmon.NewStatic(), Old: cache, New: after}, false) +} + +// interfaceEventsFor counts the journal events of one interface. +func (h *networkEventsHarness) interfaceEventsFor(t *testing.T, name string) int { + t.Helper() + count := 0 + for _, event := range jsonLogEvents(t, h.logs, interfaceChangedMessage) { + if event["interface"] == name { + count++ + } + } + return count +} + +// eventsAtLevel selects the captured events of one message and one level. +func (h *networkEventsHarness) eventsAtLevel(t *testing.T, message, level string) []map[string]any { + t.Helper() + var selected []map[string]any + for _, event := range jsonLogEvents(t, h.logs, message) { + if event["level"] == level { + selected = append(selected, event) + } + } + return selected +} + +var networkEventsHardwareIface = deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, +} + +// networkEventsAddressChange returns a delta that moves the address of en0. +func networkEventsAddressChange(t *testing.T) (before, after *netmon.State) { + t.Helper() + return deltaTestState(t, "en0", networkEventsHardwareIface), + deltaTestState(t, "en0", deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.20/24"}, + }) +} + +// transitionEvent returns the one transition event of a callback. +func (h *networkEventsHarness) transitionEvent(t *testing.T) map[string]any { + t.Helper() + events := jsonLogEvents(t, h.logs, networkTransitionMessage) + if len(events) != 1 { + t.Fatalf("got %d %q events, want 1", len(events), networkTransitionMessage) + } + return events[0] +} + +// wantNoField fails when an event carries a field that it must not. +func wantNoField(t *testing.T, event map[string]any, field string) { + t.Helper() + if value, carried := event[field]; carried { + t.Fatalf("field %q: got %v, want no field", field, value) + } +} + +// noiseDeltaStates returns a delta that moves the AirDrop link-local address. +func noiseDeltaStates(t *testing.T, step int) (before, after *netmon.State) { + t.Helper() + airdrop := func(address int) deltaTestInterface { + return deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{fmt.Sprintf("fe80::%d/64", address)}} + } + return deltaTestState(t, "en0", networkEventsHardwareIface, airdrop(step)), + deltaTestState(t, "en0", networkEventsHardwareIface, airdrop(step+1)) +} + +func wantStringsField(t *testing.T, event map[string]any, field string, want ...string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("field %q: got %v, want a list", field, event[field]) + } + got := make([]string, 0, len(values)) + for _, value := range values { + text, isText := value.(string) + if !isText { + t.Fatalf("field %q: got %v, want strings", field, event[field]) + } + got = append(got, text) + } + if !slices.Equal(got, want) { + t.Fatalf("field %q: got %v, want %v", field, got, want) + } +} + +func Test_networkEventsNoiseDeltaSkipsTheHandler(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + + before, after := noiseDeltaStates(t, 1) + h.handle(before, after, false) + + if h.reconciled != 0 || h.ignored != 0 { + t.Fatalf("noise delta ran the handler: reconciled=%d ignored=%d, want 0/0", h.reconciled, h.ignored) + } + debugLines := h.eventsAtLevel(t, noiseDeltaMessage, "debug") + if len(debugLines) != 1 { + t.Fatalf("got %d debug %q lines, want 1", len(debugLines), noiseDeltaMessage) + } + wantStringsField(t, debugLines[0], "interfaces", "awdl0") + if events := jsonLogEvents(t, h.logs, networkTransitionMessage); len(events) != 0 { + t.Fatalf("got %d %q events for a noise delta, want 0", len(events), networkTransitionMessage) + } +} + +func Test_networkEventsNoiseStormEmitsOneSummaryPerWindow(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + start := h.now + + for step := 0; step <= 10; step++ { + h.now = start.Add(time.Duration(step) * time.Minute) + before, after := noiseDeltaStates(t, step) + h.handle(before, after, false) + } + + summaries := h.eventsAtLevel(t, noiseDeltaMessage, "info") + if len(summaries) != 2 { + t.Fatalf("got %d journal %q events, want 2", len(summaries), noiseDeltaMessage) + } + wantField(t, summaries[0], "journal", true) + wantField(t, summaries[0], "count", float64(1)) + wantField(t, summaries[1], "count", float64(10)) + wantField(t, summaries[1], "first_at", start.Add(time.Minute).Format(time.RFC3339)) + wantField(t, summaries[1], "last_at", start.Add(10*time.Minute).Format(time.RFC3339)) + wantStringsField(t, summaries[1], "interfaces", "awdl0") +} + +func Test_networkEventsLogsEveryChangedInterface(t *testing.T) { + h := newNetworkEventsHarness(t, "en0", "en5") + before := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1500, ips: []string{"192.0.2.10/24", "2001:db8::10/64"}}, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.10/24"}}, + ) + before.HaveV4, before.HaveV6 = true, true + after := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1500, ips: []string{"192.0.2.20/24"}}, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.20/24"}}, + ) + after.HaveV4, after.HaveV6 = true, false + + h.handle(before, after, false) + + changes := jsonLogEvents(t, h.logs, interfaceChangedMessage) + if len(changes) != 2 { + t.Fatalf("got %d %q events, want 2", len(changes), interfaceChangedMessage) + } + for index, name := range []string{"en0", "en5"} { + wantField(t, changes[index], "journal", true) + wantField(t, changes[index], "level", "info") + wantField(t, changes[index], "interface", name) + wantField(t, changes[index], "action", "ip_removed") + wantField(t, changes[index], "class", "hardware") + wantField(t, changes[index], "transition_id", float64(1)) + wantField(t, changes[index], "mtu", float64(1500)) + } + wantField(t, changes[0], "is_default_route", true) + wantField(t, changes[1], "is_default_route", false) + wantStringsField(t, changes[0], "ips_before", "192.0.2.10/24", "2001:db8::10/64") + wantStringsField(t, changes[0], "ips_after", "192.0.2.20/24") + + transitions := jsonLogEvents(t, h.logs, networkTransitionMessage) + if len(transitions) != 1 { + t.Fatalf("got %d %q events, want 1", len(transitions), networkTransitionMessage) + } + wantField(t, transitions[0], "journal", true) + wantField(t, transitions[0], "level", "info") + wantField(t, transitions[0], "outcome", "accepted") + wantField(t, transitions[0], "time_jumped", false) + wantField(t, transitions[0], "have_v4_before", true) + wantField(t, transitions[0], "have_v4_after", true) + wantField(t, transitions[0], "have_v6_before", true) + wantField(t, transitions[0], "have_v6_after", false) + wantStringsField(t, transitions[0], "changed_interfaces", "en0", "en5") + if h.reconciled != 1 { + t.Fatalf("reconciled = %d, want 1", h.reconciled) + } + if snapshots := jsonLogEvents(t, h.logs, networkSnapshotMessage); len(snapshots) != 1 { + t.Fatalf("got %d %q events, want 1", len(snapshots), networkSnapshotMessage) + } +} + +func Test_networkEventsTimeJumpReportsTheWake(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + armed := hostWokeTestHold(h.prog) + before, after := noiseDeltaStates(t, 1) + + h.handle(before, after, true) + for _, fire := range *armed { + fire() + } + + wakes := jsonLogEvents(t, h.logs, hostWokeMessage) + if len(wakes) != 1 { + t.Fatalf("got %d %q events, want 1", len(wakes), hostWokeMessage) + } + wantField(t, wakes[0], "journal", true) + wantField(t, wakes[0], "source", "netmon") + wantField(t, wakes[0], "gap_known", false) + wantField(t, wakes[0], "default_route", "en0") + if _, measured := wakes[0]["gap_ms"]; measured { + t.Fatalf("the netmon report carries a gap that it cannot measure: %v", wakes[0]) + } +} + +func Test_networkEventsStoreTheIPv6Flag(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + // The one-time probe wins over the first store, so consume it first. + available := ctrld.HasIPv6() + t.Cleanup(func() { ctrld.SetIPv6Available(available) }) + ctrld.SetIPv6Available(true) + + before := deltaTestState(t, "en0", networkEventsHardwareIface) + before.HaveV6 = true + after := deltaTestState(t, "en0", deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.20/24"}, + }) + after.HaveV6 = false + + h.handle(before, after, false) + + if ctrld.HasIPv6() { + t.Fatal("HasIPv6 = true after a callback with have_v6 false, want false") + } +} + +// Test_networkEventsMinorDeltaDiffsAgainstTheStateBeforeIt covers a difference +// that lasts. The cached major snapshot holds no link-local address of en0, so +// a diff against the cache reports the same change in every minor callback. +func Test_networkEventsMinorDeltaDiffsAgainstTheStateBeforeIt(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + en0WithLinkLocal := deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "fe80::a/64"}, + } + airdrop := func(address int) deltaTestInterface { + return deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{fmt.Sprintf("fe80::%d/64", address)}} + } + cache := deltaTestState(t, "en0", networkEventsHardwareIface, airdrop(1)) + h.stubNetmonCache(t, cache) + + h.handleMinor(cache, deltaTestState(t, "en0", en0WithLinkLocal, airdrop(2))) + h.handleMinor(cache, deltaTestState(t, "en0", en0WithLinkLocal, airdrop(3))) + + if got := h.interfaceEventsFor(t, "en0"); got != 1 { + t.Fatalf("got %d %q events for en0, want 1", got, interfaceChangedMessage) + } + if noise := h.eventsAtLevel(t, noiseDeltaMessage, "debug"); len(noise) != 1 { + t.Fatalf("got %d debug %q lines, want 1 for the second delta", len(noise), noiseDeltaMessage) + } +} + +// Test_networkEventsNoiseDeltaReadsNoPlatformMeta pins the class-first order. +// The macOS meta read starts two subprocesses per call, so a storm of noise +// deltas must not reach it. +func Test_networkEventsNoiseDeltaReadsNoPlatformMeta(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + reads := countPlatformMetaReads(t) + + before, after := noiseDeltaStates(t, 1) + h.handle(before, after, false) + + if *reads != 0 { + t.Fatalf("a noise delta read the platform meta %d times, want 0", *reads) + } +} + +// countPlatformMetaReads counts the platform meta reads of a test. One read +// starts two subprocesses on macOS, so the count is the cost of a delta. +func countPlatformMetaReads(t *testing.T) *int { + t.Helper() + reads := 0 + original := platformInterfaceMetaFn + t.Cleanup(func() { platformInterfaceMetaFn = original }) + platformInterfaceMetaFn = func(string) (class, hardwarePort, service string) { + reads++ + return "", "", "" + } + return &reads +} + +// Test_networkEventsNoiseStormEndsWithTheNextTransition covers the last window +// of a storm. The count of the deltas that a real change follows must not wait +// for the next storm. +func Test_networkEventsNoiseStormEndsWithTheNextTransition(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + start := h.now + + for step := 0; step < 4; step++ { + h.now = start.Add(time.Duration(step) * time.Minute) + before, after := noiseDeltaStates(t, step) + h.handle(before, after, false) + } + h.now = start.Add(5 * time.Minute) + before, after := networkEventsAddressChange(t) + h.handle(before, after, false) + + summaries := h.eventsAtLevel(t, noiseDeltaMessage, "info") + if len(summaries) != 2 { + t.Fatalf("got %d journal %q events, want the first delta and the end of the storm", len(summaries), noiseDeltaMessage) + } + wantField(t, summaries[1], "count", float64(3)) + wantField(t, summaries[1], "first_at", start.Add(time.Minute).Format(time.RFC3339)) + wantField(t, summaries[1], "last_at", start.Add(3*time.Minute).Format(time.RFC3339)) + wantStringsField(t, summaries[1], "interfaces", "awdl0") +} + +// Test_networkEventsStoreTheNetworkBeforeTheHeaderRender proves that the +// callback stores its network under the source lock and renders the header +// after the unlock, so the header read of the hardware ports starts no +// process under the lock. +func Test_networkEventsStoreTheNetworkBeforeTheHeaderRender(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + lockHeld, renders := false, 0 + original := defaultRoutesFn + t.Cleanup(func() { defaultRoutesFn = original }) + // The header render reads the routes, so this seam runs where the + // callback renders the header. + defaultRoutesFn = func() (defaultRoute, defaultRoute) { + renders++ + if h.prog.networkSourceMu.TryLock() { + h.prog.networkSourceMu.Unlock() + } else { + lockHeld = true + } + return defaultRoute{Gateway: "192.0.2.1"}, defaultRoute{} + } + + before, after := networkEventsAddressChange(t) + h.handle(before, after, false) + + if renders == 0 { + t.Fatal("the callback rendered no header") + } + if lockHeld { + t.Fatal("the header render ran under the source lock") + } + if h.prog.lastNetworkState.Load() != after { + t.Fatal("the callback did not store its network") + } +} + +// Test_networkEventsIgnoredTransitionStaysOutOfTheJournal pins the level of an +// ignored delta. Such a delta arrives many times per minute, so it belongs in +// the debug stream and not in the journal. +func Test_networkEventsIgnoredTransitionStaysOutOfTheJournal(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + before := deltaTestState(t, "en0", networkEventsHardwareIface, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.10/24"}}) + after := deltaTestState(t, "en0", networkEventsHardwareIface, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.20/24"}}) + + h.handle(before, after, false) + + transition := h.transitionEvent(t) + wantField(t, transition, "outcome", transitionOutcomeIgnored) + wantField(t, transition, "level", "debug") + wantNoField(t, transition, "journal") + if snapshots := jsonLogEvents(t, h.logs, networkSnapshotMessage); len(snapshots) != 0 { + t.Fatalf("got %d %q events for an ignored delta, want 0", len(snapshots), networkSnapshotMessage) + } + if h.ignored != 1 || h.reconciled != 0 { + t.Fatalf("ignored delta: ignored=%d reconciled=%d, want 1/0", h.ignored, h.reconciled) + } +} + +// Test_networkEventsSupersededTransitionEntersTheJournal pins the level of a +// delta that a newer accepted delta replaced. Support reads it to tell a lost +// reconcile from one that never started. +func Test_networkEventsSupersededTransitionEntersTheJournal(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + h.prog.networkAcceptedGen.Store(5) + + before, after := networkEventsAddressChange(t) + h.handle(before, after, false) + + transition := h.transitionEvent(t) + wantField(t, transition, "outcome", "superseded") + wantField(t, transition, "level", "info") + wantField(t, transition, "journal", true) + if h.reconciled != 0 { + t.Fatalf("a superseded delta reconciled %d times, want 0", h.reconciled) + } +} + +// Test_networkEventsSupersededSnapshotReportsTheOutcome covers the delta whose +// snapshot netmon already replaced. The callback logged nothing at all, so the +// journal showed a silent gap between two deltas. +func Test_networkEventsSupersededSnapshotReportsTheOutcome(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + cache := deltaTestState(t, "en0", networkEventsHardwareIface) + h.stubNetmonCache(t, cache) + reads := countPlatformMetaReads(t) + stale, after := networkEventsAddressChange(t) + + // netmon holds a newer snapshot than the one that this callback carries. + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: netmon.NewStatic(), Old: stale, New: after}, false) + + transition := h.transitionEvent(t) + wantField(t, transition, "outcome", "snapshot_superseded") + wantField(t, transition, "level", "debug") + wantNoField(t, transition, "journal") + wantStringsField(t, transition, "changed_interfaces", "en0") + if events := jsonLogEvents(t, h.logs, interfaceChangedMessage); len(events) != 0 { + t.Fatalf("got %d %q events for a superseded snapshot, want 0", len(events), interfaceChangedMessage) + } + if h.reconciled != 0 || h.ignored != 0 { + t.Fatalf("superseded snapshot ran the handler: reconciled=%d ignored=%d, want 0/0", h.reconciled, h.ignored) + } + if *reads != 0 { + t.Fatalf("a superseded snapshot read the platform meta %d times, want 0", *reads) + } +} + +// Test_networkEventsLogsTheInterfacesOutsideTheNoiseClass covers a mixed +// delta. An AirDrop change beside a real one must not fill the journal, but +// the transition still names every interface that changed. +func Test_networkEventsLogsTheInterfacesOutsideTheNoiseClass(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + before := deltaTestState(t, "en0", networkEventsHardwareIface, + deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{"fe80::1/64"}}) + after := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.20/24"}}, + deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{"fe80::2/64"}}) + + h.handle(before, after, false) + + events := jsonLogEvents(t, h.logs, interfaceChangedMessage) + if len(events) != 1 { + t.Fatalf("got %d %q events, want one for the hardware port", len(events), interfaceChangedMessage) + } + wantField(t, events[0], "interface", "en0") + wantField(t, events[0], "class", "hardware") + wantStringsField(t, h.transitionEvent(t), "changed_interfaces", "awdl0", "en0") +} + +// reorderTestState builds a state with en0 on one address and awdl0 on one +// link-local address, so a test can replay callbacks out of order. +func reorderTestState(en0 string, awdl string) *netmon.State { + return &netmon.State{ + Interface: map[string]netmon.Interface{ + "en0": {Interface: &net.Interface{Name: "en0", Flags: net.FlagUp}}, + "awdl0": {Interface: &net.Interface{Name: "awdl0", Flags: net.FlagUp}}, + }, + InterfaceIPs: map[string][]netip.Prefix{ + "en0": {netip.MustParsePrefix(en0)}, + "awdl0": {netip.MustParsePrefix(awdl)}, + }, + DefaultRouteInterface: "en0", + HaveV4: true, + } +} + +// firstV4Of returns the first IPv4 address of one interface of a state. +func firstV4Of(state *netmon.State, name string) net.IP { + for _, prefix := range state.InterfaceIPs[name] { + if prefix.Addr().Is4() { + return net.IP(prefix.Addr().AsSlice()) + } + } + return nil +} + +// Test_networkEventsSupersededCallbackKeepsTheBaseline replays the callbacks +// out of order: B runs, a late A arrives while the monitor already caches C, +// then C runs. The late A must not become the baseline of C, or the address +// change from B to C looks like AirDrop noise and the stale source stays. +func Test_networkEventsSupersededCallbackKeepsTheBaseline(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + stateA := reorderTestState("192.0.2.20/24", "fe80::1/64") + stateB := reorderTestState("192.0.2.10/24", "fe80::1/64") + stateC := reorderTestState("192.0.2.20/24", "fe80::2/64") + cache := stateB + oldCurrent := networkChangeCurrentStateFn + t.Cleanup(func() { networkChangeCurrentStateFn = oldCurrent }) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return cache } + networkChangeReconcileFn = func(p *prog, _ uint64) { + h.reconciled++ + ctrld.SetDefaultLocalIPv4(firstV4Of(p.networkSourceState, "en0")) + } + monitor := netmon.NewStatic() + + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: monitor, Old: stateA, New: stateB}, true) + cache = stateC + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: monitor, Old: &netmon.State{}, New: stateA}, true) + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: monitor, Old: stateB, New: stateC}, true) + + if h.reconciled != 2 { + t.Fatalf("reconciles = %d, want 2: the late callback must not hide the change from B to C", h.reconciled) + } + if got := ctrld.GetDefaultLocalIPv4().String(); got != "192.0.2.20" { + t.Fatalf("resolver source = %s, want 192.0.2.20", got) + } +} + +// orderTestState builds a state for the callback order tests: en0 on one +// address, plus extra interfaces with their addresses and their up flag. +func orderTestState(route, en0 string, extra map[string][]string, down map[string]bool) *netmon.State { + state := &netmon.State{ + Interface: map[string]netmon.Interface{}, + InterfaceIPs: map[string][]netip.Prefix{}, + DefaultRouteInterface: route, + HaveV4: true, + } + add := func(name string, up bool, ips []string) { + flags := net.Flags(0) + if up { + flags = net.FlagUp + } + state.Interface[name] = netmon.Interface{Interface: &net.Interface{Name: name, Flags: flags}} + for _, ip := range ips { + state.InterfaceIPs[name] = append(state.InterfaceIPs[name], netip.MustParsePrefix(ip)) + } + } + if en0 != "" { + add("en0", true, []string{en0}) + } + for name, ips := range extra { + add(name, !down[name], ips) + } + return state +} + +// Test_networkEventsMajorAfterItsMinorKeepsItsChanges runs a minor callback +// of the new epoch before the major callback that opened the epoch. netmon +// starts one goroutine per callback, so this order happens. The major must +// still report the address change and reconcile. +func Test_networkEventsMajorAfterItsMinorKeepsItsChanges(t *testing.T) { + for _, order := range []string{"in_order", "minor_first", "minor_first_time_jumped"} { + t.Run(order, func(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + hostWokeTestHold(h.prog) + s0 := orderTestState("en0", "192.0.2.10/24", map[string][]string{"awdl0": {"fe80::1/64"}}, nil) + s2 := orderTestState("en0", "192.0.2.20/24", map[string][]string{"awdl0": {"fe80::1/64"}}, nil) + s3 := orderTestState("en0", "192.0.2.20/24", map[string][]string{"awdl0": {"fe80::2/64"}}, nil) + h.stubNetmonCache(t, s2) + monitor := netmon.NewStatic() + major := &netmon.ChangeDelta{Monitor: monitor, Old: s0, New: s2, TimeJumped: order == "minor_first_time_jumped"} + minor := &netmon.ChangeDelta{Monitor: monitor, Old: s2, New: s3} + if order == "in_order" { + h.prog.handleNetworkChange(major, true) + h.prog.handleNetworkChange(minor, false) + } else { + h.prog.handleNetworkChange(minor, false) + h.prog.handleNetworkChange(major, true) + } + + if h.reconciled != 1 { + t.Fatalf("reconciled = %d, want 1 for the move of en0 from .10 to .20", h.reconciled) + } + if got := h.interfaceEventsFor(t, "en0"); got != 1 { + t.Fatalf("en0 interface events = %d, want 1", got) + } + if h.prog.networkDeltaState != s3 { + t.Fatal("the baseline must stay at the newest state of the epoch") + } + }) + } +} + +// Test_networkEventsEpochChangeReportsAMinorChangeOnce closes an epoch with a +// major callback whose Old is the cache of the epoch. The interface that a +// minor callback of that epoch reported must not appear again. +func Test_networkEventsEpochChangeReportsAMinorChangeOnce(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + s0 := orderTestState("en0", "192.0.2.10/24", nil, nil) + s1 := orderTestState("en0", "192.0.2.10/24", map[string][]string{"ipsec0": {"10.0.0.2/32"}}, nil) + s2 := orderTestState("en0", "192.0.2.20/24", map[string][]string{"ipsec0": {"10.0.0.2/32"}}, nil) + cache := s0 + oldCurrent, oldRead := networkChangeCurrentStateFn, readNetworkSourceStateFn + t.Cleanup(func() { networkChangeCurrentStateFn, readNetworkSourceStateFn = oldCurrent, oldRead }) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return cache } + readNetworkSourceStateFn = func() (*netmon.State, error) { return cache, nil } + monitor := netmon.NewStatic() + + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: monitor, Old: s0, New: s1}, false) + cache = s2 + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: monitor, Old: s0, New: s2}, true) + + if got := h.interfaceEventsFor(t, "ipsec0"); got != 1 { + t.Fatalf("ipsec0 interface events = %d, want 1", got) + } + if got := h.interfaceEventsFor(t, "en0"); got != 1 { + t.Fatalf("en0 interface events = %d, want 1", got) + } +} + +// Test_networkEventsNewValidInterfaceReconciles plugs a valid interface in +// with its address in one delta while the other valid interface is down. The +// new interface is active, so the change must reconcile. +func Test_networkEventsNewValidInterfaceReconciles(t *testing.T) { + h := newNetworkEventsHarness(t, "en0", "en7") + before := orderTestState("", "", map[string][]string{"en0": nil}, map[string]bool{"en0": true}) + before.HaveV4 = false + after := orderTestState("en7", "", map[string][]string{"en0": nil, "en7": {"192.0.2.30/24"}}, map[string]bool{"en0": true}) + + h.handle(before, after, false) + + if h.reconciled != 1 { + event := h.transitionEvent(t) + t.Fatalf("reconciled = %d, want 1; outcome %v", h.reconciled, event["outcome"]) + } +} diff --git a/cmd/cli/network_snapshot.go b/cmd/cli/network_snapshot.go new file mode 100644 index 0000000..9af1c02 --- /dev/null +++ b/cmd/cli/network_snapshot.go @@ -0,0 +1,301 @@ +package cli + +import ( + "cmp" + "net/netip" + "slices" + "strings" + + "github.com/rs/zerolog" + "tailscale.com/net/netmon" + "tailscale.com/net/tsaddr" +) + +// hardwarePortIPhoneUSB is the macOS port name of a phone that shares its +// connection over the cable. +const hardwarePortIPhoneUSB = "iPhone USB" + +// snapshotInterfaceLimit bounds the interface list of one snapshot. A +// container host holds hundreds of interfaces, and each entry costs bytes in +// every journal line and in every log file header. +const snapshotInterfaceLimit = 32 + +// v6GlobalPrefix holds the v6 addresses that reach the Internet. +var v6GlobalPrefix = netip.MustParsePrefix("2000::/3") + +// hotspotGatewayPrefix is the range that an iPhone hotspot hands out. A +// gateway inside it names a tethered host on every platform. +var hotspotGatewayPrefix = netip.MustParsePrefix("172.20.10.0/28") + +// ethernetPortWords name a wired port. macOS spells the same link in several +// ways, one per adapter family. +var ethernetPortWords = []string{"Ethernet", "LAN", "Thunderbolt"} + +// defaultRoute names the next hop and the interface of one default route. Both +// values come from the same entry of the route table, so a reader can trust +// the pair. +type defaultRoute struct { + Gateway string + Interface string +} + +// snapshotInterface is one interface of a network snapshot. +type snapshotInterface struct { + Name string + Class string + Up bool + IPs []string + HardwarePort string + Service string +} + +// networkSnapshot describes the host network at one moment. The journal event +// and the header of a log file render the same value. +type networkSnapshot struct { + DefaultRouteV4 string + DefaultRouteV6 string + GatewayV4 string + GatewayV6 string + HaveV4 bool + HaveV6 bool + Interfaces []snapshotInterface + InterfacesOmitted int + Resolvers []string + SourceIPv4 string + SourceIPv6 string + InterceptTarget string + BypassActive bool + RecoveryRunning bool + PFStabilizing bool + LinkType string + Tethered bool + CLATPresent bool + NAT64Prefix string + DNSLessTarget bool +} + +// snapshotInputs holds every value a snapshot needs. The caller reads them, so +// the builder stays free of locks and of system calls. +type snapshotInputs struct { + State *netmon.State + RouteV4 defaultRoute + RouteV6 defaultRoute + Resolvers []string + SourceIPv4 string + SourceIPv6 string + InterceptTarget string + BypassActive bool + RecoveryRunning bool + PFStabilizing bool + NAT64Prefix string + CLATPresent bool + Meta interfaceMetaFunc +} + +// buildNetworkSnapshot makes the snapshot of one moment. It is pure, so a +// caller can take the inputs under a lock and build outside it. +func buildNetworkSnapshot(in snapshotInputs) networkSnapshot { + snapshot := networkSnapshot{ + GatewayV4: in.RouteV4.Gateway, + GatewayV6: in.RouteV6.Gateway, + DefaultRouteV4: in.RouteV4.Interface, + DefaultRouteV6: in.RouteV6.Interface, + Resolvers: in.Resolvers, + SourceIPv4: in.SourceIPv4, + SourceIPv6: in.SourceIPv6, + InterceptTarget: in.InterceptTarget, + BypassActive: in.BypassActive, + RecoveryRunning: in.RecoveryRunning, + PFStabilizing: in.PFStabilizing, + NAT64Prefix: in.NAT64Prefix, + CLATPresent: in.CLATPresent, + DNSLessTarget: in.InterceptTarget != "", + } + + routeMeta := interfaceMeta{} + var routeAddresses []netip.Prefix + if in.State != nil { + meta := interfaceMetaWithClass(in.Meta) + routeMeta = meta(in.State.DefaultRouteInterface) + routeAddresses = in.State.InterfaceIPs[in.State.DefaultRouteInterface] + snapshot.HaveV4, snapshot.HaveV6 = in.State.HaveV4, in.State.HaveV6 + snapshot.Interfaces, snapshot.InterfacesOmitted = snapshotInterfaces(in.State, meta) + if snapshot.DefaultRouteV4 == "" { + snapshot.DefaultRouteV4 = defaultRouteV4Interface(in.State) + } + } + snapshot.LinkType = linkTypeFor(routeMeta.HardwarePort, routeMeta.Class) + snapshot.Tethered = tetheredNetwork(in.RouteV4.Gateway, routeMeta.HardwarePort, routeAddresses) + return snapshot +} + +// interfaceMetaWithClass fills the class that the platform metadata leaves +// empty. A host that names no port still tells a tunnel from a hardware port +// by the interface name. +func interfaceMetaWithClass(meta interfaceMetaFunc) interfaceMetaFunc { + return func(name string) interfaceMeta { + info := interfaceMeta{} + if meta != nil { + info = meta(name) + } + if info.Class == "" { + info.Class = interfaceClass(name, false) + } + return info + } +} + +// snapshotInterfaces lists the interfaces of the state, sorted by name, and +// counts the ones it leaves out. A reader compares two snapshots line by line, +// so the order must not move. An interface that is down and holds no address +// carries no traffic, and a container host holds hundreds of them. +func snapshotInterfaces(state *netmon.State, meta interfaceMetaFunc) (list []snapshotInterface, omitted int) { + list = make([]snapshotInterface, 0, len(state.Interface)) + for name, iface := range state.Interface { + up := iface.Interface != nil && iface.IsUp() + ips := prefixStrings(state.InterfaceIPs[name]) + if !up && len(ips) == 0 { + omitted++ + continue + } + info := meta(name) + list = append(list, snapshotInterface{ + Name: name, + Class: info.Class, + Up: up, + IPs: ips, + HardwarePort: info.HardwarePort, + Service: info.Service, + }) + } + slices.SortFunc(list, func(a, b snapshotInterface) int { + return cmp.Compare(a.Name, b.Name) + }) + if len(list) > snapshotInterfaceLimit { + omitted += len(list) - snapshotInterfaceLimit + list = list[:snapshotInterfaceLimit] + } + return list, omitted +} + +// defaultRouteV4Interface names the interface that the network monitor holds +// for the default route. It fills the v4 route of a platform that reads no +// route table. An interface without a usable v4 address carries no v4 traffic, +// whatever the monitor says. +func defaultRouteV4Interface(state *netmon.State) string { + route := state.DefaultRouteInterface + if route == "" { + return "" + } + for _, prefix := range state.InterfaceIPs[route] { + if usableV4(prefix.Addr().Unmap()) { + return route + } + } + return "" +} + +// globalAddress reports whether an address reaches another host. +func globalAddress(address netip.Addr) bool { + return address.IsGlobalUnicast() && !address.IsLoopback() && !address.IsLinkLocalUnicast() +} + +// usableV4 and usableV6 grade one address the way the network monitor grades +// it. A fresh read of the interfaces must report the same families as a +// monitor callback, or two snapshots of one moment disagree. The monitor +// counts a unique local address of the v6 family, because some networks reach +// the Internet from it. The range of the Tailscale mesh is the exception, +// because that mesh carries no Internet traffic. +func usableV4(address netip.Addr) bool { + return address.Is4() && globalAddress(address) +} + +func usableV6(address netip.Addr) bool { + if !address.Is6() { + return false + } + if v6GlobalPrefix.Contains(address) { + return true + } + return address.IsPrivate() && !tsaddr.TailscaleULARange().Contains(address) +} + +// tetheredNetwork reports whether the host runs on a phone connection. A +// tethered host pays for its traffic and loses the link on every phone move. +// A platform that reads no route table gives no gateway, so the address that +// the phone handed to the host tells the same story. +func tetheredNetwork(gatewayV4, hardwarePort string, routeAddresses []netip.Prefix) bool { + if hardwarePort == hardwarePortIPhoneUSB { + return true + } + if gateway, err := netip.ParseAddr(gatewayV4); err == nil && hotspotGatewayPrefix.Contains(gateway.Unmap()) { + return true + } + for _, prefix := range routeAddresses { + if hotspotGatewayPrefix.Contains(prefix.Addr().Unmap()) { + return true + } + } + return false +} + +// linkTypeFor names the link of the default route. The port name tells more +// than the interface name, which is only a number on macOS. +func linkTypeFor(hardwarePort, class string) string { + switch { + case hardwarePort == "Wi-Fi": + return "wifi" + case hardwarePort == hardwarePortIPhoneUSB: + return "usb_tether" + case ethernetPort(hardwarePort): + return "ethernet" + case class == "tunnel": + return "tunnel" + } + return "unknown" +} + +func ethernetPort(hardwarePort string) bool { + for _, word := range ethernetPortWords { + if strings.Contains(hardwarePort, word) { + return true + } + } + return false +} + +// snapshotDict renders the snapshot with the field names of the spec. The +// journal event and the header line share it, so both stay in one shape. +func snapshotDict(snapshot networkSnapshot) *zerolog.Event { + interfaces := zerolog.Arr() + for _, iface := range snapshot.Interfaces { + interfaces = interfaces.Dict(zerolog.Dict(). + Str("name", iface.Name). + Str("class", iface.Class). + Bool("up", iface.Up). + Strs("ips", iface.IPs). + Str("hardware_port", iface.HardwarePort). + Str("service", iface.Service)) + } + return zerolog.Dict(). + Str("default_route_v4", snapshot.DefaultRouteV4). + Str("default_route_v6", snapshot.DefaultRouteV6). + Str("gateway_v4", snapshot.GatewayV4). + Str("gateway_v6", snapshot.GatewayV6). + Bool("have_v4", snapshot.HaveV4). + Bool("have_v6", snapshot.HaveV6). + Array("interfaces", interfaces). + Int("interfaces_omitted", snapshot.InterfacesOmitted). + Strs("resolvers", snapshot.Resolvers). + Str("source_ipv4", snapshot.SourceIPv4). + Str("source_ipv6", snapshot.SourceIPv6). + Str("intercept_target", snapshot.InterceptTarget). + Bool("bypass_active", snapshot.BypassActive). + Bool("recovery_running", snapshot.RecoveryRunning). + Bool("pf_stabilizing", snapshot.PFStabilizing). + Str("link_type", snapshot.LinkType). + Bool("tethered", snapshot.Tethered). + Bool("clat_present", snapshot.CLATPresent). + Str("nat64_prefix", snapshot.NAT64Prefix). + Bool("dns_less_target", snapshot.DNSLessTarget) +} diff --git a/cmd/cli/network_snapshot_prog.go b/cmd/cli/network_snapshot_prog.go new file mode 100644 index 0000000..22b1f43 --- /dev/null +++ b/cmd/cli/network_snapshot_prog.go @@ -0,0 +1,226 @@ +package cli + +import ( + "net" + "reflect" + "slices" + "sync" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// networkSnapshotMessage is the message of the snapshot event. A log tool +// selects every snapshot of a run by this message. +const networkSnapshotMessage = "Network snapshot" + +// snapshotTriggerStart names the snapshot of the start of a run. +const snapshotTriggerStart = "start" + +// snapshotMinInterval bounds the snapshots of a flapping network. +const snapshotMinInterval = 60 * time.Second + +// snapshotNowFn is the clock of the snapshot limit. A test replaces it. +var snapshotNowFn = time.Now + +// defaultRoutesFn reads the default route of each family. A test replaces it, +// because the route table of the host is not a fixture. +var defaultRoutesFn = defaultRoutes + +// platformInterfaceMetaFn reads the platform names of one interface. A test +// replaces it, because the ports of the host are not a fixture. +var platformInterfaceMetaFn = platformInterfaceMeta + +// The delta diff has to know the virtual adapters of the host, and only the +// platform file reads them. +func init() { + virtualInterfaceSetFn = platformVirtualInterfaces +} + +// interfaceMetaFor describes one interface for the snapshot and for the delta +// diff. +func interfaceMetaFor(name string) interfaceMeta { + platformClass, hardwarePort, service := platformInterfaceMetaFn(name) + meta := interfaceMeta{ + Class: interfaceClassFor(name, platformClass), + HardwarePort: hardwarePort, + Service: service, + } + meta.LinkType = linkTypeFor(hardwarePort, meta.Class) + return meta +} + +// interfaceClassFor picks the class of one interface. A name rule wins for the +// loopback, the AirDrop, and the tunnel interfaces, because Windows describes +// a tunnel adapter as an Ethernet adapter. +func interfaceClassFor(name, platformClass string) string { + if class := interfaceClass(name, false); class != "hardware" { + return class + } + if platformClass != "" { + return platformClass + } + if _, virtual := virtualInterfaceSetFn()[name]; virtual { + return "virtual" + } + return "hardware" +} + +// snapshotInputs reads the sources of one snapshot from the network that the +// last monitor callback reported. No callback ran before the first header of a +// run, so that header reads the network itself. +func (p *prog) snapshotInputs() snapshotInputs { + state := p.lastNetworkState.Load() + if state == nil { + state = freshNetworkState() + } + return p.snapshotInputsFor(state) +} + +// freshSnapshotInputs reads the sources of one snapshot from the network of +// this moment. Support reads the header of an upload to learn the network of +// the moment the user asked, which can differ from the last callback. +func (p *prog) freshSnapshotInputs() snapshotInputs { + return p.snapshotInputsFor(freshNetworkState()) +} + +// networkSnapshot describes the host network of this moment. +func (p *prog) networkSnapshot() networkSnapshot { + return buildNetworkSnapshot(p.snapshotInputs()) +} + +// logNetworkSnapshot puts the host network in the journal. Support reads the +// snapshot of each trigger to learn the network at the time of an outage. +func (p *prog) logNetworkSnapshot(trigger string) { + snapshot := p.networkSnapshot() + if !p.snapshotWrites.allow(trigger, snapshot, snapshotNowFn()) { + return + } + journal(mainLog.Load().Info()). + Str("trigger", trigger). + Dict("network", snapshotDict(snapshot)). + Msg(networkSnapshotMessage) +} + +// snapshotLimiter holds the snapshot that reached the journal last, so a burst +// of triggers writes one line. +type snapshotLimiter struct { + mu sync.Mutex + last networkSnapshot + written time.Time + seen bool +} + +// allow reports whether one snapshot reaches the journal, and records the +// snapshots that do. +func (s *snapshotLimiter) allow(trigger string, snapshot networkSnapshot, now time.Time) bool { + s.mu.Lock() + defer s.mu.Unlock() + if !s.permits(trigger, snapshot, now) { + return false + } + s.last, s.written, s.seen = snapshot, now, true + return true +} + +// permits decides one snapshot. The first snapshot of a run tells a reader +// where the run started. A snapshot equal to the one before it carries +// nothing. A new default route and a new resolver set explain an outage, so +// they pass at once. Every other snapshot waits out the interval, because a +// flapping interface writes some kilobytes per flap. +func (s *snapshotLimiter) permits(trigger string, snapshot networkSnapshot, now time.Time) bool { + if trigger == snapshotTriggerStart || !s.seen { + return true + } + if sameNetworkSnapshot(s.last, snapshot) { + return false + } + if routeOrResolversChanged(s.last, snapshot) { + return true + } + return now.Sub(s.written) >= snapshotMinInterval +} + +// sameNetworkSnapshot compares two snapshots field by field. The trigger is no +// part of a snapshot, so two triggers of one network give one line. +func sameNetworkSnapshot(before, after networkSnapshot) bool { + return reflect.DeepEqual(before, after) +} + +// routeOrResolversChanged names the changes that a reader of an outage needs +// at once. +func routeOrResolversChanged(before, after networkSnapshot) bool { + return before.DefaultRouteV4 != after.DefaultRouteV4 || + before.DefaultRouteV6 != after.DefaultRouteV6 || + !slices.Equal(before.Resolvers, after.Resolvers) +} + +// snapshotInputsFor reads every source of one snapshot. It takes no writer +// lock, so any path of the daemon can log a snapshot. +func (p *prog) snapshotInputsFor(state *netmon.State) snapshotInputs { + routeV4, routeV6 := defaultRoutesFn() + inputs := snapshotInputs{ + State: state, + RouteV4: routeV4, + RouteV6: routeV6, + Resolvers: ctrld.OsResolverNameservers(), + SourceIPv4: addressString(ctrld.GetDefaultLocalIPv4()), + SourceIPv6: addressString(ctrld.GetDefaultLocalIPv6()), + InterceptTarget: p.interceptTargetSnapshot(), + BypassActive: p.recoveryBypass.Load(), + RecoveryRunning: p.recoveryRunning.Load(), + PFStabilizing: p.pfStabilizing.Load(), + Meta: interfaceMetaFor, + } + if prefix, ok := p.dns64PrefixSnapshot(); ok { + inputs.NAT64Prefix = prefix.String() + } + if state != nil { + inputs.CLATPresent = dns64StateHasCLAT(state) + } + return inputs +} + +// freshNetworkState reads the interfaces of the host now. netmon fills the +// route and the families of a callback state, so a read of this package has to +// fill them itself. +func freshNetworkState() *netmon.State { + state, err := readNetworkSourceStateFn() + if err != nil || state == nil { + return nil + } + fresh := *state + if route, err := netmon.DefaultRouteInterface(); err == nil { + fresh.DefaultRouteInterface = route + } + fresh.HaveV4, fresh.HaveV6 = reachableAddressFamilies(&fresh) + return &fresh +} + +// reachableAddressFamilies reports which families the host can reach. It +// grades each address with the rule of the network monitor, so the first +// header of a run reports the same families as the first callback. +func reachableAddressFamilies(state *netmon.State) (haveV4, haveV6 bool) { + for name, iface := range state.Interface { + if iface.Interface == nil || !iface.IsUp() { + continue + } + for _, prefix := range state.InterfaceIPs[name] { + address := prefix.Addr().Unmap() + haveV4 = haveV4 || usableV4(address) + haveV6 = haveV6 || usableV6(address) + } + } + return haveV4, haveV6 +} + +// addressString names one source address. An address that ctrld does not hold +// leaves an empty field, because net.IP prints an empty value as the word for +// a nil address. +func addressString(ip net.IP) string { + if len(ip) == 0 { + return "" + } + return ip.String() +} diff --git a/cmd/cli/network_snapshot_prog_test.go b/cmd/cli/network_snapshot_prog_test.go new file mode 100644 index 0000000..7d3ab15 --- /dev/null +++ b/cmd/cli/network_snapshot_prog_test.go @@ -0,0 +1,348 @@ +package cli + +import ( + "net" + "net/netip" + "reflect" + "slices" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// stubSnapshotPlatformMeta makes the platform names of each interface a +// fixture, so no test depends on the ports of the host. +func stubSnapshotPlatformMeta(t *testing.T, meta map[string]interfaceMeta) { + t.Helper() + orig := platformInterfaceMetaFn + t.Cleanup(func() { platformInterfaceMetaFn = orig }) + platformInterfaceMetaFn = func(name string) (class, hardwarePort, service string) { + info := meta[name] + return info.Class, info.HardwarePort, info.Service + } +} + +// stubSnapshotVirtualSet names the virtual adapters of the fixture host. +func stubSnapshotVirtualSet(t *testing.T, names ...string) { + t.Helper() + orig := virtualInterfaceSetFn + t.Cleanup(func() { virtualInterfaceSetFn = orig }) + virtual := make(map[string]struct{}, len(names)) + for _, name := range names { + virtual[name] = struct{}{} + } + virtualInterfaceSetFn = func() map[string]struct{} { return virtual } +} + +// stubSnapshotGateways replaces the route table read, because the gateways of +// the host are not a fixture. +func stubSnapshotGateways(t *testing.T, v4, v6 string) { + t.Helper() + stubSnapshotRoutes(t, defaultRoute{Gateway: v4}, defaultRoute{Gateway: v6}) +} + +// stubSnapshotRoutes replaces the route table read with one default route per +// family. +func stubSnapshotRoutes(t *testing.T, v4, v6 defaultRoute) { + t.Helper() + orig := defaultRoutesFn + t.Cleanup(func() { defaultRoutesFn = orig }) + defaultRoutesFn = func() (defaultRoute, defaultRoute) { return v4, v6 } +} + +// stubSnapshotDNS64 reports a dual-stack host, so the NAT64 check starts no +// discovery and touches no name server. +func stubSnapshotDNS64(t *testing.T) { + t.Helper() + orig := dns64NetworkClassFn + t.Cleanup(func() { dns64NetworkClassFn = orig }) + dns64NetworkClassFn = func() (hasUsableIPv4, hasCLAT bool, err error) { return true, false, nil } +} + +// stubSnapshotSources sets the local source addresses and puts the values of +// the host back when the test ends. +func stubSnapshotSources(t *testing.T, v4, v6 string) { + t.Helper() + origV4, origV6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + t.Cleanup(func() { + ctrld.SetDefaultLocalIPv4(origV4) + ctrld.SetDefaultLocalIPv6(origV6) + }) + ctrld.SetDefaultLocalIPv4(net.ParseIP(v4)) + ctrld.SetDefaultLocalIPv6(net.ParseIP(v6)) +} + +// tetheredSnapshotProg returns a prog on a phone hotspot over the cable. +func tetheredSnapshotProg(t *testing.T) *prog { + t.Helper() + stubSnapshotDNS64(t) + stubSnapshotGateways(t, "172.20.10.1", "") + stubSnapshotPlatformMeta(t, map[string]interfaceMeta{ + "en7": {HardwarePort: hardwarePortIPhoneUSB, Service: "iPhone USB"}, + }) + stubSnapshotVirtualSet(t) + stubSnapshotSources(t, "172.20.10.5", "") + + p := &prog{} + p.lastNetworkState.Store(&netmon.State{ + DefaultRouteInterface: "en7", + HaveV4: true, + Interface: map[string]netmon.Interface{ + "en7": {Interface: &net.Interface{Name: "en7", Flags: net.FlagUp}}, + "lo0": {Interface: &net.Interface{Name: "lo0", Flags: net.FlagUp}}, + }, + InterfaceIPs: map[string][]netip.Prefix{ + "en7": {netip.MustParsePrefix("172.20.10.5/28")}, + "lo0": {netip.MustParsePrefix("127.0.0.1/8")}, + }, + }) + p.recoveryBypass.Store(true) + p.pfStabilizing.Store(true) + p.interceptDNSTargetSetValue = "127.0.0.1" + p.publishInterceptTarget("127.0.0.1") + return p +} + +func Test_networkSnapshotFromTheStoredState(t *testing.T) { + p := tetheredSnapshotProg(t) + + snapshot := p.networkSnapshot() + + checks := []struct { + field string + got, want any + }{ + {"default_route_v4", snapshot.DefaultRouteV4, "en7"}, + {"default_route_v6", snapshot.DefaultRouteV6, ""}, + {"gateway_v4", snapshot.GatewayV4, "172.20.10.1"}, + {"have_v4", snapshot.HaveV4, true}, + {"source_ipv4", snapshot.SourceIPv4, "172.20.10.5"}, + {"source_ipv6", snapshot.SourceIPv6, ""}, + {"intercept_target", snapshot.InterceptTarget, "127.0.0.1"}, + {"dns_less_target", snapshot.DNSLessTarget, true}, + {"bypass_active", snapshot.BypassActive, true}, + {"recovery_running", snapshot.RecoveryRunning, false}, + {"pf_stabilizing", snapshot.PFStabilizing, true}, + {"link_type", snapshot.LinkType, "usb_tether"}, + {"tethered", snapshot.Tethered, true}, + {"nat64_prefix", snapshot.NAT64Prefix, ""}, + {"clat_present", snapshot.CLATPresent, false}, + } + for _, c := range checks { + if c.got != c.want { + t.Errorf("%s = %v, want %v", c.field, c.got, c.want) + } + } + + want := []snapshotInterface{ + {Name: "en7", Class: "hardware", Up: true, IPs: []string{"172.20.10.5/28"}, HardwarePort: hardwarePortIPhoneUSB, Service: "iPhone USB"}, + {Name: "lo0", Class: "loopback", Up: true, IPs: []string{"127.0.0.1/8"}}, + } + if !reflect.DeepEqual(snapshot.Interfaces, want) { + t.Errorf("interfaces = %+v, want %+v", snapshot.Interfaces, want) + } +} + +// Test_networkSnapshotReadsTheNetworkWithoutAStoredState covers the first +// header of a run. No network callback reached ctrld yet, so the snapshot must +// read the interfaces itself and name the families it finds. +func Test_networkSnapshotReadsTheNetworkWithoutAStoredState(t *testing.T) { + stubSnapshotDNS64(t) + stubSnapshotGateways(t, "192.0.2.1", "") + stubSnapshotPlatformMeta(t, nil) + stubSnapshotSources(t, "", "") + stubLogHeaderNetworkRead(t, "en9") + p := &prog{} + + snapshot := p.networkSnapshot() + + if len(snapshot.Interfaces) != 1 || snapshot.Interfaces[0].Name != "en9" || !snapshot.Interfaces[0].Up { + t.Fatalf("interfaces = %+v, want one up en9", snapshot.Interfaces) + } + if !snapshot.HaveV4 { + t.Error("have_v4 = false, want true for a global unicast v4 address") + } + if snapshot.HaveV6 { + t.Error("have_v6 = true, want false without a v6 address") + } +} + +// Test_reachableAddressFamiliesGradesTheV6AddressLikeNetmon covers a host on a +// unique local address. The network monitor counts such an address as usable, +// so a fresh read must count it too. +func Test_reachableAddressFamiliesGradesTheV6AddressLikeNetmon(t *testing.T) { + for _, tt := range []struct { + name string + ips []string + wantV6 bool + }{ + {name: "unique_local", ips: []string{"fd00::2/64"}, wantV6: true}, + {name: "tailscale_range", ips: []string{"fd7a:115c:a1e0::2/128"}}, + {name: "global", ips: []string{"2001:db8::2/64"}, wantV6: true}, + {name: "link_local", ips: []string{"fe80::2/64"}}, + } { + t.Run(tt.name, func(t *testing.T) { + state := snapshotTestState("en0", snapshotTestInterface{"en0", true, tt.ips}) + + _, haveV6 := reachableAddressFamilies(state) + + if haveV6 != tt.wantV6 { + t.Errorf("have_v6 = %v, want %v for %v", haveV6, tt.wantV6, tt.ips) + } + }) + } +} + +func Test_networkSnapshotLogsOneJournalLine(t *testing.T) { + logs := captureDebugMainLog(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("start") + + events := jsonLogEvents(t, logs, networkSnapshotMessage) + if len(events) != 1 { + t.Fatalf("%q lines = %d, want 1", networkSnapshotMessage, len(events)) + } + wantField(t, events[0], "trigger", "start") + wantField(t, events[0], journalField, true) + wantField(t, events[0], "level", "info") + network, ok := events[0]["network"].(map[string]any) + if !ok { + t.Fatalf("network = %v, want an object", events[0]["network"]) + } + wantField(t, network, "default_route_v4", "en7") + wantField(t, network, "tethered", true) + wantField(t, network, "link_type", "usb_tether") +} + +// snapshotTestClock hands out the time that a test sets, because the snapshot +// limit counts seconds between two lines. +type snapshotTestClock struct { + now time.Time +} + +// stubSnapshotClock puts the clock of the snapshot limit under the control of +// one test. +func stubSnapshotClock(t *testing.T) *snapshotTestClock { + t.Helper() + clock := &snapshotTestClock{now: time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC)} + orig := snapshotNowFn + t.Cleanup(func() { snapshotNowFn = orig }) + snapshotNowFn = func() time.Time { return clock.now } + return clock +} + +func (c *snapshotTestClock) advance(d time.Duration) { c.now = c.now.Add(d) } + +// snapshotTriggerLines names the trigger of every snapshot line, in order. +func snapshotTriggerLines(t *testing.T, logs *syncBuffer) []string { + t.Helper() + var triggers []string + for _, event := range jsonLogEvents(t, logs, networkSnapshotMessage) { + trigger, _ := event["trigger"].(string) + triggers = append(triggers, trigger) + } + return triggers +} + +// Test_logNetworkSnapshotSkipsAnEqualSnapshot covers a recovery on a network +// that did not change. The begin and the end of that recovery render the same +// object, so the second line carries nothing. +func Test_logNetworkSnapshotSkipsAnEqualSnapshot(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("recovery_begin") + clock.advance(5 * time.Minute) + p.logNetworkSnapshot("recovery_end") + + if got := snapshotTriggerLines(t, logs); !slices.Equal(got, []string{"recovery_begin"}) { + t.Errorf("snapshot triggers = %v, want the first line alone", got) + } +} + +// Test_logNetworkSnapshotCapsTheRate covers a flapping interface. Two +// snapshots that differ in no route and in no resolver give one line per +// minute. +func Test_logNetworkSnapshotCapsTheRate(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("transition") + clock.advance(30 * time.Second) + p.recoveryRunning.Store(true) + p.logNetworkSnapshot("transition") + clock.advance(30 * time.Second) + p.logNetworkSnapshot("transition") + + if got := snapshotTriggerLines(t, logs); len(got) != 2 { + t.Errorf("snapshot lines = %d, want 2: %v", len(got), got) + } +} + +// Test_logNetworkSnapshotWritesARouteChangeAtOnce covers the change that +// explains an outage. The cap must not hold it back. +func Test_logNetworkSnapshotWritesARouteChangeAtOnce(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("transition") + clock.advance(time.Second) + stubSnapshotRoutes(t, defaultRoute{Gateway: "192.168.1.1", Interface: "en0"}, defaultRoute{}) + p.logNetworkSnapshot("transition") + clock.advance(time.Second) + p.recoveryRunning.Store(true) + p.logNetworkSnapshot("recovery_begin") + + if got := snapshotTriggerLines(t, logs); !slices.Equal(got, []string{"transition", "transition"}) { + t.Errorf("snapshot triggers = %v, want the two transitions", got) + } +} + +// Test_logNetworkSnapshotAlwaysWritesTheStartLine covers the first line of a +// run. A reader needs the network of the start, whatever came before it. +func Test_logNetworkSnapshotAlwaysWritesTheStartLine(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("transition") + clock.advance(time.Second) + p.logNetworkSnapshot(snapshotTriggerStart) + + if got := snapshotTriggerLines(t, logs); !slices.Equal(got, []string{"transition", "start"}) { + t.Errorf("snapshot triggers = %v, want the transition and the start", got) + } +} + +func Test_interfaceMetaFor(t *testing.T) { + stubSnapshotVirtualSet(t, "bridge100") + stubSnapshotPlatformMeta(t, map[string]interfaceMeta{ + "lo0": {Class: "hardware"}, + "en0": {HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + "eth0": {Class: "hardware", HardwarePort: "Intel I219-V"}, + }) + + for _, tt := range []struct { + name string + want interfaceMeta + }{ + {"lo0", interfaceMeta{Class: "loopback", LinkType: "unknown"}}, + {"awdl0", interfaceMeta{Class: "airdrop", LinkType: "unknown"}}, + {"utun3", interfaceMeta{Class: "tunnel", LinkType: "tunnel"}}, + {"en0", interfaceMeta{Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi", LinkType: "wifi"}}, + {"bridge100", interfaceMeta{Class: "virtual", LinkType: "unknown"}}, + {"eth0", interfaceMeta{Class: "hardware", HardwarePort: "Intel I219-V", LinkType: "unknown"}}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := interfaceMetaFor(tt.name); got != tt.want { + t.Errorf("interfaceMetaFor(%q) = %+v, want %+v", tt.name, got, tt.want) + } + }) + } +} diff --git a/cmd/cli/network_snapshot_test.go b/cmd/cli/network_snapshot_test.go new file mode 100644 index 0000000..e582e91 --- /dev/null +++ b/cmd/cli/network_snapshot_test.go @@ -0,0 +1,557 @@ +package cli + +import ( + "encoding/json" + "fmt" + "net" + "net/netip" + "slices" + "strings" + "testing" + + "github.com/rs/zerolog" + "tailscale.com/net/netmon" +) + +// snapshotTestInterface names one interface of a test network state. +type snapshotTestInterface struct { + name string + up bool + ips []string +} + +// snapshotTestState builds a netmon state with several interfaces, because the +// shared helper of the transition tests holds one interface only. +func snapshotTestState(route string, interfaces ...snapshotTestInterface) *netmon.State { + state := &netmon.State{ + DefaultRouteInterface: route, + Interface: map[string]netmon.Interface{}, + InterfaceIPs: map[string][]netip.Prefix{}, + } + for _, iface := range interfaces { + flags := net.Flags(0) + if iface.up { + flags = net.FlagUp + } + state.Interface[iface.name] = netmon.Interface{Interface: &net.Interface{Name: iface.name, Flags: flags}} + for _, ip := range iface.ips { + state.InterfaceIPs[iface.name] = append(state.InterfaceIPs[iface.name], netip.MustParsePrefix(ip)) + } + } + return state +} + +func snapshotTestMeta(meta map[string]interfaceMeta) interfaceMetaFunc { + return func(name string) interfaceMeta { return meta[name] } +} + +func Test_tetheredNetwork(t *testing.T) { + for _, tt := range []struct { + name string + gatewayV4 string + hardwarePort string + hostIPs []string + want bool + }{ + {name: "hotspot_gateway", gatewayV4: "172.20.10.1", hardwarePort: "Wi-Fi", want: true}, + {name: "hotspot_range_end", gatewayV4: "172.20.10.15", hardwarePort: "Wi-Fi", want: true}, + {name: "above_hotspot_range", gatewayV4: "172.20.10.16", hardwarePort: "Wi-Fi"}, + {name: "below_hotspot_range", gatewayV4: "172.20.9.255", hardwarePort: "Wi-Fi"}, + {name: "home_gateway", gatewayV4: "192.168.1.1", hardwarePort: "Wi-Fi"}, + {name: "iphone_usb_port", gatewayV4: "192.168.1.1", hardwarePort: "iPhone USB", want: true}, + {name: "no_gateway_no_port"}, + {name: "unparsable_gateway", gatewayV4: "not-an-address"}, + {name: "v6_gateway", gatewayV4: "fe80::1", hardwarePort: "Ethernet"}, + {name: "hotspot_host_address_without_a_gateway", hostIPs: []string{"172.20.10.4/28"}, want: true}, + {name: "hotspot_host_address_of_a_second_address", hostIPs: []string{"fe80::1/64", "172.20.10.4/28"}, want: true}, + {name: "home_host_address_without_a_gateway", hostIPs: []string{"192.168.1.20/24"}}, + } { + t.Run(tt.name, func(t *testing.T) { + hostIPs := make([]netip.Prefix, 0, len(tt.hostIPs)) + for _, ip := range tt.hostIPs { + hostIPs = append(hostIPs, netip.MustParsePrefix(ip)) + } + if got := tetheredNetwork(tt.gatewayV4, tt.hardwarePort, hostIPs); got != tt.want { + t.Fatalf("tetheredNetwork(%q, %q, %v) = %v, want %v", tt.gatewayV4, tt.hardwarePort, tt.hostIPs, got, tt.want) + } + }) + } +} + +// Test_buildNetworkSnapshot_tetheredWithoutAGateway covers Linux and Windows. +// Both read no route table in this release, so the address of the host is the +// only sign of a phone hotspot. +func Test_buildNetworkSnapshot_tetheredWithoutAGateway(t *testing.T) { + state := snapshotTestState("wlan0", snapshotTestInterface{"wlan0", true, []string{"172.20.10.4/28"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + if !snapshot.Tethered { + t.Error("tethered = false, want true for a host address of the hotspot range") + } +} + +// Test_usableV6 pins the grade of a v6 address to the rule of the network +// monitor. A fresh read and a monitor callback must report the same have_v6. +func Test_usableV6(t *testing.T) { + for _, tt := range []struct { + address string + want bool + }{ + {"2001:db8::1", true}, + {"2606:4700:4700::1111", true}, + {"fd00::1", true}, + {"fd7a:115c:a1e0::1", false}, + {"fe80::1", false}, + {"::1", false}, + {"192.0.2.10", false}, + } { + t.Run(tt.address, func(t *testing.T) { + if got := usableV6(netip.MustParseAddr(tt.address).Unmap()); got != tt.want { + t.Errorf("usableV6(%q) = %v, want %v", tt.address, got, tt.want) + } + }) + } +} + +func Test_linkTypeFor(t *testing.T) { + for _, tt := range []struct { + name string + hardwarePort string + class string + want string + }{ + {"wifi_port", "Wi-Fi", "hardware", "wifi"}, + {"ethernet_port", "Ethernet", "hardware", "ethernet"}, + {"thunderbolt_port", "Thunderbolt Ethernet Slot 1", "hardware", "ethernet"}, + {"usb_lan_port", "USB 10/100/1000 LAN", "hardware", "ethernet"}, + {"thunderbolt_bridge", "Thunderbolt Bridge", "hardware", "ethernet"}, + {"iphone_usb_port", "iPhone USB", "hardware", "usb_tether"}, + {"tunnel_class", "", "tunnel", "tunnel"}, + {"no_port_no_class", "", "", "unknown"}, + {"unknown_port", "Bluetooth PAN", "hardware", "unknown"}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := linkTypeFor(tt.hardwarePort, tt.class); got != tt.want { + t.Fatalf("linkTypeFor(%q, %q) = %q, want %q", tt.hardwarePort, tt.class, got, tt.want) + } + }) + } +} + +func Test_buildNetworkSnapshot_tetheredWiFiHotspot(t *testing.T) { + state := snapshotTestState("en0", snapshotTestInterface{"en0", true, []string{"172.20.10.2/28"}}) + state.HaveV4 = true + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "172.20.10.1", Interface: "en0"}, + Meta: snapshotTestMeta(map[string]interfaceMeta{"en0": {Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi"}}), + }) + + if !snapshot.Tethered { + t.Error("tethered = false, want true for the hotspot gateway") + } + if snapshot.LinkType != "wifi" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "wifi") + } + if snapshot.DefaultRouteV4 != "en0" || snapshot.DefaultRouteV6 != "" { + t.Errorf("default routes = %q/%q, want en0 and empty", snapshot.DefaultRouteV4, snapshot.DefaultRouteV6) + } + if !snapshot.HaveV4 || snapshot.HaveV6 { + t.Errorf("have_v4/have_v6 = %v/%v, want true/false", snapshot.HaveV4, snapshot.HaveV6) + } + want := []snapshotInterface{{Name: "en0", Class: "hardware", Up: true, IPs: []string{"172.20.10.2/28"}, HardwarePort: "Wi-Fi", Service: "Wi-Fi"}} + if !slices.EqualFunc(snapshot.Interfaces, want, sameSnapshotInterface) { + t.Errorf("interfaces = %+v, want %+v", snapshot.Interfaces, want) + } +} + +func Test_buildNetworkSnapshot_usbTether(t *testing.T) { + state := snapshotTestState("en7", snapshotTestInterface{"en7", true, []string{"172.20.10.3/28"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{"en7": {Class: "hardware", HardwarePort: "iPhone USB", Service: "iPhone USB"}}), + }) + + if snapshot.LinkType != "usb_tether" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "usb_tether") + } + if !snapshot.Tethered { + t.Error("tethered = false, want true for the iPhone USB port") + } +} + +func Test_buildNetworkSnapshot_tunnelDefaultRoute(t *testing.T) { + state := snapshotTestState("utun3", snapshotTestInterface{"utun3", true, []string{"100.64.0.2/32"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{"utun3": {Class: "tunnel"}}), + }) + + if snapshot.LinkType != "tunnel" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "tunnel") + } + if snapshot.Tethered { + t.Error("tethered = true, want false without a hotspot gateway") + } +} + +// Test_buildNetworkSnapshot_defaultRouteV4WithoutARouteTable covers the +// platforms that read no route table. The v4 route falls back to the interface +// of the network monitor, and that interface must hold a usable v4 address. +func Test_buildNetworkSnapshot_defaultRouteV4WithoutARouteTable(t *testing.T) { + for _, tt := range []struct { + name string + ips []string + wantV4 string + otherIPs []string + }{ + {name: "dual_stack", ips: []string{"192.0.2.10/24", "2001:db8::10/64"}, wantV4: "en0"}, + {name: "v4_only", ips: []string{"192.0.2.10/24", "fe80::1/64"}, wantV4: "en0"}, + {name: "v6_only", ips: []string{"2001:db8::10/64"}}, + {name: "link_local_only", ips: []string{"169.254.1.2/16", "fe80::1/64"}}, + {name: "loopback_only", ips: []string{"127.0.0.1/8"}}, + {name: "address_on_another_interface", ips: nil, otherIPs: []string{"192.0.2.20/24", "2001:db8::20/64"}}, + } { + t.Run(tt.name, func(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, tt.ips}, + snapshotTestInterface{"en1", true, tt.otherIPs}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + if snapshot.DefaultRouteV4 != tt.wantV4 { + t.Errorf("default_route_v4 = %q, want %q", snapshot.DefaultRouteV4, tt.wantV4) + } + if snapshot.DefaultRouteV6 != "" { + t.Errorf("default_route_v6 = %q, want an empty value without a route table", snapshot.DefaultRouteV6) + } + }) + } +} + +// Test_buildNetworkSnapshot_defaultRoutesComeFromTheRouteTable covers a host +// whose v6 default route leaves over a tunnel. The v6 route must name that +// tunnel, not the interface of the v4 route. +func Test_buildNetworkSnapshot_defaultRoutesComeFromTheRouteTable(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24", "2001:db8::10/64"}}, + snapshotTestInterface{"utun4", true, []string{"2001:db8:1::2/64"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "192.0.2.1", Interface: "en0"}, + RouteV6: defaultRoute{Gateway: "fe80::1%utun4", Interface: "utun4"}, + }) + + if snapshot.DefaultRouteV4 != "en0" || snapshot.DefaultRouteV6 != "utun4" { + t.Errorf("default routes = %q/%q, want en0 and utun4", snapshot.DefaultRouteV4, snapshot.DefaultRouteV6) + } + if snapshot.GatewayV4 != "192.0.2.1" || snapshot.GatewayV6 != "fe80::1%utun4" { + t.Errorf("gateways = %q/%q, want the gateways of the route table", snapshot.GatewayV4, snapshot.GatewayV6) + } +} + +func Test_buildNetworkSnapshot_sortsInterfacesAndTakesMeta(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en1", false, []string{"192.0.2.11/24"}}, + snapshotTestInterface{"awdl0", true, []string{"fe80::2/64"}}, + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{ + "en0": {Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + "awdl0": {Class: "airdrop"}, + "en1": {Class: "hardware", HardwarePort: "Ethernet", Service: "Ethernet"}, + }), + }) + + want := []snapshotInterface{ + {Name: "awdl0", Class: "airdrop", Up: true, IPs: []string{"fe80::2/64"}}, + {Name: "en0", Class: "hardware", Up: true, IPs: []string{"192.0.2.10/24"}, HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + {Name: "en1", Class: "hardware", Up: false, IPs: []string{"192.0.2.11/24"}, HardwarePort: "Ethernet", Service: "Ethernet"}, + } + if !slices.EqualFunc(snapshot.Interfaces, want, sameSnapshotInterface) { + t.Fatalf("interfaces = %+v, want %+v", snapshot.Interfaces, want) + } +} + +// Test_buildNetworkSnapshot_capsTheInterfaceList covers a container host. Such +// a host holds hundreds of interfaces, and a snapshot of all of them fills the +// journal. +func Test_buildNetworkSnapshot_capsTheInterfaceList(t *testing.T) { + interfaces := make([]snapshotTestInterface, 0, 40) + for i := range 36 { + interfaces = append(interfaces, snapshotTestInterface{fmt.Sprintf("veth%02d", i), true, nil}) + } + for i := range 4 { + interfaces = append(interfaces, snapshotTestInterface{fmt.Sprintf("down%d", i), false, nil}) + } + state := snapshotTestState("veth00", interfaces...) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + if len(snapshot.Interfaces) != snapshotInterfaceLimit { + t.Fatalf("interfaces = %d entries, want %d", len(snapshot.Interfaces), snapshotInterfaceLimit) + } + if snapshot.InterfacesOmitted != 8 { + t.Errorf("interfaces_omitted = %d, want 8", snapshot.InterfacesOmitted) + } + if first, last := snapshot.Interfaces[0].Name, snapshot.Interfaces[snapshotInterfaceLimit-1].Name; first != "veth00" || last != "veth31" { + t.Errorf("interfaces run from %q to %q, want veth00 to veth31", first, last) + } +} + +// Test_buildNetworkSnapshot_keepsUpOrAddressedInterfaces covers the rule that +// selects the interfaces of the list. +func Test_buildNetworkSnapshot_keepsUpOrAddressedInterfaces(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}, + snapshotTestInterface{"en1", true, nil}, + snapshotTestInterface{"en2", false, []string{"192.0.2.20/24"}}, + snapshotTestInterface{"en3", false, nil}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + var names []string + for _, iface := range snapshot.Interfaces { + names = append(names, iface.Name) + } + if !slices.Equal(names, []string{"en0", "en1", "en2"}) { + t.Errorf("interfaces = %v, want en0, en1, and en2", names) + } + if snapshot.InterfacesOmitted != 1 { + t.Errorf("interfaces_omitted = %d, want 1", snapshot.InterfacesOmitted) + } +} + +func Test_buildNetworkSnapshot_classesByNameWithoutMeta(t *testing.T) { + state := snapshotTestState("utun3", + snapshotTestInterface{"utun3", true, []string{"100.64.0.2/32"}}, + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + classes := map[string]string{} + for _, iface := range snapshot.Interfaces { + classes[iface.Name] = iface.Class + } + if classes["utun3"] != "tunnel" || classes["en0"] != "hardware" { + t.Fatalf("classes = %v, want utun3 tunnel and en0 hardware", classes) + } + if snapshot.LinkType != "tunnel" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "tunnel") + } +} + +func Test_buildNetworkSnapshot_fillsTheEmptyClassOfTheMeta(t *testing.T) { + state := snapshotTestState("utun3", + snapshotTestInterface{"utun3", true, []string{"100.64.0.2/32"}}, + snapshotTestInterface{"awdl0", true, []string{"fe80::2/64"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{"utun3": {HardwarePort: ""}}), + }) + + classes := map[string]string{} + for _, iface := range snapshot.Interfaces { + classes[iface.Name] = iface.Class + } + if classes["utun3"] != "tunnel" || classes["awdl0"] != "airdrop" { + t.Fatalf("classes = %v, want utun3 tunnel and awdl0 airdrop", classes) + } + if snapshot.LinkType != "tunnel" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "tunnel") + } +} + +func Test_buildNetworkSnapshot_nilStateKeepsFlags(t *testing.T) { + snapshot := buildNetworkSnapshot(snapshotInputs{ + RouteV4: defaultRoute{Gateway: "192.168.1.1"}, + RouteV6: defaultRoute{Gateway: "fe80::1"}, + Resolvers: []string{"192.168.1.1:53"}, + SourceIPv4: "192.168.1.20", + SourceIPv6: "2001:db8::20", + InterceptTarget: "127.0.0.1", + BypassActive: true, + RecoveryRunning: true, + PFStabilizing: true, + NAT64Prefix: "64:ff9b::/96", + CLATPresent: true, + }) + + if len(snapshot.Interfaces) != 0 { + t.Errorf("interfaces = %+v, want none", snapshot.Interfaces) + } + if snapshot.DefaultRouteV4 != "" || snapshot.DefaultRouteV6 != "" || snapshot.HaveV4 || snapshot.HaveV6 { + t.Errorf("state fields are set without a state: %+v", snapshot) + } + if snapshot.LinkType != "unknown" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "unknown") + } + if !snapshot.BypassActive || !snapshot.RecoveryRunning || !snapshot.PFStabilizing || !snapshot.CLATPresent { + t.Errorf("flags dropped: %+v", snapshot) + } + if !snapshot.DNSLessTarget { + t.Error("dns_less_target = false, want true with an intercept target") + } + if snapshot.GatewayV4 != "192.168.1.1" || snapshot.GatewayV6 != "fe80::1" { + t.Errorf("gateways = %q/%q, want the input values", snapshot.GatewayV4, snapshot.GatewayV6) + } + if snapshot.SourceIPv4 != "192.168.1.20" || snapshot.SourceIPv6 != "2001:db8::20" { + t.Errorf("sources = %q/%q, want the input values", snapshot.SourceIPv4, snapshot.SourceIPv6) + } + if snapshot.NAT64Prefix != "64:ff9b::/96" || !slices.Equal(snapshot.Resolvers, []string{"192.168.1.1:53"}) { + t.Errorf("prefix and resolvers dropped: %+v", snapshot) + } +} + +func Test_buildNetworkSnapshot_noInterceptTargetIsNotDNSLess(t *testing.T) { + snapshot := buildNetworkSnapshot(snapshotInputs{State: snapshotTestState("en0", snapshotTestInterface{"en0", true, nil})}) + if snapshot.DNSLessTarget { + t.Error("dns_less_target = true, want false without an intercept target") + } +} + +// snapshotDictKeys are the field names that spec R2.2 names for the snapshot, +// plus the count of the interfaces that the list leaves out. +var snapshotDictKeys = []string{ + "default_route_v4", "default_route_v6", "gateway_v4", "gateway_v6", + "have_v4", "have_v6", "interfaces", "interfaces_omitted", "resolvers", + "source_ipv4", "source_ipv6", "intercept_target", "bypass_active", + "recovery_running", "pf_stabilizing", "link_type", "tethered", + "clat_present", "nat64_prefix", "dns_less_target", +} + +var snapshotInterfaceDictKeys = []string{"class", "hardware_port", "ips", "name", "service", "up"} + +// renderSnapshotDict writes the dict under one field and parses the line, so +// the test reads the keys that a reader of the journal sees. +func renderSnapshotDict(t *testing.T, snapshot networkSnapshot) (string, map[string]any) { + t.Helper() + buf := &syncBuffer{} + logger := zerolog.New(buf) + logger.Log().Dict("network", snapshotDict(snapshot)).Msg("Network snapshot") + + line := strings.TrimSpace(buf.String()) + event := map[string]any{} + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("log line is not JSON: %q: %v", line, err) + } + network, ok := event["network"].(map[string]any) + if !ok { + t.Fatalf("line has no network object: %q", line) + } + return line, network +} + +func Test_snapshotDict_rendersEveryKeyOnce(t *testing.T) { + line, network := renderSnapshotDict(t, buildNetworkSnapshot(snapshotInputs{})) + + if got := sortedKeys(network); !slices.Equal(got, slices.Sorted(slices.Values(snapshotDictKeys))) { + t.Fatalf("keys = %v, want %v", got, snapshotDictKeys) + } + for _, key := range snapshotDictKeys { + if count := strings.Count(line, `"`+key+`":`); count != 1 { + t.Errorf("key %q appears %d times, want 1: %s", key, count, line) + } + } + if network["interfaces"] == nil || len(network["interfaces"].([]any)) != 0 { + t.Errorf("interfaces = %v, want an empty array", network["interfaces"]) + } + if network["resolvers"] == nil || len(network["resolvers"].([]any)) != 0 { + t.Errorf("resolvers = %v, want an empty array", network["resolvers"]) + } +} + +func Test_snapshotDict_rendersSortedInterfaces(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en1", true, nil}, + snapshotTestInterface{"awdl0", true, []string{"fe80::2/64"}}, + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "192.0.2.1", Interface: "en0"}, + Resolvers: []string{"192.0.2.1:53"}, + Meta: snapshotTestMeta(map[string]interfaceMeta{"en0": {Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi"}}), + }) + _, network := renderSnapshotDict(t, snapshot) + + interfaces, ok := network["interfaces"].([]any) + if !ok || len(interfaces) != 3 { + t.Fatalf("interfaces = %v, want three entries", network["interfaces"]) + } + var names []string + for _, entry := range interfaces { + object, ok := entry.(map[string]any) + if !ok { + t.Fatalf("interface entry is not an object: %v", entry) + } + if got := sortedKeys(object); !slices.Equal(got, snapshotInterfaceDictKeys) { + t.Fatalf("interface keys = %v, want %v", got, snapshotInterfaceDictKeys) + } + names = append(names, object["name"].(string)) + } + if !slices.Equal(names, []string{"awdl0", "en0", "en1"}) { + t.Errorf("interface order = %v, want awdl0, en0, en1", names) + } + if ips, ok := interfaces[2].(map[string]any)["ips"].([]any); !ok || len(ips) != 0 { + t.Errorf("ips of en1 = %v, want an empty array", interfaces[2].(map[string]any)["ips"]) + } + wantValues := map[string]any{"gateway_v4": "192.0.2.1", "link_type": "wifi", "default_route_v4": "en0", "tethered": false} + for key, want := range wantValues { + wantField(t, network, key, want) + } + if resolvers, ok := network["resolvers"].([]any); !ok || len(resolvers) != 1 || resolvers[0] != "192.0.2.1:53" { + t.Errorf("resolvers = %v, want one entry", network["resolvers"]) + } +} + +func Test_defaultRoutes_returnAddressesOrNothing(t *testing.T) { + v4, v6 := defaultRoutes() + // A host without a route of a family gives an empty value, so the test + // logs the pair and holds every value it gets to the format. + t.Logf("default routes: v4 %+v, v6 %+v", v4, v6) + + if v4.Gateway != "" { + address, err := netip.ParseAddr(v4.Gateway) + if err != nil || !address.Is4() { + t.Errorf("gateway_v4 = %q, want an IPv4 address or nothing: %v", v4.Gateway, err) + } + } + if v6.Gateway != "" { + address, err := netip.ParseAddr(v6.Gateway) + if err != nil || !address.Is6() { + t.Errorf("gateway_v6 = %q, want an IPv6 address or nothing: %v", v6.Gateway, err) + } + } + // A tunnel default has no next hop, so an interface without a gateway is a + // valid pair. A gateway without an interface is not. + if v4.Gateway != "" && v4.Interface == "" { + t.Errorf("gateway_v4 = %q without an interface, want the interface of the route", v4.Gateway) + } + if v6.Gateway != "" && v6.Interface == "" { + t.Errorf("gateway_v6 = %q without an interface, want the interface of the route", v6.Gateway) + } +} + +func sameSnapshotInterface(a, b snapshotInterface) bool { + return a.Name == b.Name && a.Class == b.Class && a.Up == b.Up && + a.HardwarePort == b.HardwarePort && a.Service == b.Service && + slices.Equal(a.IPs, b.IPs) +} + +func sortedKeys(object map[string]any) []string { + keys := make([]string, 0, len(object)) + for key := range object { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} diff --git a/cmd/cli/network_transition.go b/cmd/cli/network_transition.go index 5638e2c..711e592 100644 --- a/cmd/cli/network_transition.go +++ b/cmd/cli/network_transition.go @@ -153,6 +153,13 @@ type recoveryDiagnostic struct { started time.Time firstFailure sync.Once failed atomic.Bool + + // The flow fills these while it runs. The end event reports them, so one + // journal line tells what the recovery found and what it changed. + recoveredUpstream string + dhcpServers []string + interceptTargetAction string + bypassActive bool } func recoveryReasonName(reason RecoveryReason) string { @@ -191,10 +198,33 @@ func (d *recoveryDiagnostic) failure(err error) { }) } +// The outcomes that one recovery pass ends with. +const ( + recoveryOutcomeCompleted = "completed" + recoveryOutcomeCanceled = "canceled" + recoveryOutcomeSuperseded = "superseded" +) + +// journalRecoveredUpstream bounds the upstream name of the end event. An empty +// name means that no upstream recovered, so it stays empty. +func journalRecoveredUpstream(upstream string) string { + if upstream == "" { + return "" + } + return journalUpstreamName(upstream) +} + func (d *recoveryDiagnostic) end(outcome string) { - e := mainLog.Load().Debug() - if outcome == "canceled" || d.failed.Load() { + e := mainLog.Load().Info() + if outcome == recoveryOutcomeCanceled || d.failed.Load() { e = mainLog.Load().Warn() } - d.event(e).Str("outcome", outcome).Bool("had_failure", d.failed.Load()).Dur("duration_ms", time.Since(d.started)).Msg("Recovery end") + journal(d.event(e)).Str("outcome", outcome).Bool("had_failure", d.failed.Load()). + Dur("duration_ms", time.Since(d.started)). + Str("recovered_upstream", journalRecoveredUpstream(d.recoveredUpstream)). + Strs("dhcp_servers", d.dhcpServers). + Int("dhcp_server_count", len(d.dhcpServers)). + Str("intercept_target_action", d.interceptTargetAction). + Bool("bypass_active", d.bypassActive). + Msg("Recovery end") } diff --git a/cmd/cli/network_transition_test.go b/cmd/cli/network_transition_test.go index eb68d7a..716c619 100644 --- a/cmd/cli/network_transition_test.go +++ b/cmd/cli/network_transition_test.go @@ -44,6 +44,10 @@ func captureTransitionLogs(t *testing.T) *syncBuffer { func sourceTestGlobals(t *testing.T) { t.Helper() + // A transition reads the hardware ports and the virtual adapters, and no + // test may start the commands behind them. + stubHeaderSnapshotSources(t) + stubSnapshotVirtualSet(t) v4, v6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() valid, route := networkChangeValidInterfacesFn, networkChangeDefaultRouteIPFn reconcile, ignored := networkChangeReconcileFn, networkChangeIgnoredInterceptFn @@ -189,6 +193,7 @@ func TestNetworkChangeSourceMovedToUpInterface(t *testing.T) { func TestRecoveryTransitionDiagnostics(t *testing.T) { logs := captureTransitionLogs(t) + stubHeaderSnapshotSources(t) original, intercept := recoveryResetDNSFn, dnsIntercept dnsIntercept = false t.Cleanup(func() { recoveryResetDNSFn = original; dnsIntercept = intercept }) @@ -289,7 +294,7 @@ func TestTransitionWarningsSurviveDebugRotation(t *testing.T) { t.Fatalf("debug buffer did not rotate %q", msg) } } - if strings.Count(text, "Recovery end") != 1 || !strings.Contains(text, `"outcome":"completed"`) { + if strings.Count(text, "Recovery end") != 2 || !strings.Contains(text, `"outcome":"completed"`) { t.Fatal("recovery completion must survive rotation beside its retained failure") } if strings.Contains(text, "private.customer") || strings.Contains(text, "secret payload") { diff --git a/cmd/cli/pf_state.go b/cmd/cli/pf_state.go new file mode 100644 index 0000000..2b10cc6 --- /dev/null +++ b/cmd/cli/pf_state.go @@ -0,0 +1,82 @@ +package cli + +import ( + "bufio" + "io" + "maps" + "slices" + "strings" +) + +// parsePFAnchorNames returns the anchor names that the running ruleset refers to, +// unique and sorted. It takes both pfctl show commands because each one holds half +// of the list: "pfctl -sr" the scrub and filter anchors, "pfctl -sn" the +// translation anchors. +func parsePFAnchorNames(rules, nat io.Reader) []string { + unique := make(map[string]struct{}) + // A read that stopped early gives a shorter list, and a shorter list reads + // as an anchor that went away. + if !collectPFAnchorNames(rules, unique) || !collectPFAnchorNames(nat, unique) { + return nil + } + return slices.Sorted(maps.Keys(unique)) +} + +// collectPFAnchorNames reports whether it read the whole output. +func collectPFAnchorNames(r io.Reader, into map[string]struct{}) bool { + if r == nil { + return true + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + name := pfAnchorNameOf(scanner.Text()) + if name == "" { + continue + } + into[name] = struct{}{} + } + return scanner.Err() == nil +} + +// pfAnchorNameOf returns the anchor name of one pfctl rule line. The rule verb is +// anchor, nat-anchor, rdr-anchor, scrub-anchor or dummynet-anchor, and the quoted +// name follows it. Every other line, including the merged ALTQ warnings, gives "". +func pfAnchorNameOf(line string) string { + verb, rest, ok := strings.Cut(strings.TrimSpace(line), " ") + if !ok || !strings.HasSuffix(verb, "anchor") { + return "" + } + _, quoted, ok := strings.Cut(rest, `"`) + if !ok { + return "" + } + name, _, ok := strings.Cut(quoted, `"`) + if !ok { + return "" + } + return name +} + +// parsePFStatus reads "pfctl -si" output and reports whether pf runs, with the +// uptime text that pfctl prints beside it. +func parsePFStatus(r io.Reader) (enabled bool, since string) { + if r == nil { + return false, "" + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + status, ok := strings.CutPrefix(strings.TrimSpace(scanner.Text()), "Status:") + if !ok { + continue + } + // pfctl pads the status into a column and prints the debug level next to it. + status, _, _ = strings.Cut(status, "Debug:") + state, uptime, _ := strings.Cut(strings.TrimSpace(status), " for ") + if state != "Enabled" { + return false, "" + } + return true, strings.TrimSpace(uptime) + } + // A read that stopped early may have stopped before the status line. + return false, "" +} diff --git a/cmd/cli/pf_state_darwin.go b/cmd/cli/pf_state_darwin.go new file mode 100644 index 0000000..b89dcfe --- /dev/null +++ b/cmd/cli/pf_state_darwin.go @@ -0,0 +1,119 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "os/exec" + "time" +) + +// pfStateReadTimeout bounds each state read. These reads run on the network change +// path, so a pfctl that waits on a busy kernel lock must not hold it. +const pfStateReadTimeout = 3 * time.Second + +// pfStateReadMaxBytes bounds the output of one state read. A ruleset or an +// interface dump of this size is not the state ctrld reads, and these reads run +// on the network change path. +const pfStateReadMaxBytes = 1 << 20 + +// errPFStateOutputTooLarge marks output that passed the bound. The bytes kept +// are a prefix of the real output, so the caller must not parse them. +var errPFStateOutputTooLarge = errors.New("pf state output passed its bound") + +// boundedBuffer keeps at most limit bytes and remembers that it dropped the +// rest. +type boundedBuffer struct { + bytes.Buffer + limit int + exceeded bool +} + +func (b *boundedBuffer) Write(p []byte) (int, error) { + room := b.limit - b.Len() + if len(p) > room { + b.exceeded = true + p = p[:max(room, 0)] + } + if _, err := b.Buffer.Write(p); err != nil { + return 0, err + } + return len(p), nil +} + +// pfStateRunCommand runs one read-only state command. A var so tests replace it +// instead of shelling out. +var pfStateRunCommand = func(name string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), pfStateReadTimeout) + defer cancel() + out := &boundedBuffer{limit: pfStateReadMaxBytes} + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdout = out + cmd.Stderr = out + err := cmd.Run() + if out.exceeded { + return nil, errPFStateOutputTooLarge + } + return out.Bytes(), err +} + +// pfRuleDump carries pfctl rule output that the caller already read. A nil +// field means that the caller holds no bytes for that flag, so the read runs +// here. +type pfRuleDump struct { + rules []byte // "pfctl -sr" + nat []byte // "pfctl -sn" +} + +// pfAnchorNames returns the anchor names of the running ruleset. A failed read +// gives no names, because a partial list reads as an anchor that went away. +func (p *prog) pfAnchorNames(dump pfRuleDump) []string { + rules, ok := p.pfRuleBytes(dump.rules, "-sr", "list filter anchors") + if !ok { + return nil + } + nat, ok := p.pfRuleBytes(dump.nat, "-sn", "list translation anchors") + if !ok { + return nil + } + return parsePFAnchorNames(bytes.NewReader(rules), bytes.NewReader(nat)) +} + +// pfRuleBytes returns the rule output of one pfctl flag. It starts pfctl only +// when the caller holds no bytes, and a failed read feeds the backoff that +// keeps ctrld off an exhausted host. +func (p *prog) pfRuleBytes(held []byte, flag, operation string) ([]byte, bool) { + if held != nil { + return held, true + } + out, err := pfStateRunCommand("pfctl", flag) + if err != nil { + p.pfBackoffResourceExhaustion(err, out, operation) + return nil, false + } + return out, true +} + +// pfStatus reports whether pf runs and for how long. A failed read gives false +// and "". +func (p *prog) pfStatus() (bool, string) { + out, err := pfStateRunCommand("pfctl", "-si") + if err != nil { + p.pfBackoffResourceExhaustion(err, out, "read pf status") + return false, "" + } + return parsePFStatus(bytes.NewReader(out)) +} + +// tunnelOwner returns the network extension that owns a tunnel interface, or "" +// when the read fails or no extension owns it. +func tunnelOwner(iface string) string { + if iface == "" { + return "" + } + out, err := pfStateRunCommand("ifconfig", "-v", iface) + if err != nil { + return "" + } + return parseIfconfigAgent(bytes.NewReader(out)) +} diff --git a/cmd/cli/pf_state_test.go b/cmd/cli/pf_state_test.go new file mode 100644 index 0000000..b7c4723 --- /dev/null +++ b/cmd/cli/pf_state_test.go @@ -0,0 +1,137 @@ +package cli + +import ( + "slices" + "strings" + "testing" +) + +// pfStatusEnabled is "pfctl -si" output from a Mac with pf enabled. +const pfStatusEnabled = `Status: Enabled for 0 days 02:11:05 Debug: Urgent +` + +// Captured from macOS 26.6 with ctrld installed. "pfctl -sr" prints the scrub and +// filter anchors, "pfctl -sn" prints the translation anchors, and both carry the +// ALTQ warnings because every reader here uses CombinedOutput. +const pfShowRules = altqNoise + `scrub-anchor "com.apple/*" all fragment reassemble +anchor "com.apple/*" all +anchor "com.controld.ctrld" all +pass in quick on lo0 reply-to lo0 inet proto udp from any to 127.0.0.1 port = 5354 +` + +const pfShowNAT = altqNoise + `nat-anchor "com.apple/*" all +rdr-anchor "com.apple/*" all +rdr-anchor "com.controld.ctrld" all +` + +func Test_parsePFAnchorNames(t *testing.T) { + tests := []struct { + name string + rules string + nat string + want []string + }{ + { + name: "a real ruleset with ctrld installed", + rules: pfShowRules, + nat: pfShowNAT, + want: []string{"com.apple/*", "com.controld.ctrld"}, + }, + { + // The names arrive out of order, so the result proves the sort. + name: "a VPN adds its own anchors", + rules: altqNoise + `anchor "windscribe" all +anchor "com.apple/*" all +`, + nat: altqNoise + `dummynet-anchor "com.apple/*" all +rdr-anchor "windscribe" all +`, + want: []string{"com.apple/*", "windscribe"}, + }, + { + name: "anchors in the translation output only", + rules: altqNoise, + nat: altqNoise + "rdr-anchor \"com.controld.ctrld\" all\n", + want: []string{"com.controld.ctrld"}, + }, + { + name: "a ruleset with no anchor", + rules: altqNoise + "block drop in all\n", + nat: altqNoise + "(no rules)\n", + want: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := parsePFAnchorNames(strings.NewReader(tc.rules), strings.NewReader(tc.nat)) + if !slices.Equal(got, tc.want) { + t.Fatalf("parsePFAnchorNames() = %v, want %v", got, tc.want) + } + }) + } +} + +func Test_parsePFStatus(t *testing.T) { + tests := []struct { + name string + output string + wantEnabled bool + wantSince string + }{ + { + name: "enabled with an uptime", + output: altqNoise + `Status: Enabled for 3 days 02:11:05 Debug: Urgent + +Interface Stats for en0 IPv4 IPv6 + Bytes In 123456789 0 +`, + wantEnabled: true, + wantSince: "3 days 02:11:05", + }, + { + name: "disabled", + output: altqNoise + "Status: Disabled Debug: Urgent\n\n", + }, + { + name: "enabled without an uptime", + output: "Status: Enabled Debug: Urgent\n", + wantEnabled: true, + }, + { + name: "no status line", + output: altqNoise, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + enabled, since := parsePFStatus(strings.NewReader(tc.output)) + if enabled != tc.wantEnabled || since != tc.wantSince { + t.Fatalf("parsePFStatus() = %v, %q, want %v, %q", enabled, since, tc.wantEnabled, tc.wantSince) + } + }) + } +} + +// oversizedLine is longer than the largest line a scanner holds, so the read +// of it stops with an error. +var oversizedLine = strings.Repeat("a", 70<<10) + "\n" + +// Test_parsePFAnchorNamesRejectsAnOversizedLine covers a truncated read. The +// names collected before the error read as a ruleset that lost its anchors, +// and that is the state the watchdog acts on. +func Test_parsePFAnchorNamesRejectsAnOversizedLine(t *testing.T) { + if got := parsePFAnchorNames(strings.NewReader(pfShowRules+oversizedLine), strings.NewReader(pfShowNAT)); got != nil { + t.Fatalf("anchor names of a truncated rule list = %v, want none", got) + } + if got := parsePFAnchorNames(strings.NewReader(pfShowRules), strings.NewReader(pfShowNAT+oversizedLine)); got != nil { + t.Fatalf("anchor names of a truncated translation list = %v, want none", got) + } +} + +// Test_parsePFStatusRejectsAnOversizedLine covers a truncated status read. +func Test_parsePFStatusRejectsAnOversizedLine(t *testing.T) { + enabled, since := parsePFStatus(strings.NewReader(oversizedLine + pfStatusEnabled)) + if enabled || since != "" { + t.Fatalf("pf status of a truncated read = %v, %q, want false and an empty uptime", enabled, since) + } +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 487ca31..8cf4a2d 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -147,6 +147,17 @@ type prog struct { hasLocalDNS bool runningOnDomainController bool + // The network journal state. The value fields here hold a mutex, so no + // code may copy a prog. Every user of a prog holds a pointer to it. + networkNoise noiseCoalescer + wake wakeReporter + repeats repeatLogger //lint:ignore U1000 used on darwin + dnsConfig *dnsConfigPoller + health *queryHealth + snapshotWrites snapshotLimiter + lastNAT64Mu sync.Mutex + lastNAT64Logged string + selfUninstallMu sync.Mutex refusedQueryCount int canSelfUninstall atomic.Bool @@ -170,6 +181,10 @@ type prog struct { networkSourceState *netmon.State networkSourceEpoch *netmon.State networkSourceReadFailed bool + // netmon hands a minor callback the cached major snapshot in Old, so the + // diff keeps the state that the callback before it reported. + networkDeltaState *netmon.State + networkDeltaEpoch *netmon.State // recoveryDebounceTimer coalesces rapid NetworkChange recovery triggers // into a single handleRecovery call. Only handleRecovery is debounced — @@ -192,15 +207,18 @@ type prog struct { // interceptDNSTargetSetValue records the exact value set. Both empty when // no target is set. Guarded by interceptDNSTargetMu. // - //lint:ignore U1000 used in Darwin code. interceptDNSTargetMu sync.Mutex //lint:ignore U1000 used in Darwin code. - interceptDNSTargetService string - //lint:ignore U1000 used in Darwin code. + interceptDNSTargetService string interceptDNSTargetSetValue string //lint:ignore U1000 used in Darwin code. interceptDNSTargetLoaded bool + // interceptTargetMirror holds the same value as interceptDNSTargetSetValue + // for readers that must not wait: the mutex above covers networksetup + // calls that take seconds. + interceptTargetMirror atomic.Value // holds string + // DNS intercept mode state (platform-specific). // On Windows: *wfpState, on macOS: *pfState, nil on other platforms. dnsInterceptState any @@ -224,7 +242,11 @@ type prog struct { // lastTunnelIfaces tracks the tunnel set included in the last successfully loaded // pf anchor. Pending tunnel state is kept separately so failed PF work is retried // instead of being mistaken for an applied update. Protected by mu. - lastTunnelIfaces []string //lint:ignore U1000 used on darwin + lastTunnelIfaces []string //lint:ignore U1000 used on darwin + // lastLoggedTunnelIfaces is the tunnel set of the last logged event. The + // reconcile retries a set until it succeeds, so a diff of the applied set + // repeats one event at every retry. + lastLoggedTunnelIfaces []string //lint:ignore U1000 used on darwin pendingTunnelIfaces []string //lint:ignore U1000 used on darwin hasPendingTunnelIfaces bool //lint:ignore U1000 used on darwin @@ -496,12 +518,11 @@ func (p *prog) postRun() { if !p.skipInitialDNSReset() { p.resetDNS(false, false) } - ns, systemNameservers := initializeOsResolverWithSystemNameserversFn(false) + ns, systemNameservers := initializeOsResolverWithSystemNameserversFn(false, osResolverReasonStart) mainLog.Load().Debug().Msgf("initialized OS resolver with nameservers: %v", ns) p.setDNS(systemNameservers) p.csSetDnsDone <- struct{}{} close(p.csSetDnsDone) - p.logInterfacesState() } } @@ -770,13 +791,7 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { } } - // The new monitor starts with every upstream up, so the old one closes its - // open outages first. Without this the journal holds a down event that no - // up event ever follows. - if p.um != nil { - p.um.retire() - } - p.um = newUpstreamMonitor(p.cfg) + p.replaceUpstreamMonitor() if !reload { p.sema = &chanSemaphore{ready: make(chan struct{}, defaultSemaphoreCap)} @@ -806,12 +821,8 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { } if !reload { - go func() { - // Start network monitoring - if err := p.monitorNetworkChanges(); err != nil { - mainLog.Load().Error().Err(err).Msg("Failed to start network monitoring") - } - }() + p.newNetworkJournal() + go p.startNetworkMonitorAndJournal() } for listenerNum := range p.cfg.Listener { @@ -884,6 +895,96 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { wg.Wait() } +// dnsConfigChangedMessage names the journal event of the resolver table of the +// host. A log tool selects every resolver change of a run by this message. +const dnsConfigChangedMessage = "DNS configuration changed" + +// runSCUtilDNSFn reads the resolver table of the host. A test replaces it, +// because the configuration daemon of the host is not a fixture. +var runSCUtilDNSFn = runSCUtilDNS + +// replaceUpstreamMonitor puts a new upstream monitor in place of the one of +// the run before. The new monitor starts with every upstream up, so the old one +// closes its open outages first. Without this the journal holds a down event +// that no up event ever follows. +func (p *prog) replaceUpstreamMonitor() { + p.mu.Lock() + defer p.mu.Unlock() + if p.um != nil { + p.um.retire() + } + p.um = newUpstreamMonitor(p.cfg) +} + +// upstreamMonitorNow reads the monitor of this run under the lock that a +// replacement takes, because a reload swaps it while the journal loops run. +func (p *prog) upstreamMonitorNow() *upstreamMonitor { + p.mu.Lock() + defer p.mu.Unlock() + return p.um +} + +// newNetworkJournal creates the trackers that follow the host network. The +// query path reads them without a lock, so they exist before a listener starts. +func (p *prog) newNetworkJournal() { + p.health = newQueryHealth() + p.querySampler.onFailure = p.health.countFailure + if runtime.GOOS == "darwin" { + p.dnsConfig = newDNSConfigPoller(runSCUtilDNSFn) + } +} + +// startNetworkMonitorAndJournal starts the network monitor and then opens the +// journal. The trackers do not depend on the monitor, so a monitor that cannot +// start leaves them running. It passes on the wait function of the journal. +func (p *prog) startNetworkMonitorAndJournal() func() { + if err := p.monitorNetworkChanges(); err != nil { + mainLog.Load().Error().Err(err).Msg("Failed to start network monitoring") + } + return p.startNetworkJournal() +} + +// startNetworkJournal runs the trackers and opens the journal of this run with +// one snapshot. Support reads the grade of the query path and the resolver +// table of each moment after an incident. It returns a function that waits for +// both loops, so a caller that closed the stop channel knows when they ended. +func (p *prog) startNetworkJournal() func() { + healthDone := p.health.startLoop(p.stopCh, func() (int, bool) { + return p.upstreamMonitorNow().countDownExcept(isInternalDomainUpstream), p.recoveryBypass.Load() + }) + var configDone <-chan struct{} + if p.dnsConfig != nil { + configDone = p.dnsConfig.startLoop(p.stopCh, p.logDNSConfigChanges) + } + p.logNetworkSnapshot("start") + return func() { + <-healthDone + if configDone != nil { + <-configDone + } + } +} + +// logDNSConfigChanges puts each changed resolver of the host in the journal. A +// resolver that vanished comes without a nameserver, so the reader sees the +// drop. The search suffixes name the organization of the endpoint, so the +// journal holds their number only. +func (p *prog) logDNSConfigChanges(entries []dnsResolverEntry) { + for _, entry := range entries { + journal(mainLog.Load().Info()). + Strs("nameservers", entry.Nameservers). + Int("if_index", entry.IfIndex). + Str("interface", entry.Interface). + Bool("scoped", entry.Scoped). + Str("action", entry.Action). + Str("flags", entry.Flags). + Int("search_domain_count", len(entry.SearchDomains)). + Int("order", entry.Order). + Str("reachable", entry.Reachable). + Msg(dnsConfigChangedMessage) + } +} + // setupClientInfoDiscover performs necessary works for running client info discover. func (p *prog) setupClientInfoDiscover(selfIP string) { p.ciTable = clientinfo.NewTable(&cfg, selfIP, cdUID, p.ptrNameservers) @@ -998,7 +1099,7 @@ var ( startDNSInterceptFn = (*prog).startDNSIntercept ensureInterceptDNSTargetFn = (*prog).ensureInterceptDNSTarget removeInterceptDNSTargetFn = (*prog).removeInterceptDNSTarget - initializeOsResolverWithSystemNameserversFn = ctrld.InitializeOsResolverWithSystemNameservers + initializeOsResolverWithSystemNameserversFn = ctrld.InitializeOsResolverWithSystemNameserversReason setDnsForRunningIfaceFn = (*prog).setDnsForRunningIface resetDNSFn = (*prog).resetDNS // refuseFallbackFatal reports a startup failure the interface-DNS fallback @@ -1463,28 +1564,6 @@ func (p *prog) resetDNSForRunningIface(isStart bool, restoreStatic bool) (runnin return } -func (p *prog) logInterfacesState() { - withEachPhysicalInterfaces("", "", func(i *net.Interface) error { - addrs, err := i.Addrs() - if err != nil { - mainLog.Load().Warn().Str("interface", i.Name).Err(err).Msg("failed to get addresses") - } - nss, err := currentStaticDNS(i) - if err != nil { - mainLog.Load().Warn().Str("interface", i.Name).Err(err).Msg("failed to get DNS") - } - if len(nss) == 0 { - nss = currentDNS(i) - } - mainLog.Load().Debug(). - Any("addrs", addrs). - Strs("nameservers", nss). - Int("index", i.Index). - Msgf("interface state: %s", i.Name) - return nil - }) -} - // findWorkingInterface looks for a network interface with a valid IP configuration func findWorkingInterface(currentIface string) string { // Helper to check if IP is valid (not link-local) diff --git a/cmd/cli/query_error_sampler.go b/cmd/cli/query_error_sampler.go index 78d017b..af382f7 100644 --- a/cmd/cli/query_error_sampler.go +++ b/cmd/cli/query_error_sampler.go @@ -12,6 +12,10 @@ const ( sampleClassResolveFailed = "resolve_failed" sampleClassAllEndpointsFailed = "all_endpoints_failed" sampleClassSendResponseFailed = "send_response_failed" + // sampleClassInternalDomain names a failure of an Internal Domain resolver. + // It stays apart from the other classes, because an endpoint away from the + // organization network cannot reach such a resolver. + sampleClassInternalDomain = "internal_domain" ) const ( @@ -56,6 +60,9 @@ type errorSampler struct { windows map[sampleKey]*sampleWindow now func() time.Time after func(time.Duration, func()) *time.Timer + // onFailure counts the event for the query health tracker. The daemon sets + // it once at start, before a listener serves a query. + onFailure func(class string) } // event returns the event a call site logs with. The first querySampleLimit @@ -83,6 +90,11 @@ func (s *errorSampler) event(class, upstream string) *zerolog.Event { } s.mu.Unlock() + // A lowered line keeps its place in the breakdown of the window. It is one + // event, not one failed query, so it does not change the grade. + if s.onFailure != nil { + s.onFailure(class) + } s.logSummaries(summaries) if suppressed { diff --git a/cmd/cli/query_health.go b/cmd/cli/query_health.go new file mode 100644 index 0000000..b28fa60 --- /dev/null +++ b/cmd/cli/query_health.go @@ -0,0 +1,284 @@ +package cli + +import ( + "maps" + "slices" + "sync" + "time" + + "github.com/rs/zerolog" +) + +// Health classes of the query path. The names go into the journal. +const ( + queryHealthHealthy = "healthy" + queryHealthDegraded = "degraded" + queryHealthFailing = "failing" +) + +const ( + // queryHealthWindow is the length of one counting window and the time + // between two heartbeats. + queryHealthWindow = 15 * time.Minute + // queryHealthFailingRatio and queryHealthDegradedRatio grade the failed + // share of a window. + queryHealthFailingRatio = 0.5 + queryHealthDegradedRatio = 0.05 + // queryHealthFailingQueries is the smallest sample that can leave the + // healthy class and the smallest sample that can report a class change. + queryHealthFailingQueries = 10 + // queryHealthClassTicks is the number of ticks in a row that a new class + // must hold before it reports. + queryHealthClassTicks = 2 + // queryHealthMessage names the journal event of the tracker. + queryHealthMessage = "Query health" +) + +// queryHealthTick is the time between two evaluations. A test shortens it. +var queryHealthTick = time.Minute + +// queryHealth grades the query path over a window. Support reads the grade +// after an incident, so the journal needs one class per window instead of a +// line per query. +type queryHealth struct { + mu sync.Mutex + window time.Duration + now func() time.Time + windowStart time.Time + queries uint64 + cacheHits uint64 + // failedQueries counts the client queries that ended without an answer. + // The grade follows this count, and failures holds the sampler classes + // behind it, which raise several events for one failed query. + failedQueries uint64 + failures map[string]uint64 + lastClass string + // pendingClass is the class that the last tick graded, and pendingTicks + // counts how many ticks in a row graded it. A class reports only after it + // held, so one noisy minute opens no event. + pendingClass string + pendingTicks int +} + +// queryHealthEvent holds the class and the counters of one window. +type queryHealthEvent struct { + Class string + WindowSeconds int + Queries uint64 + CacheHits uint64 + FailedQueries uint64 + Failures map[string]uint64 + UpstreamsDown int + BypassActive bool + CacheHitRatio float64 +} + +func newQueryHealth() *queryHealth { + return &queryHealth{ + window: queryHealthWindow, + now: time.Now, + failures: make(map[string]uint64), + } +} + +// A prog outside a run holds no tracker, so every counter of the query path +// and every loop accepts a nil receiver. +func (q *queryHealth) countQuery() { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + q.queries++ +} + +func (q *queryHealth) countCacheHit() { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + q.cacheHits++ +} + +// countFailedQuery counts one client query that ended without an answer. A +// query that failed on several upstreams counts once, because the grade +// measures the service the client got. +func (q *queryHealth) countFailedQuery() { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + q.failedQueries++ +} + +func (q *queryHealth) countFailure(class string) { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + if q.failures == nil { + q.failures = make(map[string]uint64) + } + q.failures[class]++ +} + +// evaluate grades the counters and reports whether the event goes out. The +// report covers the window that just closed, or the running one. The window +// starts at the first evaluate, so the caller owns the clock. +func (q *queryHealth) evaluate(now time.Time, upstreamsDown int, bypassActive bool) (queryHealthEvent, bool) { + if q == nil { + return queryHealthEvent{}, false + } + q.mu.Lock() + defer q.mu.Unlock() + + if q.windowStart.IsZero() { + q.windowStart = now + } + event := queryHealthEvent{ + Class: healthClass(q.queries, q.failedQueries), + WindowSeconds: int(q.window / time.Second), + Queries: q.queries, + CacheHits: q.cacheHits, + FailedQueries: q.failedQueries, + Failures: copyFailures(q.failures), + UpstreamsDown: upstreamsDown, + BypassActive: bypassActive, + CacheHitRatio: shareOf(q.cacheHits, q.queries), + } + // The end of a window always reports, so a quiet daemon stays visible and + // no window is lost after an early report. A class change waits for a + // sample and for a second tick. + windowEnded := !now.Before(q.windowStart.Add(q.window)) + emit := q.lastClass == "" || windowEnded || q.classChangeHolds(event.Class) + if emit { + q.lastClass = event.Class + q.pendingClass, q.pendingTicks = "", 0 + } + if windowEnded { + q.startWindow(now) + } + return event, emit +} + +// classChangeHolds reports whether a new class may open an event. It needs a +// window with enough queries, and the same class on two ticks in a row. The +// caller holds the lock. +func (q *queryHealth) classChangeHolds(class string) bool { + if class == q.lastClass || q.queries < queryHealthFailingQueries { + q.pendingClass, q.pendingTicks = "", 0 + return false + } + if class != q.pendingClass { + q.pendingClass, q.pendingTicks = class, 1 + return false + } + q.pendingTicks++ + return q.pendingTicks >= queryHealthClassTicks +} + +// startWindow drops the counters of the window that closed. Each report covers +// one window, so a quiet hour cannot hide inside the totals of a busy one. The +// caller holds the lock. +func (q *queryHealth) startWindow(now time.Time) { + q.windowStart = now + q.queries = 0 + q.cacheHits = 0 + q.failedQueries = 0 + q.failures = make(map[string]uint64) +} + +func (q *queryHealth) log(event queryHealthEvent) { + if q == nil { + return + } + journal(mainLog.Load().Info()). + Str("class", event.Class). + Int("window_s", event.WindowSeconds). + Uint64("queries", event.Queries). + Uint64("cache_hits", event.CacheHits). + Uint64("failed_queries", event.FailedQueries). + Dict("failures_by_class", failuresDict(event.Failures)). + Int("upstreams_down", event.UpstreamsDown). + Bool("bypass_active", event.BypassActive). + Float64("cache_hit_ratio", event.CacheHitRatio). + Msg(queryHealthMessage) +} + +// startLoop runs the grading loop and returns a channel that closes when the +// loop returns. A caller joins that channel before it reads the log. +func (q *queryHealth) startLoop(stop <-chan struct{}, inputs func() (upstreamsDown int, bypassActive bool)) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) + q.loop(stop, inputs) + }() + return done +} + +// loop grades the counters on every tick until the caller stops it. +func (q *queryHealth) loop(stop <-chan struct{}, inputs func() (upstreamsDown int, bypassActive bool)) { + if q == nil { + return + } + ticker := time.NewTicker(queryHealthTick) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + upstreamsDown, bypassActive := inputs() + event, emit := q.evaluate(q.now(), upstreamsDown, bypassActive) + if !emit { + continue + } + q.log(event) + } + } +} + +// healthClass grades the failed share of a window. A small sample stays +// healthy, because a few failures at a start are not an outage. +func healthClass(queries, failures uint64) string { + if queries < queryHealthFailingQueries { + return queryHealthHealthy + } + failed := shareOf(failures, queries) + if failed >= queryHealthFailingRatio { + return queryHealthFailing + } + if failed >= queryHealthDegradedRatio { + return queryHealthDegraded + } + return queryHealthHealthy +} + +// shareOf reports the part of the total, and 0 for an empty total. +func shareOf(part, total uint64) float64 { + if total == 0 { + return 0 + } + return float64(part) / float64(total) +} + +// copyFailures gives the report its own map, because the counters keep running +// while the caller holds the report. +func copyFailures(failures map[string]uint64) map[string]uint64 { + copied := make(map[string]uint64, len(failures)) + maps.Copy(copied, failures) + return copied +} + +// failuresDict renders the classes in a stable order, so two lines of the same +// window compare by eye. +func failuresDict(failures map[string]uint64) *zerolog.Event { + dict := zerolog.Dict() + for _, class := range slices.Sorted(maps.Keys(failures)) { + dict = dict.Uint64(class, failures[class]) + } + return dict +} diff --git a/cmd/cli/query_health_prog_test.go b/cmd/cli/query_health_prog_test.go new file mode 100644 index 0000000..cd0520d --- /dev/null +++ b/cmd/cli/query_health_prog_test.go @@ -0,0 +1,459 @@ +package cli + +import ( + "context" + "errors" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/stretchr/testify/require" + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/dnscache" + "github.com/Control-D-Inc/ctrld/testhelper" +) + +// wantDNSConfigStrings compares one array field of a logged resolver entry. +func wantDNSConfigStrings(t *testing.T, event map[string]any, field string, want ...string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("field %q: got %T, want an array", field, event[field]) + } + if len(values) != len(want) { + t.Fatalf("field %q: got %v, want %v", field, values, want) + } + for i, value := range values { + if value != want[i] { + t.Fatalf("field %q at %d: got %v, want %v", field, i, value, want[i]) + } + } +} + +// Test_queryHealthCountsFailedProxyQueries drives the real query path against a +// refusing upstream: every sampled error reaches the tracker, so the window of +// the daemon grades as failing. +func Test_queryHealthCountsFailedProxyQueries(t *testing.T) { + captureDebugMainLog(t) + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg), health: newQueryHealth()} + // A new monitor holds back the failure counts for one second, and its + // timer would mark the upstream down after the test. + p.um.clearRecovered(upstreamOS) + p.um.after = func(time.Duration, func()) {} + p.querySampler.onFailure = p.health.countFailure + // The zero value sampler arms a real timer. Close its window after the + // assertions, so no late summary line reaches another test. + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + origOSUpstream := osUpstreamConfig + t.Cleanup(func() { osUpstreamConfig = origOSUpstream }) + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "dead resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 500, + } + + const failedQueries = 12 + const servedQueries = 20 + ctx := context.Background() + for i := 0; i < failedQueries; i++ { + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeServerFailure { + t.Fatalf("query %d: rcode %s, want SERVFAIL", i+1, dns.RcodeToString[rcode]) + } + } + // serveDNS counts every query it serves, and the eight answered ones never + // reach the sampler. + for i := 0; i < servedQueries; i++ { + p.health.countQuery() + } + + event, emit := p.health.evaluate(time.Now(), 0, false) + if !emit { + t.Fatal("the first evaluate did not emit") + } + if event.Class != queryHealthFailing { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthFailing) + } + if event.Queries != servedQueries { + t.Fatalf("queries: got %d, want %d", event.Queries, servedQueries) + } + // The grade follows one failure per failed query. Each of those queries + // also raised two sampler events, and the breakdown still shows them. + if event.FailedQueries != failedQueries { + t.Fatalf("failed queries: got %d, want %d", event.FailedQueries, failedQueries) + } + if got := event.Failures[sampleClassResolveFailed]; got != failedQueries { + t.Fatalf("%s failures: got %d, want %d", sampleClassResolveFailed, got, failedQueries) + } + if got := event.Failures[sampleClassAllEndpointsFailed]; got != failedQueries { + t.Fatalf("%s failures: got %d, want %d", sampleClassAllEndpointsFailed, got, failedQueries) + } +} + +// Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade drives the same +// number of failed queries against an Internal Domain resolver. An endpoint +// away from the organization network cannot reach that resolver, so the class +// stays healthy, the failures keep their own name, and the down entry of that +// resolver stays out of upstreams_down. +func Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade(t *testing.T) { + captureDebugMainLog(t) + const internalUpstream = upstreamPrefix + internalDomainUpstreamPrefix + "1" + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + cfg.Upstream = map[string]*ctrld.UpstreamConfig{ + internalDomainUpstreamPrefix + "1": { + Name: "dead internal resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 500, + }, + } + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg), health: newQueryHealth()} + p.um.after = func(time.Duration, func()) {} + p.um.clearRecovered(internalUpstream) + p.querySampler.onFailure = p.health.countFailure + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + const failedQueries = 12 + const servedQueries = 20 + ctx := context.Background() + msg := new(dns.Msg) + msg.SetQuestion("internal.corp.example.", dns.TypeA) + for i := 0; i < failedQueries; i++ { + res := p.proxy(ctx, &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{upstreams: []string{internalUpstream}, srcAddr: "192.168.0.1:1234"}, + }) + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeServerFailure { + t.Fatalf("query %d: the internal resolver answered", i+1) + } + } + for i := 0; i < servedQueries; i++ { + p.health.countQuery() + } + p.um.mu.Lock() + p.um.markDown(internalUpstream, 3, "immediate") + p.um.mu.Unlock() + + event, emit := p.health.evaluate(time.Now(), p.um.countDownExcept(isInternalDomainUpstream), false) + if !emit { + t.Fatal("the first evaluate did not emit") + } + if event.Class != queryHealthHealthy { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthHealthy) + } + if event.FailedQueries != 0 { + t.Fatalf("failed queries: got %d, want 0", event.FailedQueries) + } + if got := event.Failures[sampleClassInternalDomain]; got != failedQueries { + t.Fatalf("%s failures: got %d, want %d", sampleClassInternalDomain, got, failedQueries) + } + if event.UpstreamsDown != 0 { + t.Fatalf("upstreams_down: got %d, want 0", event.UpstreamsDown) + } + if got := p.um.countDown(); got != 1 { + t.Fatalf("countDown: got %d, want 1", got) + } +} + +// Test_queryHealthCountsProxyCacheHits proves that an answer from the cache +// counts a cache hit, so the ratio of the journal follows the query path. +func Test_queryHealthCountsProxyCacheHits(t *testing.T) { + captureDebugMainLog(t) + p := &prog{cfg: testhelper.SampleConfig(t), health: newQueryHealth()} + cache, err := dnscache.NewLRUCache(16) + require.NoError(t, err) + p.cache = cache + + msg := new(dns.Msg) + msg.SetQuestion("cache.test.", dns.TypeA) + answer := new(dns.Msg) + answer.SetRcode(msg, dns.RcodeSuccess) + p.cache.Add(dnscache.NewKey(msg, "upstream.1"), dnscache.NewValue(answer, time.Now().Add(time.Minute))) + + res := p.proxy(context.Background(), &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{upstreams: []string{"upstream.1"}}, + }) + require.NotNil(t, res) + require.True(t, res.cached, "the answer did not come from the cache") + + event, _ := p.health.evaluate(time.Now(), 0, false) + if event.CacheHits != 1 { + t.Fatalf("cache hits: got %d, want 1", event.CacheHits) + } +} + +// Test_logDNSConfigChangesLogsEveryEntry proves that each changed resolver of +// the host reaches the journal with its fields. A resolver that vanished keeps +// an empty nameserver list. +func Test_logDNSConfigChangesLogsEveryEntry(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + + p.logDNSConfigChanges([]dnsResolverEntry{ + { + Order: 200000, + Nameservers: []string{"192.168.50.200", "192.168.50.1"}, + IfIndex: 16, + Interface: "en0", + Flags: "Request A records, Request AAAA records", + SearchDomains: []string{"corp.example"}, + Reachable: "Reachable,Directly Reachable Address", + }, + {Order: 101600, IfIndex: 21, Interface: "utun4", Flags: "Scoped"}, + }) + + events := jsonLogEvents(t, logs, dnsConfigChangedMessage) + if len(events) != 2 { + t.Fatalf("configuration lines: got %d, want 2", len(events)) + } + + wantField(t, events[0], "journal", true) + wantField(t, events[0], "level", "info") + wantField(t, events[0], "if_index", float64(16)) + wantField(t, events[0], "interface", "en0") + wantField(t, events[0], "flags", "Request A records, Request AAAA records") + wantField(t, events[0], "order", float64(200000)) + wantField(t, events[0], "reachable", "Reachable,Directly Reachable Address") + wantDNSConfigStrings(t, events[0], "nameservers", "192.168.50.200", "192.168.50.1") + // The suffixes of an organization stay out of the journal. Only their + // number goes in. + wantField(t, events[0], "search_domain_count", float64(1)) + if _, ok := events[0]["search_domains"]; ok { + t.Fatal("the journal holds the search domains") + } + + wantField(t, events[1], "journal", true) + wantField(t, events[1], "interface", "utun4") + wantDNSConfigStrings(t, events[1], "nameservers") + wantField(t, events[1], "search_domain_count", float64(0)) +} + +// Test_startNetworkJournalLogsTheStartSnapshot proves that the daemon opens the +// journal with one snapshot and runs the trackers that follow the query path +// and the DNS configuration of the host. +func Test_startNetworkJournalLogsTheStartSnapshot(t *testing.T) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + origTick := queryHealthTick + t.Cleanup(func() { queryHealthTick = origTick }) + queryHealthTick = time.Millisecond + + var polls atomic.Int64 + // The poll loop reads the delay on every pass, so it parks on a long delay + // before the test puts the package value back. + parked := make(chan struct{}, 1) + origRunner, origDelay := runSCUtilDNSFn, dnsConfigDelayFn + t.Cleanup(func() { runSCUtilDNSFn, dnsConfigDelayFn = origRunner, origDelay }) + movedResolver := wifiResolverEntry() + movedResolver.Nameservers = []string{"10.0.0.1"} + runSCUtilDNSFn = func() ([]dnsResolverEntry, error) { + if polls.Add(1) == 1 { + return []dnsResolverEntry{wifiResolverEntry()}, nil + } + return []dnsResolverEntry{movedResolver}, nil + } + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { + if polls.Load() < 2 { + return time.Millisecond + } + select { + case parked <- struct{}{}: + default: + } + return time.Hour + } + + cfg := &ctrld.Config{} + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg), stopCh: make(chan struct{})} + // The monitor arms a real timer for each upstream it resets. Nothing in + // this test waits for one. + p.um.after = func(time.Duration, func()) {} + p.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.9/24")) + p.recoveryBypass.Store(true) + p.um.mu.Lock() + p.um.markDown(upstreamOS, 3, "timer") + p.um.mu.Unlock() + + p.newNetworkJournal() + if p.health == nil || p.querySampler.onFailure == nil { + t.Fatal("the trackers do not exist before the loops start") + } + // A counter of the query path works as soon as the trackers exist. + p.health.countQuery() + waitForJournalLoops := p.startNetworkJournal() + t.Cleanup(func() { + close(p.stopCh) + waitForJournalLoops() + }) + + events := jsonLogEvents(t, logs, networkSnapshotMessage) + if len(events) != 1 { + t.Fatalf("snapshot lines: got %d, want 1", len(events)) + } + wantField(t, events[0], "journal", true) + wantField(t, events[0], "level", "info") + wantField(t, events[0], "trigger", "start") + + waitForLogLine(t, logs, queryHealthMessage) + health := jsonLogEvents(t, logs, queryHealthMessage) + wantField(t, health[0], "upstreams_down", float64(1)) + wantField(t, health[0], "bypass_active", true) + + if runtime.GOOS != "darwin" { + if p.dnsConfig != nil { + t.Fatal("the DNS configuration poller runs outside macOS") + } + return + } + if p.dnsConfig == nil { + t.Fatal("macOS starts no DNS configuration poller") + } + waitForLogLine(t, logs, dnsConfigChangedMessage) + select { + case <-parked: + case <-time.After(5 * time.Second): + t.Fatal("the poll loop did not park") + } + resolvers := jsonLogEvents(t, logs, dnsConfigChangedMessage) + if len(resolvers) != 1 { + t.Fatalf("configuration lines: got %d, want 1", len(resolvers)) + } + wantField(t, resolvers[0], "interface", "en0") + wantDNSConfigStrings(t, resolvers[0], "nameservers", "10.0.0.1") +} + +// Test_startNetworkJournalReadsTheMonitorAcrossAReload replaces the upstream +// monitor while the health loop runs. A reload builds a new monitor, and the +// loop read the field of the prog with no lock, which the race detector +// reports. +func Test_startNetworkJournalReadsTheMonitorAcrossAReload(t *testing.T) { + captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + origTick := queryHealthTick + t.Cleanup(func() { queryHealthTick = origTick }) + queryHealthTick = time.Millisecond + + cfg := &ctrld.Config{} + p := &prog{cfg: cfg, stopCh: make(chan struct{})} + p.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.9/24")) + p.replaceUpstreamMonitor() + p.health = newQueryHealth() + + waitForJournalLoops := p.startNetworkJournal() + for i := 0; i < 30; i++ { + p.replaceUpstreamMonitor() + time.Sleep(time.Millisecond) + } + close(p.stopCh) + waitForJournalLoops() +} + +// Test_startNetworkJournalRunsAfterAFailedMonitorStart proves that a network +// monitor that cannot start leaves the journal running. The error line goes +// out, the start snapshot opens the journal, and the health loop reports. +func Test_startNetworkJournalRunsAfterAFailedMonitorStart(t *testing.T) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + origTick := queryHealthTick + t.Cleanup(func() { queryHealthTick = origTick }) + queryHealthTick = time.Millisecond + + origMonitor := newNetworkChangeMonitorFn + t.Cleanup(func() { newNetworkChangeMonitorFn = origMonitor }) + newNetworkChangeMonitorFn = func(func(string, ...any)) (networkChangeMonitor, error) { + return nil, errors.New("no network monitor here") + } + origRunner, origDelay := runSCUtilDNSFn, dnsConfigDelayFn + t.Cleanup(func() { runSCUtilDNSFn, dnsConfigDelayFn = origRunner, origDelay }) + runSCUtilDNSFn = func() ([]dnsResolverEntry, error) { + return []dnsResolverEntry{wifiResolverEntry()}, nil + } + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return time.Hour } + + cfg := &ctrld.Config{} + p := &prog{cfg: cfg, stopCh: make(chan struct{})} + p.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.9/24")) + p.replaceUpstreamMonitor() + p.upstreamMonitorNow().after = func(time.Duration, func()) {} + + p.newNetworkJournal() + p.health.countQuery() + p.health.mu.Lock() + queries := p.health.queries + p.health.mu.Unlock() + if queries != 1 { + t.Fatalf("queries after one count: got %d, want 1", queries) + } + + waitForJournalLoops := p.startNetworkMonitorAndJournal() + t.Cleanup(func() { + close(p.stopCh) + waitForJournalLoops() + }) + + failures := jsonLogEvents(t, logs, "Failed to start network monitoring") + if len(failures) != 1 { + t.Fatalf("monitor failure lines: got %d, want 1", len(failures)) + } + wantField(t, failures[0], "level", "error") + + snapshots := jsonLogEvents(t, logs, networkSnapshotMessage) + if len(snapshots) != 1 { + t.Fatalf("snapshot lines: got %d, want 1", len(snapshots)) + } + wantField(t, snapshots[0], "trigger", "start") + + waitForLogLine(t, logs, queryHealthMessage) +} + +// Test_queryHealthCountsNoFailureWhenAnUpstreamAnswers proves that a failure on +// one upstream does not grade the query. The client got an answer, so the +// failure stays in the breakdown only. The DNS64 companion lookup fails through +// the same call, so it cannot grade a query either. +func Test_queryHealthCountsNoFailureWhenAnUpstreamAnswers(t *testing.T) { + captureDebugMainLog(t) + fixture := startDNSFixture(t) + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + cfg.Upstream = map[string]*ctrld.UpstreamConfig{ + "0": {Name: "dead resolver", Type: ctrld.ResolverTypeLegacy, Endpoint: deadUpstreamEndpoint, Timeout: 500}, + "1": {Name: "live resolver", Type: ctrld.ResolverTypeLegacy, Endpoint: fixture.addr, Timeout: 2000}, + } + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg), health: newQueryHealth()} + p.um.after = func(time.Duration, func()) {} + p.um.clearRecovered(upstreamPrefix + "0") + p.querySampler.onFailure = p.health.countFailure + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + msg := new(dns.Msg) + msg.SetQuestion("failover.test.", dns.TypeA) + res := p.proxy(context.Background(), &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{ + upstreams: []string{upstreamPrefix + "0", upstreamPrefix + "1"}, + srcAddr: "192.168.0.1:1234", + }, + }) + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Fatal("the second upstream did not answer") + } + p.health.countQuery() + + event, _ := p.health.evaluate(time.Now(), 0, false) + if event.FailedQueries != 0 { + t.Fatalf("failed queries: got %d, want 0", event.FailedQueries) + } + if got := event.Failures[sampleClassResolveFailed]; got != 1 { + t.Fatalf("%s failures: got %d, want 1", sampleClassResolveFailed, got) + } + if event.Class != queryHealthHealthy { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthHealthy) + } +} diff --git a/cmd/cli/query_health_test.go b/cmd/cli/query_health_test.go new file mode 100644 index 0000000..88f120c --- /dev/null +++ b/cmd/cli/query_health_test.go @@ -0,0 +1,424 @@ +package cli + +import ( + "fmt" + "reflect" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/rs/zerolog" +) + +// queryHealthBase is the start time of every tracker test. The tracker takes +// the time from the caller, so no test waits for a real clock. +var queryHealthBase = time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + +// feedQueryHealth counts queries, and one failed query for each failure of a +// class. The grade follows the failed queries, so a test that wants a class +// gives the failures that produced it. +func feedQueryHealth(q *queryHealth, queries int, failures map[string]int) { + for i := 0; i < queries; i++ { + q.countQuery() + } + for class, count := range failures { + for i := 0; i < count; i++ { + q.countFailure(class) + q.countFailedQuery() + } + } +} + +// waitForLogLine polls the buffer, because the loop logs from its own goroutine. +func waitForLogLine(t *testing.T, logs *syncBuffer, message string) { + t.Helper() + waitForLogLineOrExplain(t, logs, message, nil) +} + +// waitForLogLineOrExplain is waitForLogLine with extra text for the failure. +func waitForLogLineOrExplain(t *testing.T, logs *syncBuffer, message string, explain func() string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if strings.Contains(logs.String(), message) { + return + } + time.Sleep(time.Millisecond) + } + extra := "" + if explain != nil { + extra = " " + explain() + } + t.Fatalf("no %q line before the deadline (global level %s, captured %q)%s", message, zerolog.GlobalLevel(), logs.String(), extra) +} + +// loggerSwapRecord is one change of the main logger or of the global level. +type loggerSwapRecord struct { + at time.Duration + level zerolog.Level + writer string +} + +// watchLoggerSwaps records every change of the main logger and of the global +// level while the test runs, so a missing line names what took the logger. +func watchLoggerSwaps(t *testing.T) func() string { + t.Helper() + start := time.Now() + stop := make(chan struct{}) + done := make(chan struct{}) + var mu sync.Mutex + var records []loggerSwapRecord + go func() { + defer close(done) + lastLogger, lastLevel := mainLog.Load(), zerolog.GlobalLevel() + for { + select { + case <-stop: + return + default: + } + logger, level := mainLog.Load(), zerolog.GlobalLevel() + if logger != lastLogger || level != lastLevel { + mu.Lock() + records = append(records, loggerSwapRecord{time.Since(start), level, loggerWriterType(logger)}) + mu.Unlock() + lastLogger, lastLevel = logger, level + } + time.Sleep(50 * time.Microsecond) + } + }() + t.Cleanup(func() { close(stop); <-done }) + return func() string { + mu.Lock() + defer mu.Unlock() + parts := make([]string, 0, len(records)) + for _, r := range slices.Clone(records) { + parts = append(parts, fmt.Sprintf("{at=%s level=%s writer=%s}", r.at, r.level, r.writer)) + } + return fmt.Sprintf("swaps=%d %s current=%s", len(records), strings.Join(parts, " "), loggerWriterType(mainLog.Load())) + } +} + +// loggerWriterType names the writer behind a logger through its unexported +// field, so a swapped logger tells which init function built it. +func loggerWriterType(l *zerolog.Logger) string { + if l == nil { + return "nil" + } + w := reflect.ValueOf(l).Elem().FieldByName("w") + if !w.IsValid() || w.IsNil() { + return "none" + } + inner := w.Elem() + name := inner.Type().String() + if inner.Kind() == reflect.Struct && inner.NumField() == 1 && inner.Field(0).Kind() == reflect.Interface && !inner.Field(0).IsNil() { + name += "(" + inner.Field(0).Elem().Type().String() + ")" + } + return name +} + +func Test_queryHealthClassThresholds(t *testing.T) { + // The class names go into the journal, so the test pins the text. + tests := []struct { + name string + queries int + failures int + want string + }{ + {"no queries", 0, 0, "healthy"}, + {"two percent", 100, 2, "healthy"}, + {"five percent", 100, 5, "degraded"}, + {"six percent", 100, 6, "degraded"}, + {"half of twenty", 20, 12, "failing"}, + {"half of ten", 10, 5, "failing"}, + {"small sample cannot fail", 9, 9, "healthy"}, + {"small sample cannot degrade", 9, 1, "healthy"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, tc.queries, map[string]int{sampleClassResolveFailed: tc.failures}) + + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit { + t.Fatal("the first evaluate did not emit") + } + if event.Class != tc.want { + t.Fatalf("class: got %q, want %q", event.Class, tc.want) + } + }) + } +} + +func Test_queryHealthEmitsHeartbeatWithoutClassChange(t *testing.T) { + q := newQueryHealth() + if _, emit := q.evaluate(queryHealthBase, 0, false); !emit { + t.Fatal("the first evaluate did not emit") + } + + for _, minutes := range []int{1, 7, 14} { + at := queryHealthBase.Add(time.Duration(minutes) * time.Minute) + if _, emit := q.evaluate(at, 0, false); emit { + t.Fatalf("emit at minute %d without a class change", minutes) + } + } + + event, emit := q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit { + t.Fatal("no heartbeat at 15 minutes") + } + if event.Class != "healthy" { + t.Fatalf("heartbeat class: got %q, want %q", event.Class, "healthy") + } + if _, emit := q.evaluate(queryHealthBase.Add(16*time.Minute), 0, false); emit { + t.Fatal("a second emit follows the heartbeat") + } +} + +func Test_queryHealthEmitsOnClassChange(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 100, nil) + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit || event.Class != "healthy" { + t.Fatalf("first report: emit %v, class %q", emit, event.Class) + } + + feedQueryHealth(q, 0, map[string]int{sampleClassResolveFailed: 50}) + if _, emit := q.evaluate(queryHealthBase.Add(time.Minute), 0, false); emit { + t.Fatal("emit on the first tick of the new class") + } + + event, emit = q.evaluate(queryHealthBase.Add(2*time.Minute), 0, false) + if !emit { + t.Fatal("no emit after the new class held two ticks") + } + if event.Class != "failing" { + t.Fatalf("class: got %q, want %q", event.Class, "failing") + } + + if _, emit := q.evaluate(queryHealthBase.Add(3*time.Minute), 0, false); emit { + t.Fatal("emit without a class change") + } +} + +// Test_queryHealthHoldsAClassChangeForTwoTicks proves that a class which lasts +// one tick reports nothing. A minute of noise must not open an event. +func Test_queryHealthHoldsAClassChangeForTwoTicks(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 20, nil) + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit || event.Class != queryHealthHealthy { + t.Fatalf("first report: emit %v, class %q", emit, event.Class) + } + + feedQueryHealth(q, 0, map[string]int{sampleClassResolveFailed: 12}) + if event, emit := q.evaluate(queryHealthBase.Add(time.Minute), 0, false); emit { + t.Fatalf("emit after one tick of class %q", event.Class) + } + + // The next tick grades the same window as degraded, so the failing class + // never held two ticks and never reported. + feedQueryHealth(q, 80, nil) + if event, emit := q.evaluate(queryHealthBase.Add(2*time.Minute), 0, false); emit { + t.Fatalf("emit after the class moved again to %q", event.Class) + } + + event, emit = q.evaluate(queryHealthBase.Add(3*time.Minute), 0, false) + if !emit { + t.Fatal("no emit after the new class held two ticks") + } + if event.Class != queryHealthDegraded { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthDegraded) + } +} + +// Test_queryHealthNeedsASampleBeforeAClassChange proves that the empty window +// which follows a heartbeat reports nothing. The counters start at zero, and +// zero queries say nothing about the query path. +func Test_queryHealthNeedsASampleBeforeAClassChange(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 20, map[string]int{sampleClassResolveFailed: 12}) + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit || event.Class != queryHealthFailing { + t.Fatalf("first report: emit %v, class %q", emit, event.Class) + } + + event, emit = q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit || event.Class != queryHealthFailing { + t.Fatalf("window end: emit %v, class %q", emit, event.Class) + } + + for _, minutes := range []int{16, 17} { + at := queryHealthBase.Add(time.Duration(minutes) * time.Minute) + if event, emit := q.evaluate(at, 0, false); emit { + t.Fatalf("emit at minute %d with %d queries and class %q", minutes, event.Queries, event.Class) + } + } +} + +func Test_queryHealthCacheHitRatio(t *testing.T) { + q := newQueryHealth() + event, _ := q.evaluate(queryHealthBase, 0, false) + if event.CacheHitRatio != 0 { + t.Fatalf("ratio without queries: got %v, want 0", event.CacheHitRatio) + } + + feedQueryHealth(q, 20, nil) + for i := 0; i < 5; i++ { + q.countCacheHit() + } + + event, _ = q.evaluate(queryHealthBase.Add(time.Minute), 0, false) + if event.Queries != 20 || event.CacheHits != 5 { + t.Fatalf("counters: got %d queries and %d cache hits, want 20 and 5", event.Queries, event.CacheHits) + } + if event.CacheHitRatio != 0.25 { + t.Fatalf("ratio: got %v, want 0.25", event.CacheHitRatio) + } +} + +func Test_queryHealthResetsCountersAtWindowEnd(t *testing.T) { + q := newQueryHealth() + if _, emit := q.evaluate(queryHealthBase, 0, false); !emit { + t.Fatal("the first evaluate did not emit") + } + feedQueryHealth(q, 20, map[string]int{sampleClassResolveFailed: 12}) + q.countCacheHit() + + closed, emit := q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit { + t.Fatal("no emit at the window end") + } + if closed.Queries != 20 || closed.CacheHits != 1 || closed.Failures[sampleClassResolveFailed] != 12 { + t.Fatalf("the closed window lost its counters: %+v", closed) + } + + q.countFailure(sampleClassResolveFailed) + next, _ := q.evaluate(queryHealthBase.Add(16*time.Minute), 0, false) + if next.Queries != 0 || next.CacheHits != 0 { + t.Fatalf("the counters survived the window end: %+v", next) + } + if next.Failures[sampleClassResolveFailed] != 1 { + t.Fatalf("failures of the new window: got %d, want 1", next.Failures[sampleClassResolveFailed]) + } +} + +func Test_queryHealthReportKeepsItsOwnFailures(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 10, map[string]int{sampleClassResolveFailed: 1}) + + report, _ := q.evaluate(queryHealthBase, 0, false) + q.countFailure(sampleClassResolveFailed) + + if got := report.Failures[sampleClassResolveFailed]; got != 1 { + t.Fatalf("the report shares the live failure map: got %d, want 1", got) + } +} + +func Test_queryHealthLogsJournalEvent(t *testing.T) { + swaps := watchLoggerSwaps(t) + logs := captureDebugMainLog(t) + q := newQueryHealth() + feedQueryHealth(q, 20, map[string]int{ + sampleClassResolveFailed: 8, + sampleClassAllEndpointsFailed: 4, + }) + for i := 0; i < 5; i++ { + q.countCacheHit() + } + + event, emit := q.evaluate(queryHealthBase, 2, true) + if !emit { + t.Fatal("the first evaluate did not emit") + } + q.log(event) + + events := jsonLogEvents(t, logs, queryHealthMessage) + if len(events) != 1 { + t.Fatalf("health lines: got %d, want 1 (global level %s, captured %q) %s", len(events), zerolog.GlobalLevel(), logs.String(), swaps()) + } + logged := events[0] + wantField(t, logged, "journal", true) + wantField(t, logged, "level", "info") + wantField(t, logged, "class", "failing") + wantField(t, logged, "window_s", float64(900)) + wantField(t, logged, "queries", float64(20)) + wantField(t, logged, "cache_hits", float64(5)) + wantField(t, logged, "upstreams_down", float64(2)) + wantField(t, logged, "bypass_active", true) + wantField(t, logged, "cache_hit_ratio", 0.25) + + failures, ok := logged["failures_by_class"].(map[string]any) + if !ok { + t.Fatalf("failures_by_class: got %T, want an object", logged["failures_by_class"]) + } + if len(failures) != 2 { + t.Fatalf("failure classes: got %d, want 2", len(failures)) + } + wantField(t, failures, sampleClassResolveFailed, float64(8)) + wantField(t, failures, sampleClassAllEndpointsFailed, float64(4)) +} + +func Test_queryHealthLoopEmitsUntilStop(t *testing.T) { + swaps := watchLoggerSwaps(t) + logs := captureDebugMainLog(t) + tick := queryHealthTick + defer func() { queryHealthTick = tick }() + queryHealthTick = time.Millisecond + + q := newQueryHealth() + stop := make(chan struct{}) + done := make(chan struct{}) + var once sync.Once + stopLoop := func() { + once.Do(func() { + close(stop) + <-done + }) + } + defer stopLoop() + + go func() { + defer close(done) + q.loop(stop, func() (int, bool) { return 1, true }) + }() + + waitForLogLineOrExplain(t, logs, queryHealthMessage, swaps) + stopLoop() + + events := jsonLogEvents(t, logs, queryHealthMessage) + if len(events) == 0 { + t.Fatal("the loop logged no health event") + } + wantField(t, events[0], "journal", true) + wantField(t, events[0], "upstreams_down", float64(1)) + wantField(t, events[0], "bypass_active", true) +} + +// Test_queryHealthReportsEveryWindowAfterAnEarlyEmit proves that a class +// change in the middle of a window does not make the rest of the window +// vanish. The heartbeat follows the window, not the last report. +func Test_queryHealthReportsEveryWindowAfterAnEarlyEmit(t *testing.T) { + q := newQueryHealth() + if _, emit := q.evaluate(queryHealthBase, 0, false); !emit { + t.Fatal("the first evaluate did not emit") + } + feedQueryHealth(q, 20, map[string]int{sampleClassResolveFailed: 20}) + if _, emit := q.evaluate(queryHealthBase.Add(5*time.Minute), 0, false); emit { + t.Fatal("emit on the first tick of the new class") + } + event, emit := q.evaluate(queryHealthBase.Add(6*time.Minute), 0, false) + if !emit || event.Class != queryHealthFailing { + t.Fatalf("class change at 6 minutes: emit %v, class %q", emit, event.Class) + } + feedQueryHealth(q, 7, nil) + event, emit = q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit { + t.Fatal("the end of the window did not emit") + } + if event.Queries != 27 { + t.Fatalf("queries in the window report: got %d, want 27", event.Queries) + } +} diff --git a/cmd/cli/recovery_events_test.go b/cmd/cli/recovery_events_test.go new file mode 100644 index 0000000..03ca001 --- /dev/null +++ b/cmd/cli/recovery_events_test.go @@ -0,0 +1,349 @@ +package cli + +import ( + "context" + "reflect" + "strings" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// The two events that bound one recovery pass. +const ( + recoveryBeginMessage = "Recovery begin" + recoveryEndMessage = "Recovery end" +) + +// recoveredUpstreamName is the upstream that the stubbed probe reports. +const recoveredUpstreamName = upstreamPrefix + "0" + +// recoveryEventHarness runs one whole recovery pass. It stubs every seam of +// the pass, because a real pass probes an upstream, reads the host resolvers, +// and rewrites the PF anchor. +type recoveryEventHarness struct { + prog *prog + logs *syncBuffer + dhcpServers []string + target string + recovered string +} + +func newRecoveryEventHarness(t *testing.T) *recoveryEventHarness { + t.Helper() + h := &recoveryEventHarness{ + logs: captureDebugMainLog(t), + prog: &prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}, + dhcpServers: []string{"192.168.10.1"}, + target: "127.0.0.53", + recovered: recoveredUpstreamName, + } + stubHeaderSnapshotSources(t) + h.prog.um = newUpstreamMonitor(&ctrld.Config{}) + h.prog.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.10/24")) + // A stabilizing host defers the DHCP exemption, so the pass rewrites no + // PF anchor. + h.prog.pfStabilizing.Store(true) + + intercept := dnsIntercept + discovery := initializeOsResolverWithSystemNameserversFn + ensure := ensureInterceptDNSTargetFn + wait := waitForUpstreamRecoveryFn + t.Cleanup(func() { + dnsIntercept = intercept + initializeOsResolverWithSystemNameserversFn = discovery + ensureInterceptDNSTargetFn = ensure + waitForUpstreamRecoveryFn = wait + }) + dnsIntercept = true + initializeOsResolverWithSystemNameserversFn = func(bool, string) ([]string, []string) { + return effectiveNameserversOf(h.dhcpServers), h.dhcpServers + } + ensureInterceptDNSTargetFn = func(p *prog, _ []string) { + p.interceptDNSTargetMu.Lock() + defer p.interceptDNSTargetMu.Unlock() + p.interceptDNSTargetSetValue = h.target + p.publishInterceptTarget(h.target) + } + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + return h.recovered, nil + } + return h +} + +// effectiveNameserversOf models the OS resolver list that the root package +// builds: every discovered server with a port, and a synthetic public resolver +// when the network gave no public one. +func effectiveNameserversOf(discovered []string) []string { + effective := make([]string, 0, len(discovered)+1) + for _, server := range discovered { + effective = append(effective, server+":53") + } + return append(effective, "76.76.2.0:53") +} + +// oneRecoveryEvent returns the single event with the given message. +func oneRecoveryEvent(t *testing.T, logs *syncBuffer, message string) map[string]any { + t.Helper() + events := jsonLogEvents(t, logs, message) + if len(events) != 1 { + t.Fatalf("%q events: got %d, want 1", message, len(events)) + } + return events[0] +} + +// snapshotTriggers lists the trigger of every snapshot event, in order. +func snapshotTriggers(t *testing.T, logs *syncBuffer) []string { + t.Helper() + var triggers []string + for _, event := range jsonLogEvents(t, logs, networkSnapshotMessage) { + trigger, _ := event["trigger"].(string) + triggers = append(triggers, trigger) + } + return triggers +} + +func Test_recoveryCompletedJournalsBeginAndEnd(t *testing.T) { + h := newRecoveryEventHarness(t) + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + begin := oneRecoveryEvent(t, h.logs, recoveryBeginMessage) + wantField(t, begin, "journal", true) + wantField(t, begin, "level", "info") + + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + wantField(t, end, "journal", true) + wantField(t, end, "outcome", "completed") + wantField(t, end, "recovered_upstream", recoveredUpstreamName) + wantField(t, end, "bypass_active", false) + wantField(t, end, "intercept_target_action", "set") + wantField(t, end, "dhcp_server_count", float64(1)) + if got, want := end["dhcp_servers"], []any{"192.168.10.1"}; !reflect.DeepEqual(got, want) { + t.Fatalf("dhcp_servers = %v, want %v", got, want) + } + + // The network of this recovery does not change, so the end snapshot equals + // the begin snapshot and the writer drops it. + triggers := snapshotTriggers(t, h.logs) + if want := []string{"recovery_begin"}; !reflect.DeepEqual(triggers, want) { + t.Fatalf("snapshot triggers = %v, want %v", triggers, want) + } +} + +// Test_recoveryEndReportsTheDiscoveredDHCPNameservers drives the real OS +// resolver initialization. The effective list carries a synthetic public +// resolver, and a reader who counts it reads a DHCP server that the network +// never gave. +func Test_recoveryEndReportsTheDiscoveredDHCPNameservers(t *testing.T) { + h := newRecoveryEventHarness(t) + previousRead := ctrld.NameserversFn + ctrld.NameserversFn = func() []string { return []string{"192.168.1.1"} } + t.Cleanup(func() { ctrld.NameserversFn = previousRead }) + initializeOsResolverWithSystemNameserversFn = ctrld.InitializeOsResolverWithSystemNameserversReason + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + wantField(t, end, "dhcp_server_count", float64(1)) + if got, want := end["dhcp_servers"], []any{"192.168.1.1"}; !reflect.DeepEqual(got, want) { + t.Fatalf("dhcp_servers = %v, want %v", got, want) + } +} + +// Test_recoveryJournalsTheReinitializedOSResolver covers the resolver line of +// a recovered network. The line names the nameservers that the recovery +// installed, and an outage report needs them. +func Test_recoveryJournalsTheReinitializedOSResolver(t *testing.T) { + h := newRecoveryEventHarness(t) + previousRead := ctrld.NameserversFn + ctrld.NameserversFn = func() []string { return []string{"192.168.1.1"} } + t.Cleanup(func() { ctrld.NameserversFn = previousRead }) + + h.prog.handleRecoveryForTransition(RecoveryReasonNetworkChange, 0) + + event := oneRecoveryEvent(t, h.logs, "Reinitialized OS resolver with nameservers: [192.168.1.1:53 76.76.2.0:53]") + wantField(t, event, "journal", true) + wantField(t, event, "level", "info") +} + +// Test_recoveryEndSpeedsUpTheDNSConfigurationPoll covers the poll cadence +// after a recovery. The resolver table of the host settles late, and a poll +// every five minutes reports that change long after the outage. +func Test_recoveryEndSpeedsUpTheDNSConfigurationPoll(t *testing.T) { + h := newRecoveryEventHarness(t) + now := time.Date(2026, 9, 17, 9, 0, 0, 0, time.UTC) + previousNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return now } + t.Cleanup(func() { networkEventsNowFn = previousNow }) + h.prog.dnsConfig = newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil }) + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + if got := h.prog.dnsConfig.nextDelay(now); got != dnsConfigFastInterval { + t.Fatalf("DNS configuration poll delay after a recovery = %s, want %s", got, dnsConfigFastInterval) + } +} + +func Test_recoveryJournalsTheRecoveredUpstream(t *testing.T) { + h := newRecoveryEventHarness(t) + h.prog.um.mu.Lock() + h.prog.um.markDown(recoveredUpstreamName, 7, "timer") + h.prog.um.mu.Unlock() + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + event := oneRecoveryEvent(t, h.logs, "Upstream \""+recoveredUpstreamName+"\" recovered; re-applying DNS settings") + wantField(t, event, "journal", true) + wantField(t, event, "upstream", recoveredUpstreamName) + if _, ok := event["down_for_ms"]; !ok { + t.Fatalf("event has no down_for_ms field: %v", event) + } +} + +// Test_recoveryBoundsTheRecoveredUpstreamName covers an upstream key that +// holds a token. The key is operator text, and the journal leaves the host +// for a support case, so no line may carry it. +func Test_recoveryBoundsTheRecoveredUpstreamName(t *testing.T) { + const secret = "SECRET0123" + h := newRecoveryEventHarness(t) + h.recovered = upstreamPrefix + "doh-" + secret + h.prog.um.mu.Lock() + h.prog.um.markDown(h.recovered, 1, "timer") + h.prog.um.mu.Unlock() + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + const boundedName = upstreamPrefix + "custom" + event := oneRecoveryEvent(t, h.logs, "Upstream \""+boundedName+"\" recovered; re-applying DNS settings") + wantField(t, event, "upstream", boundedName) + wantField(t, oneRecoveryEvent(t, h.logs, recoveryEndMessage), "recovered_upstream", boundedName) + if strings.Contains(h.logs.String(), secret) { + t.Fatalf("a log line holds the upstream key: %s", h.logs.String()) + } +} + +func Test_recoveryJournalsTheDHCPBypassLines(t *testing.T) { + h := newRecoveryEventHarness(t) + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + for _, message := range []string{ + "DNS intercept recovery: enabling DHCP bypass (filters stay active)", + "DNS intercept recovery: found DHCP nameservers: [192.168.10.1]", + "DNS intercept recovery complete: disabling DHCP bypass, resuming normal flow", + } { + wantField(t, oneRecoveryEvent(t, h.logs, message), "journal", true) + } +} + +func Test_recoveryJournalsTheMissingDHCPNameservers(t *testing.T) { + h := newRecoveryEventHarness(t) + h.dhcpServers = nil + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + event := oneRecoveryEvent(t, h.logs, "DNS intercept recovery: no DHCP nameservers found") + wantField(t, event, "journal", true) + wantField(t, oneRecoveryEvent(t, h.logs, recoveryEndMessage), "dhcp_server_count", float64(0)) +} + +// wantOneRecoveryEndPerGeneration asserts the two lines that bound one pass: +// one end event and one snapshot beside it. +func wantOneRecoveryEndPerGeneration(t *testing.T, h *recoveryEventHarness) map[string]any { + t.Helper() + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + triggers := snapshotTriggers(t, h.logs) + // Nothing changes between the begin and the end of these passes, so the + // snapshot limiter drops the end snapshot as a repeat of the begin one. + if want := []string{"recovery_begin"}; !reflect.DeepEqual(triggers, want) { + t.Fatalf("snapshot triggers = %v, want %v", triggers, want) + } + return end +} + +// Test_recoveryCanceledEndReportsTheCanceledPass covers the canceled outcome. +// A canceled pass releases the bypass, and support reads the end event to tell +// a canceled pass from a pass that never ended. +func Test_recoveryCanceledEndReportsTheCanceledPass(t *testing.T) { + h := newRecoveryEventHarness(t) + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + return "", context.Canceled + } + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + end := wantOneRecoveryEndPerGeneration(t, h) + wantField(t, end, "outcome", "canceled") + wantField(t, end, "bypass_active", false) + wantField(t, end, "recovered_upstream", "") +} + +// Test_recoverySupersededEndReportsNoChange covers the superseded outcome. The +// successor owns the target and the bypass flag, so the end event of the +// superseded pass must not report the state of its successor. +func Test_recoverySupersededEndReportsNoChange(t *testing.T) { + h := newRecoveryEventHarness(t) + waitForUpstreamRecoveryFn = func(p *prog, _ context.Context, _ map[string]*ctrld.UpstreamConfig, _ *recoveryDiagnostic) (string, error) { + // A network change took ownership while the probe ran. + p.recoveryGen.Add(1) + return recoveredUpstreamName, nil + } + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + end := wantOneRecoveryEndPerGeneration(t, h) + wantField(t, end, "outcome", "superseded") + wantField(t, end, "bypass_active", false) + wantField(t, end, "intercept_target_action", "unchanged") +} + +func Test_recoveryCanceledWithNoSuccessorIsJournaled(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + setupInterceptRecovery(t, p) + + p.recoveryCanceledCleanup(p.recoveryGen.Add(1)) + + event := oneRecoveryEvent(t, logs, "Recovery canceled with no successor; cleared recovery state and DHCP bypass") + wantField(t, event, "journal", true) +} + +func Test_interceptTargetAction(t *testing.T) { + for _, tc := range []struct { + name string + before, after string + want string + }{ + {"no target at all", "", "", "unchanged"}, + {"target kept", "127.0.0.53", "127.0.0.53", "unchanged"}, + {"target written", "", "127.0.0.53", "set"}, + {"target moved", "127.0.0.53", "127.0.0.54", "set"}, + {"target dropped", "127.0.0.53", "", "removed"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := interceptTargetAction(tc.before, tc.after); got != tc.want { + t.Fatalf("interceptTargetAction(%q, %q) = %q, want %q", tc.before, tc.after, got, tc.want) + } + }) + } +} + +func Test_noteNAT64ResultReportsChangesOnly(t *testing.T) { + p := &prog{} + const prefix = "64:ff9b::/96" + + for _, tc := range []struct { + result string + want bool + }{ + {prefix, true}, + {prefix, false}, + {"", true}, + } { + if got := p.noteNAT64Result(tc.result); got != tc.want { + t.Fatalf("noteNAT64Result(%q) = %v, want %v", tc.result, got, tc.want) + } + } +} diff --git a/cmd/cli/recovery_state.go b/cmd/cli/recovery_state.go index fa2e9de..0eae9cd 100644 --- a/cmd/cli/recovery_state.go +++ b/cmd/cli/recovery_state.go @@ -32,7 +32,7 @@ func (p *prog) recoveryOwnsState(gen uint64) bool { } func systemNameserversForInterceptRetry() []string { - _, system := initializeOsResolverWithSystemNameserversFn(true) + _, system := initializeOsResolverWithSystemNameserversFn(true, recoveryResolverReason) if system == nil { return []string{} } @@ -71,5 +71,5 @@ func (p *prog) recoveryCanceledCleanup(gen uint64) { // Superseded: the newer recovery owns the shared state. return } - mainLog.Load().Info().Msg("Recovery canceled with no successor; cleared recovery state and DHCP bypass") + journal(mainLog.Load().Info()).Msg("Recovery canceled with no successor; cleared recovery state and DHCP bypass") } diff --git a/cmd/cli/recovery_state_test.go b/cmd/cli/recovery_state_test.go index 4e615ad..4e821eb 100644 --- a/cmd/cli/recovery_state_test.go +++ b/cmd/cli/recovery_state_test.go @@ -143,7 +143,7 @@ func TestRecoveryCleanupClearsBypassAfterInterceptStateDisappears(t *testing.T) func TestSystemNameserversForInterceptRetryNormalizesEmptyDiscovery(t *testing.T) { original := initializeOsResolverWithSystemNameserversFn called := false - initializeOsResolverWithSystemNameserversFn = func(guard bool) ([]string, []string) { + initializeOsResolverWithSystemNameserversFn = func(guard bool, _ string) ([]string, []string) { called = true if !guard { t.Error("intercept retry discovery did not guard the existing resolver") diff --git a/cmd/cli/route_darwin.go b/cmd/cli/route_darwin.go new file mode 100644 index 0000000..15d81db --- /dev/null +++ b/cmd/cli/route_darwin.go @@ -0,0 +1,158 @@ +package cli + +import ( + "net" + "net/netip" + "syscall" + + "golang.org/x/net/route" +) + +// routeInterfaceNameFn names the interface of one route table index. A test +// replaces it, because the interfaces of the host are not a fixture. +var routeInterfaceNameFn = interfaceNameByIndex + +// defaultRoutes reads the default route of each family. It reads the kernel +// route table, because a snapshot follows every network change and a +// subprocess per change costs too much on a busy host. A table that the kernel +// does not give leaves both routes empty. +func defaultRoutes() (v4, v6 defaultRoute) { + rib, err := route.FetchRIB(syscall.AF_UNSPEC, route.RIBTypeRoute, 0) + if err != nil { + return defaultRoute{}, defaultRoute{} + } + messages, err := route.ParseRIB(route.RIBTypeRoute, rib) + if err != nil { + return defaultRoute{}, defaultRoute{} + } + return defaultRoutesFromMessages(messages) +} + +// defaultRoutesFromMessages picks the first default route of each family. The +// next hop and the interface come from one message, so a reader sees the pair +// that the kernel holds. The kernel keeps one unscoped default per family, so +// the first match is the only one. +func defaultRoutesFromMessages(messages []route.Message) (v4, v6 defaultRoute) { + foundV4, foundV6 := false, false + for _, message := range messages { + match, ok := matchDefaultRoute(message) + if !ok { + continue + } + found := defaultRoute{Interface: routeInterfaceNameFn(match.index)} + if match.gateway.IsValid() { + found.Gateway = match.gateway.String() + } + if match.v4 && !foundV4 { + v4, foundV4 = found, true + } + if !match.v4 && !foundV6 { + v6, foundV6 = found, true + } + if foundV4 && foundV6 { + return v4, v6 + } + } + return v4, v6 +} + +// defaultRouteMatch is one default route of the table. A tunnel default has +// no next hop, so gateway can be invalid while the interface is set. +type defaultRouteMatch struct { + v4 bool + gateway netip.Addr + index int +} + +// defaultRouteGateway reports the next hop and the interface index of one +// default route, for the tests of the selector. See matchDefaultRoute. +func defaultRouteGateway(message route.Message) (netip.Addr, int, bool) { + match, ok := matchDefaultRoute(message) + return match.gateway, match.index, ok +} + +// matchDefaultRoute reports whether one message is the default route of the +// host for its family. A route to another destination or to a part of the +// address space gives nothing: a VPN splits the default into two /1 routes, +// and the zero address alone does not make a route the default. A route bound +// to one interface serves that interface only, a host route is no default, and +// a reject or blackhole route delivers nothing. With the gateway flag, the +// next hop must be a host of the same family. Without it, the link itself +// delivers, which is how a tunnel owns the default; the match then has no +// gateway and names the interface. +func matchDefaultRoute(message route.Message) (defaultRouteMatch, bool) { + routeMessage, ok := message.(*route.RouteMessage) + if !ok || len(routeMessage.Addrs) <= syscall.RTAX_NETMASK { + return defaultRouteMatch{}, false + } + if routeMessage.Flags&(syscall.RTF_IFSCOPE|syscall.RTF_HOST|syscall.RTF_REJECT|syscall.RTF_BLACKHOLE) != 0 { + return defaultRouteMatch{}, false + } + destination, ok := routeMessageAddr(routeMessage.Addrs[syscall.RTAX_DST]) + if !ok || !destination.IsUnspecified() || !zeroNetmask(routeMessage.Addrs[syscall.RTAX_NETMASK], destination.Is4()) { + return defaultRouteMatch{}, false + } + match := defaultRouteMatch{v4: destination.Is4(), index: routeMessage.Index} + if routeMessage.Flags&syscall.RTF_GATEWAY == 0 { + if _, link := routeMessage.Addrs[syscall.RTAX_GATEWAY].(*route.LinkAddr); !link { + return defaultRouteMatch{}, false + } + return match, true + } + gateway, ok := routeMessageAddr(routeMessage.Addrs[syscall.RTAX_GATEWAY]) + if !ok || gateway.Is4() != destination.Is4() { + return defaultRouteMatch{}, false + } + match.gateway = gateway + return match, true +} + +// zeroNetmask reports whether the netmask of a route covers the whole address +// space of its family, which only the default route does. +func zeroNetmask(addr route.Addr, v4 bool) bool { + switch mask := addr.(type) { + case *route.Inet4Addr: + return v4 && mask.IP == [4]byte{} + case *route.Inet6Addr: + return !v4 && mask.IP == [16]byte{} + } + return false +} + +func routeMessageAddr(addr route.Addr) (netip.Addr, bool) { + switch address := addr.(type) { + case *route.Inet4Addr: + return netip.AddrFrom4(address.IP), true + case *route.Inet6Addr: + return withInterfaceZone(netip.AddrFrom16(address.IP), address.ZoneID), true + } + return netip.Addr{}, false +} + +// withInterfaceZone names the interface of a link-local next hop. Every link +// repeats the same range, so the address alone names no host. A tunnel gives +// the bare address fe80:: and the interface is the only part that tells the +// links apart. +func withInterfaceZone(address netip.Addr, zoneID int) netip.Addr { + if zoneID == 0 || !address.IsLinkLocalUnicast() { + return address + } + name := routeInterfaceNameFn(zoneID) + if name == "" { + return address + } + return address.WithZone(name) +} + +// interfaceNameByIndex names one interface of the route table. An index that +// no interface holds gives an empty name. +func interfaceNameByIndex(index int) string { + if index == 0 { + return "" + } + iface, err := net.InterfaceByIndex(index) + if err != nil { + return "" + } + return iface.Name +} diff --git a/cmd/cli/route_darwin_test.go b/cmd/cli/route_darwin_test.go new file mode 100644 index 0000000..8384cf1 --- /dev/null +++ b/cmd/cli/route_darwin_test.go @@ -0,0 +1,291 @@ +package cli + +import ( + "net/netip" + "syscall" + "testing" + + "golang.org/x/net/route" +) + +// stubRouteInterfaceNames names the interface of each index, because the +// interfaces of the host are not a fixture. +func stubRouteInterfaceNames(t *testing.T, names map[int]string) { + t.Helper() + orig := routeInterfaceNameFn + t.Cleanup(func() { routeInterfaceNameFn = orig }) + routeInterfaceNameFn = func(index int) string { return names[index] } +} + +// routeAddrs puts one destination, one netmask, and one gateway at the places +// of the route message that the kernel uses. A nil netmask leaves the slot +// empty, the shape of a host route. +func routeAddrs(destination, netmask, gateway route.Addr) []route.Addr { + addrs := make([]route.Addr, syscall.RTAX_NETMASK+1) + addrs[syscall.RTAX_DST] = destination + addrs[syscall.RTAX_GATEWAY] = gateway + addrs[syscall.RTAX_NETMASK] = netmask + return addrs +} + +func inet4(a, b, c, d byte) *route.Inet4Addr { + return &route.Inet4Addr{IP: [4]byte{a, b, c, d}} +} + +func inet6(address string, zoneID int) *route.Inet6Addr { + addr := netip.MustParseAddr(address) + return &route.Inet6Addr{IP: addr.As16(), ZoneID: zoneID} +} + +func Test_defaultRouteGateway(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{8: "utun4"}) + + for _, tt := range []struct { + name string + message route.Message + wantGateway string + wantIndex int + wantOK bool + }{ + { + name: "v4_gateway", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + wantGateway: "192.168.1.1", + wantIndex: 5, + wantOK: true, + }, + { + name: "v6_link_local_gateway_with_a_zone", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet6("::", 0), inet6("::", 0), inet6("fe80::1", 8)), + }, + wantGateway: "fe80::1%utun4", + wantIndex: 8, + wantOK: true, + }, + { + name: "destination_is_not_the_default_route", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(192, 168, 1, 0), inet4(255, 255, 255, 0), inet4(192, 168, 1, 1)), + }, + }, + { + name: "gateway_names_a_link", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), &route.LinkAddr{Index: 5, Name: "en0"}), + }, + }, + { + name: "gateway_family_differs_from_the_destination", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet6("fe80::1", 8)), + }, + }, + { + name: "address_list_holds_no_gateway", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: []route.Addr{inet4(0, 0, 0, 0)}, + }, + }, + { + name: "message_is_not_a_route", + message: &route.InterfaceMessage{Index: 5}, + }, + { + name: "route_has_no_gateway_flag", + message: &route.RouteMessage{ + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + }, + { + name: "route_is_bound_to_one_interface", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY | syscall.RTF_IFSCOPE, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + gateway, index, ok := defaultRouteGateway(tt.message) + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v", ok, tt.wantOK) + } + if !ok { + return + } + if gateway.String() != tt.wantGateway { + t.Errorf("gateway = %q, want %q", gateway.String(), tt.wantGateway) + } + if index != tt.wantIndex { + t.Errorf("index = %d, want %d", index, tt.wantIndex) + } + }) + } +} + +// Test_defaultRoutesFromMessages covers the table of a host with a v4 default +// route and a v6 default route on another interface. The v6 route must not +// take the interface of the v4 route. +func Test_defaultRoutesFromMessages(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{4: "en0", 8: "utun4"}) + + v4, v6 := defaultRoutesFromMessages([]route.Message{ + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY | syscall.RTF_IFSCOPE, + Index: 9, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(10, 0, 0, 1)), + }, + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 4, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet6("::", 0), inet6("::", 0), inet6("fe80::1", 8)), + }, + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(172, 16, 0, 1)), + }, + }) + + if want := (defaultRoute{Gateway: "192.168.1.1", Interface: "en0"}); v4 != want { + t.Errorf("v4 route = %+v, want %+v", v4, want) + } + if want := (defaultRoute{Gateway: "fe80::1%utun4", Interface: "utun4"}); v6 != want { + t.Errorf("v6 route = %+v, want %+v", v6, want) + } +} + +// Test_defaultRoutesFromMessagesWithoutADefaultRoute covers a table that holds +// no default route, for example a host with no link. +func Test_defaultRoutesFromMessagesWithoutADefaultRoute(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{4: "en0"}) + + v4, v6 := defaultRoutesFromMessages([]route.Message{ + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 4, + Addrs: routeAddrs(inet4(192, 168, 1, 0), inet4(255, 255, 255, 0), inet4(192, 168, 1, 1)), + }, + }) + + if v4 != (defaultRoute{}) || v6 != (defaultRoute{}) { + t.Errorf("routes = %+v / %+v, want two empty routes", v4, v6) + } +} + +// Test_defaultRouteGatewayNeedsAZeroNetmask covers routes whose destination is +// the zero address but whose prefix is not /0. OpenVPN splits the default into +// two /1 routes, and a host route to 0.0.0.0 carries no mask at all. +func Test_defaultRouteGatewayNeedsAZeroNetmask(t *testing.T) { + for _, tt := range []struct { + name string + message route.Message + }{ + { + name: "half of the v4 space", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(128, 0, 0, 0), inet4(10, 8, 0, 1)), + }, + }, + { + name: "host route to the zero address", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY | syscall.RTF_HOST, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), nil, inet4(10, 8, 0, 1)), + }, + }, + { + name: "one eighth of the v6 space", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet6("::", 0), inet6("e000::", 0), inet6("fe80::1", 8)), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + if gateway, _, ok := defaultRouteGateway(tt.message); ok { + t.Fatalf("a %s route counted as the default route with gateway %s", tt.name, gateway) + } + }) + } + v4, _ := defaultRoutesFromMessages([]route.Message{&route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(128, 0, 0, 0), inet4(10, 8, 0, 1)), + }}) + if v4.Gateway != "" || v4.Interface != "" { + t.Fatalf("a table with one /1 route reported the default route %+v, want none", v4) + } +} + +// Test_defaultRoutesTakeALinkDefault covers a VPN that owns the default +// route. Its default has no next hop: the gateway is the link itself and the +// gateway flag is off. The snapshot must name that interface, with an empty +// gateway, instead of the interface of the network monitor. +func Test_defaultRoutesTakeALinkDefault(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{28: "utun4", 16: "en0"}) + linkDefault := &route.RouteMessage{ + Flags: syscall.RTF_UP | syscall.RTF_STATIC, + Index: 28, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), &route.LinkAddr{Index: 28, Name: "utun4"}), + } + scopedEn0 := &route.RouteMessage{ + Flags: syscall.RTF_UP | syscall.RTF_GATEWAY | syscall.RTF_IFSCOPE, + Index: 16, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 50, 1)), + } + + v4, _ := defaultRoutesFromMessages([]route.Message{linkDefault, scopedEn0}) + if v4.Interface != "utun4" || v4.Gateway != "" { + t.Fatalf("default route = %+v, want utun4 with no gateway", v4) + } +} + +// Test_defaultRouteGatewaySkipsRejectAndBlackholeRoutes covers a kill switch +// that installs a default route to nowhere. Such a route carries the gateway +// flag, but it delivers nothing, so it is not the default route of the host. +func Test_defaultRouteGatewaySkipsRejectAndBlackholeRoutes(t *testing.T) { + for _, tt := range []struct { + name string + flags int + }{ + {"reject", syscall.RTF_UP | syscall.RTF_GATEWAY | syscall.RTF_REJECT}, + {"blackhole", syscall.RTF_UP | syscall.RTF_GATEWAY | syscall.RTF_BLACKHOLE}, + } { + t.Run(tt.name, func(t *testing.T) { + message := &route.RouteMessage{ + Flags: tt.flags, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(10, 8, 0, 1)), + } + if gateway, _, ok := defaultRouteGateway(message); ok { + t.Fatalf("a %s route counted as the default route with gateway %s", tt.name, gateway) + } + }) + } +} diff --git a/cmd/cli/route_others.go b/cmd/cli/route_others.go new file mode 100644 index 0000000..65f1330 --- /dev/null +++ b/cmd/cli/route_others.go @@ -0,0 +1,10 @@ +//go:build !darwin + +package cli + +// defaultRoutes gives no route. Only macOS reads the route table in this +// release, so the snapshot of another system holds no gateway and no default +// route of the v6 family. +func defaultRoutes() (v4, v6 defaultRoute) { + return defaultRoute{}, defaultRoute{} +} diff --git a/cmd/cli/tunnel_owner.go b/cmd/cli/tunnel_owner.go new file mode 100644 index 0000000..b57b085 --- /dev/null +++ b/cmd/cli/tunnel_owner.go @@ -0,0 +1,35 @@ +package cli + +import ( + "bufio" + "io" + "strings" +) + +// agentDescriptionKey starts the quoted product name on an ifconfig agent line. +const agentDescriptionKey = `desc:"` + +// parseIfconfigAgent returns the owner of a tunnel from "ifconfig -v utunN" +// output. The owner is the description of the first agent line, and a tunnel that +// no network extension owns has no agent line, so the result is "". +func parseIfconfigAgent(r io.Reader) string { + if r == nil { + return "" + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if !strings.HasPrefix(line, "agent ") { + continue + } + _, quoted, ok := strings.Cut(line, agentDescriptionKey) + if !ok { + continue + } + if description, _, ok := strings.Cut(quoted, `"`); ok && description != "" { + return description + } + } + // An owner that the read did not reach is not an owner that does not exist. + return "" +} diff --git a/cmd/cli/tunnel_owner_test.go b/cmd/cli/tunnel_owner_test.go new file mode 100644 index 0000000..76ba623 --- /dev/null +++ b/cmd/cli/tunnel_owner_test.go @@ -0,0 +1,83 @@ +package cli + +import ( + "strings" + "testing" +) + +// Captured from "ifconfig -v utunN" on macOS. A tunnel that a network extension +// owns carries an agent line with the product name in desc. +const ifconfigWindscribeTunnel = `utun4: flags=8051 mtu 1420 + index 22 + eflags=1002080 + options=6460 + inet 100.96.0.2 --> 100.96.0.2 netmask 0xffffffff + nd6 options=201 + agent domain:NetworkExtension type:NetworkExtension flags:0xf desc:"Windscribe VPN" + agent domain:Multipath type:Path Source flags:0x3 desc:"Multipath path manager" + netif: 6d1e1b2a-4c55-4f0b-9f0a-1d2e3f4a5b6c + scheduler: FQ_CODEL + link quality: 100 (good) +` + +const ifconfigTailscaleTunnel = `utun6: flags=8051 mtu 1280 + index 24 + agent domain:NetworkExtension type:NetworkExtension flags:0xf desc:"Tailscale Tunnel" + nd6 options=201 +` + +const ifconfigSystemTunnel = `utun0: flags=8051 mtu 1500 + index 16 + eflags=1002080 + inet6 fe80::e1a2:3b4c:5d6e:7f80%utun0 prefixlen 64 scopeid 0x10 + nd6 options=201 + scheduler: FQ_CODEL +` + +func Test_parseIfconfigAgent(t *testing.T) { + tests := []struct { + name string + output string + want string + }{ + { + name: "a VPN network extension owns the tunnel", + output: ifconfigWindscribeTunnel, + want: "Windscribe VPN", + }, + { + name: "a mesh client owns the tunnel", + output: ifconfigTailscaleTunnel, + want: "Tailscale Tunnel", + }, + { + name: "no agent line", + output: ifconfigSystemTunnel, + }, + { + name: "no output", + output: "", + }, + { + name: "an agent line without a description", + output: "\tagent domain:Multipath type:Path Source flags:0x3\n\tagent domain:NetworkExtension type:NetworkExtension flags:0xf desc:\"Windscribe VPN\"\n", + want: "Windscribe VPN", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := parseIfconfigAgent(strings.NewReader(tc.output)); got != tc.want { + t.Fatalf("parseIfconfigAgent() = %q, want %q", got, tc.want) + } + }) + } +} + +// Test_parseIfconfigAgentRejectsAnOversizedLine covers a truncated read. An +// owner that the reader did not reach is not an owner that does not exist. +func Test_parseIfconfigAgentRejectsAnOversizedLine(t *testing.T) { + output := strings.Repeat("a", 70<<10) + "\n" + ifconfigWindscribeTunnel + if got := parseIfconfigAgent(strings.NewReader(output)); got != "" { + t.Fatalf("tunnel owner of a truncated read = %q, want an empty owner", got) + } +} diff --git a/cmd/cli/upstream_monitor.go b/cmd/cli/upstream_monitor.go index 432d611..6d1e62f 100644 --- a/cmd/cli/upstream_monitor.go +++ b/cmd/cli/upstream_monitor.go @@ -230,6 +230,44 @@ func (um *upstreamMonitor) noteSuccess(upstream string) { um.markUp(upstream, "recovered") } +// countDown reports how many upstreams are down at this time. +func (um *upstreamMonitor) countDown() int { + um.mu.RLock() + defer um.mu.RUnlock() + return len(um.down) +} + +// countDownExcept reports how many upstreams are down at this time, without +// the ones that skip accepts. The grade of the query path leaves out the +// resolvers that only an endpoint on the organization network can reach. +func (um *upstreamMonitor) countDownExcept(skip func(string) bool) int { + if skip == nil { + return um.countDown() + } + um.mu.RLock() + defer um.mu.RUnlock() + count := 0 + for upstream := range um.down { + if skip(upstream) { + continue + } + count++ + } + return count +} + +// downFor reports how long an upstream has been down. An upstream that is up +// gives zero, so a caller logs the field without a branch. +func (um *upstreamMonitor) downFor(upstream string) time.Duration { + um.mu.RLock() + defer um.mu.RUnlock() + state, down := um.down[upstream] + if !down { + return 0 + } + return um.now().Sub(state.since) +} + // isDown reports whether the given upstream is being marked as down. func (um *upstreamMonitor) isDown(upstream string) bool { um.mu.Lock() diff --git a/cmd/cli/upstream_monitor_test.go b/cmd/cli/upstream_monitor_test.go index c45b4e2..68780d8 100644 --- a/cmd/cli/upstream_monitor_test.go +++ b/cmd/cli/upstream_monitor_test.go @@ -229,3 +229,58 @@ func Test_upstreamMonitorKeepsACustomNameOutOfTheJournal(t *testing.T) { } } } + +func Test_upstreamMonitorCountDown(t *testing.T) { + captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}) + if got := um.countDown(); got != 0 { + t.Fatalf("countDown on a new monitor = %d, want 0", got) + } + um.mu.Lock() + um.markDown(upstreamPrefix+"0", 3, "immediate") + um.markDown(upstreamPrefix+"1", 3, "immediate") + um.mu.Unlock() + if got := um.countDown(); got != 2 { + t.Fatalf("countDown with two down upstreams = %d, want 2", got) + } +} + +func Test_upstreamMonitorDownFor(t *testing.T) { + const upstream = upstreamPrefix + "0" + um, _, now := downMonitor(t, upstream) + + *now = now.Add(2500 * time.Millisecond) + + if got := um.downFor(upstream); got != 2500*time.Millisecond { + t.Fatalf("downFor a down upstream = %s, want 2.5s", got) + } + if got := um.downFor(upstreamPrefix + "9"); got != 0 { + t.Fatalf("downFor an unknown upstream = %s, want 0", got) + } + + um.reset(upstream) + + if got := um.downFor(upstream); got != 0 { + t.Fatalf("downFor an upstream that is up = %s, want 0", got) + } +} + +func Test_upstreamMonitorCountDownExcept(t *testing.T) { + captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}) + um.mu.Lock() + um.markDown(upstreamPrefix+"0", 3, "immediate") + um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"1", 3, "immediate") + um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"2", 3, "immediate") + um.mu.Unlock() + + if got := um.countDown(); got != 3 { + t.Fatalf("countDown = %d, want 3", got) + } + if got := um.countDownExcept(isInternalDomainUpstream); got != 1 { + t.Fatalf("countDownExcept(isInternalDomainUpstream) = %d, want 1", got) + } + if got := um.countDownExcept(nil); got != 3 { + t.Fatalf("countDownExcept(nil) = %d, want 3", got) + } +} diff --git a/cmd/cli/vpn_dns.go b/cmd/cli/vpn_dns.go index 691cf17..2c8516a 100644 --- a/cmd/cli/vpn_dns.go +++ b/cmd/cli/vpn_dns.go @@ -2,6 +2,7 @@ package cli import ( "context" + "fmt" "net" "runtime" "strings" @@ -60,10 +61,16 @@ type vpnDNSManager struct { refreshRunning bool refreshPending bool discoveryMu sync.Mutex + // repeats holds the last refresh summary. The manager has no prog, and a + // network change storm runs one refresh per delta. + repeats repeatLogger // Called when VPN DNS server list changes, to update intercept exemptions. onServersChanged vpnDNSExemptFunc } +// vpnDNSRefreshKey names the repeat counter of the refresh summary. +const vpnDNSRefreshKey = "vpn_dns_refresh" + // newVPNDNSManager creates a new manager. Only call when dnsIntercept is active. // exemptFunc is called whenever VPN DNS servers are discovered/changed, to update // the OS-level intercept rules to permit ctrld's outbound queries to those IPs. @@ -220,8 +227,11 @@ func (m *vpnDNSManager) refreshOnce(guardAgainstNoNameservers bool) { } m.domainlessServers = domainlessServers - logger.Debug().Msgf("VPN DNS refresh completed: %d configs, %d routes, %d domainless servers, %d unique exemptions", + summary := fmt.Sprintf("%d configs, %d routes, %d domainless servers, %d unique exemptions", len(m.configs), len(m.routes), len(m.domainlessServers), len(exemptions)) + if changed, repeats := m.repeats.changed(vpnDNSRefreshKey, summary); changed { + logger.Debug().Uint64("repeats", repeats).Msgf("VPN DNS refresh completed: %s", summary) + } // Update intercept rules only when desired exemptions differ from the last // successfully applied set. Failed PF/WFP callbacks remain retryable on the diff --git a/cmd/cli/vpn_dns_test.go b/cmd/cli/vpn_dns_test.go index 94cf9bf..ebe004e 100644 --- a/cmd/cli/vpn_dns_test.go +++ b/cmd/cli/vpn_dns_test.go @@ -3,6 +3,7 @@ package cli import ( "context" "errors" + "strings" "sync" "sync/atomic" "testing" @@ -296,3 +297,53 @@ func TestVPNDNSFullAndRouteOnlyDiscoveryAreSerialized(t *testing.T) { t.Fatalf("serialized exemption updates = %v, want old then new", exemptionUpdates) } } + +// eventsWithPrefix returns the captured events whose message starts with the +// prefix. Several lines of this module carry a value in the message, so no +// constant matches the whole of it. +func eventsWithPrefix(t *testing.T, logs *syncBuffer, prefix string) []map[string]any { + t.Helper() + var matched []map[string]any + for _, event := range jsonLogEvents(t, logs, "") { + message, _ := event["message"].(string) + if strings.HasPrefix(message, prefix) { + matched = append(matched, event) + } + } + return matched +} + +// TestVPNDNSRefreshSummaryLogsOnChangeOnly covers the refresh summary. A network +// change storm runs the refresh every few seconds, and the same summary in every +// one of those lines buries the refresh that changed something. +func TestVPNDNSRefreshSummaryLogsOnChangeOnly(t *testing.T) { + logs := captureDebugMainLog(t) + m := newVPNDNSManager(nil) + configs := []ctrld.VPNDNSConfig{{ + InterfaceName: "utun4", + Servers: []string{"10.0.0.1"}, + Domains: []string{"corp.internal"}, + }} + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { return configs } + + m.Refresh(true) + m.Refresh(true) + + summaries := eventsWithPrefix(t, logs, "VPN DNS refresh completed") + if len(summaries) != 1 { + t.Fatalf("got %d refresh summaries for two identical results, want 1", len(summaries)) + } + wantField(t, summaries[0], "repeats", float64(0)) + + configs = append(configs, ctrld.VPNDNSConfig{ + InterfaceName: "utun6", + Servers: []string{"10.0.0.2"}, + }) + m.Refresh(true) + + summaries = eventsWithPrefix(t, logs, "VPN DNS refresh completed") + if len(summaries) != 2 { + t.Fatalf("got %d refresh summaries after the result changed, want 2", len(summaries)) + } + wantField(t, summaries[1], "repeats", float64(1)) +} diff --git a/docs/dns-intercept-mode.md b/docs/dns-intercept-mode.md index 29463de..cb362f0 100644 --- a/docs/dns-intercept-mode.md +++ b/docs/dns-intercept-mode.md @@ -235,6 +235,10 @@ DNS intercept: pf anchor "com.controld.ctrld" active with 3 rules DNS intercept: pf redirect active — all outbound DNS (port 53) redirected to 127.0.0.1:53 ``` +DNS intercept mode on macOS also runs the wake detector. That detector reports a `Host woke` journal event with the source `detector` and the length of the sleep. +Outside DNS intercept mode the netmon delta is the only wake source, and it reports no length of its own. +The events of the journal are listed in [Network-recovery diagnostics](network-recovery-diagnostics.md#journal-events). + ## Troubleshooting ### Windows @@ -306,12 +310,14 @@ VPN apps commonly add rules like `pass out quick on ipsec0 inet all` that match ### 3. Dynamic Tunnel Interface Detection -The network change monitor (`validInterfacesMap()`) only tracks physical hardware ports (en0, bridge0, etc.) — it doesn't see tunnel interfaces (utun*, ipsec*, etc.) created by VPN software. When a VPN connects and creates a new interface (e.g., utun420 for WireGuard), ctrld detects this through a separate tunnel interface change check and rebuilds the pf anchor to include explicit intercept rules for the new interface. This runs on every network change event, even if no physical interface changed. +The network change monitor (`validInterfacesMap()`) only tracks physical hardware ports (en0, bridge0, etc.) — it doesn't see tunnel interfaces (utun*, ipsec*, etc.) created by VPN software. When a VPN connects and creates a new interface (e.g., utun420 for WireGuard), ctrld detects this through a separate tunnel interface change check and rebuilds the pf anchor to include explicit intercept rules for the new interface. This runs on every network change event that is not noise, even if no physical interface changed. A delta that touches AirDrop or virtual adapters alone is noise, and ctrld skips the handler for it, so no tunnel discovery runs. ### 4. pf Watchdog + Network Change Hooks A background watchdog (30s interval) plus immediate checks on network change events detect when another program replaces the entire pf ruleset (e.g., Windscribe's `pfctl -f /etc/pf.conf`). When detected, ctrld rebuilds its anchor with up-to-date tunnel interface rules and re-injects the anchor reference at the top of the ruleset. A 2-second delayed re-check catches race conditions where the other program clears rules slightly after the network event. +A noise delta starts no immediate check and no delayed re-check. The watchdog keeps its own 30 s cadence, so it still restores a missing anchor during a delta storm. + ### 4a. Active Interception Probe (pf Translation State Corruption) Programs like Parallels Desktop reload `/etc/pf.conf` when creating/destroying virtual network interfaces (bridge100, vmenet0). This can corrupt pf's internal translation engine — rdr rules survive in text form but stop evaluating, causing DNS interception to silently fail while the watchdog reports "intact." @@ -639,6 +645,8 @@ Network change detected (netmon callback) └─ (same as 2s — catches slower VPN teardowns) ``` +A noise delta skips this whole chain. ctrld runs no pf read, no tunnel discovery, no OS resolver read, and no VPN DNS refresh for it. The noise class holds the AirDrop interfaces (`awdl*`, `llw*`) and the virtual adapters. The pf watchdog keeps its own 30 s cadence, so it still tests the rules during a delta storm. + ### VPN Connect Sequence 1. VPN creates tunnel interface (e.g., utun420) diff --git a/docs/network-recovery-diagnostics.md b/docs/network-recovery-diagnostics.md index 856c679..d676ae1 100644 --- a/docs/network-recovery-diagnostics.md +++ b/docs/network-recovery-diagnostics.md @@ -8,10 +8,139 @@ Every `ctrld log send` upload holds the whole journal, so a reproduction does no Match timestamps and IDs. The last retained error does not describe the current network in every case. Each upload starts with a header line that names the build, the intercept mode, and the network at send time. +## Journal events + +These events describe the host network. Each one carries `journal=true`, so it stays on disk after a rotation and after a restart. +netmon reports every network change as a delta, which holds the state before the change and the state after it. +The table gives the trigger, the level, and the fields of each event. + +| Event | Trigger | Level | Fields | +|---|---|---|---| +| `Network snapshot` | Start, an accepted transition, a recovery begin, and a recovery end. Two limits hold some of them back | info | `trigger` (`start`, `transition`, `recovery_begin`, `recovery_end`), `network` | +| `Network interface changed` | One event per changed interface of a delta that is not noise. A mixed delta gives an event for the interfaces outside the noise class only | info | `transition_id`, `interface`, `action`, `class`, `hardware_port`, `service`, `ips_before`, `ips_after`, `flags`, `mtu`, `is_default_route` | +| `Network transition` | The end of every network callback that is not noise | info, debug for the outcomes `ignored` and `snapshot_superseded` | `transition_id`, `outcome`, `is_major_change`, `changed`, `interface`, `changed_interfaces`, `time_jumped`, `default_route_before`, `default_route_after`, `have_v4_before`, `have_v4_after`, `have_v6_before`, `have_v6_after`, `source_ipv4_before`, `source_ipv4_after`, `source_ipv6_before`, `source_ipv6_after`, `source_snapshot_available` | +| `Network delta noise` | Each noise delta, and one journal line per 10 minutes of a storm | debug, info for the journal line | `interfaces`, `count`, `first_at`, `last_at` | +| `Host woke` | One event per wake: a time jump in a netmon delta, or the macOS wake detector, which runs in DNS intercept mode only | info | `gap_known`, `gap_ms` (only when `gap_known` is true), `source` (`netmon`, `detector`), `default_route`, `interfaces_up` | +| `OS resolver set changed` | A nameserver read that differs from the read before | info | `source` (`scutil`, `dhcp`, `resolv.conf`, `unknown`), `before`, `after`, `default_route`, `reason` (`start`, `recovery`, `delayed_recheck`, `wake_probe`, `vpn_settle`, `unspecified`) | +| `Reinitialized OS resolver with nameservers: …` | ctrld read the nameservers of the system again and built the OS resolver again | info | none, the nameservers are part of the message | +| `DNS configuration changed` | A `scutil --dns` poll that finds a changed resolver, one event per resolver (macOS) | info | `nameservers`, `if_index`, `interface`, `scoped`, `action` (`added`, `changed`, `removed`), `flags`, `search_domain_count`, `order`, `reachable` | +| `Recovery begin` | The start of every recovery | info | `transition_id`, `recovery_generation`, `recovery_reason`, `intercept` | +| `Recovery end` | The end of every recovery | info, warn for a canceled recovery and for a recovery with a failure | `transition_id`, `recovery_generation`, `recovery_reason`, `outcome`, `had_failure`, `duration_ms`, `recovered_upstream`, `dhcp_servers`, `dhcp_server_count`, `intercept_target_action`, `bypass_active` | +| `Upstream … recovered …` | An upstream that was down answers again | info | `down_for_ms` | +| `Recovery canceled with no successor …` | A recovery stops and no other recovery follows it | info | none | +| `DNS intercept recovery: enabling DHCP bypass …` | ctrld sends the queries of a recovery to the OS resolver | info | none | +| `DNS intercept recovery complete: disabling DHCP bypass …` | The recovery ends and the normal query path returns | info | none | +| `DNS intercept recovery: found DHCP nameservers: …` | ctrld read the DHCP nameservers of the service | info | `dhcp_servers` | +| `DNS intercept recovery: no DHCP nameservers found` | The service gave no DHCP nameserver | warn | none | +| `intercept DNS target: service … set …` | ctrld wrote the loopback target into a network service | warn | `service`, `target`, `reason` | +| `intercept DNS target: removed …` | ctrld took the loopback target out of the service | info | `service`, `target`, `reason` | +| `intercept DNS target: … DNS changed externally …` | Another program changed the DNS of the service, so the target of ctrld is gone (macOS) | info | `service`, `target`, `reason` (`external_change`) | +| `dns64: discovered NAT64 prefix …` | A NAT64 prefix that differs from the result before | info | none | +| `dns64: no NAT64 prefix present …` | A no-prefix result that differs from the result before | info | none | +| `DNS intercept: interface appeared/disappeared …` | An interface appeared or disappeared, and the probe monitor starts | info | `interface`, `class`, `action` | +| `PF anchor list changed` | A missing anchor, a restored anchor, a stabilization start, and a stabilization end (macOS) | warn for the reason `missing`, info for the rest | `reason`, `anchors`, `anchors_known`, `pf_enabled`, `pf_since` | +| `Tunnel interface changed` | The set of tunnel interfaces changes (macOS). A retry of the same change writes no second event | info | `added`, `removed`, `owner` | +| `DNS intercept: VPN connecting …` | A new tunnel interface starts the stabilization mode (macOS) | info | none | +| `DNS intercept: pf stable for …` | The pf ruleset stops changing and stabilization ends (macOS) | info | none | +| `Query health` | A change of the health class that holds for two evaluations, and a heartbeat every 15 minutes | info | `class`, `window_s`, `queries`, `failed_queries`, `cache_hits`, `failures_by_class`, `upstreams_down`, `bypass_active`, `cache_hit_ratio` | + +No event in this catalog holds a query name, an upstream URL, or a token. +The journal does hold network addresses. It holds the LAN and VPN resolver addresses, the gateways, the addresses of the host, and the DHCP servers of a service. +It holds no search suffix. `DNS configuration changed` counts the suffixes in `search_domain_count` and logs the count alone. +The field `owner` of `Tunnel interface changed` names the VPN product that owns the tunnel. On Windows the field `hardware_port` names the VPN product of an adapter in the same way. +The Wi-Fi network name is never logged. On macOS 14 and later, only a process with the Location Services permission can read it, and a daemon cannot request that permission. + +### The network object + +The `network` object describes the host network at one moment. +The `Network snapshot` event and the header line of every log file render the same object, with these fields: + +- Routes: `default_route_v4`, `default_route_v6`, `gateway_v4`, `gateway_v6` +- Families: `have_v4`, `have_v6` +- Interfaces: `interfaces`, an array of `name`, `class`, `up`, `ips`, `hardware_port`, `service`, and the count `interfaces_omitted` +- Resolvers: `resolvers`, the host and the port of each OS nameserver +- Source addresses: `source_ipv4`, `source_ipv6` +- Intercept state: `intercept_target`, `dns_less_target`, `bypass_active`, `recovery_running`, `pf_stabilizing` +- Link: `link_type`, `tethered`, `clat_present`, `nat64_prefix` + +The field `default_route_interface` is gone. Read `default_route_v4` and `default_route_v6` in its place. +The two routes and the two gateways come from the route table of macOS, one route per address family. A tunnel that owns the default route has no next hop, so its route names the tunnel and its gateway stays empty. +Windows and Linux read no route table in this release. They fill `default_route_v4` from the network monitor and leave `default_route_v6`, `gateway_v4`, and `gateway_v6` empty. + +`link_type` is one of `wifi`, `ethernet`, `usb_tether`, `tunnel`, and `unknown`. +macOS reads it from the name of the hardware port. Linux reads `wifi`, `ethernet`, `usb_tether`, and `tunnel` from the kernel. Windows names a tunnel and leaves the other links unknown. +The field `service` names a macOS network service. It stays empty on Windows and on Linux. +`tethered` is true on every platform when the interface of the default route holds an address of the range 172.20.10.0/28. +It is also true when the default gateway is in that range, and when the hardware port of the default route is `iPhone USB`. + +The list `interfaces` holds the interfaces that are up or hold an address, and it stops at 32 entries. +`interfaces_omitted` counts the interfaces that the list leaves out, the ones over the limit included. +A container host holds hundreds of interfaces, and a list of all of them fills the journal. + +ctrld does not write a snapshot for every trigger. A snapshot equal to the snapshot written last reaches no journal. +Every other snapshot waits 60 s after the snapshot written last. A new value of `default_route_v4`, `default_route_v6`, or `resolvers` passes that wait. +The trigger `start` always writes. + +### Noise class + +A noise delta changes interfaces of the noise class only, and it does not move the default route. +The noise class holds `awdl*` and `llw*` on macOS. +On Linux it holds the names `docker*`, `veth*`, `virbr*`, and `br-*`, plus the devices of the tun and tap types. +On Windows it holds the virtual adapters without a connector. A Windows adapter whose description names TAP, Wintun, WireGuard, or VPN is a tunnel, not a noise interface. +On Linux and on Windows, an interface that holds a global address on either side of the delta is never noise. +A delta is also not noise when it flips `have_v4` or `have_v6`, or when it carries a time jump. +A noise delta runs no reconcile work. +ctrld starts no `scutil` process, reads no PF rule, discovers no tunnel, refreshes no VPN DNS, and schedules no delayed recheck for it. +A delta that touches a hardware port and a noise interface in the same callback is not noise. +The PF watchdog keeps its own 30 s timer, so it still tests the PF rules during a delta storm. + +Each noise delta logs one debug line. The first noise delta of a storm also writes a journal line with `count` 1. +While the storm lasts, one journal line of the same name arrives every 10 minutes, with the count of that window. +AirDrop in the Finder is the common source of such a storm on macOS. + +### DNS configuration poll (macOS) + +ctrld polls `scutil --dns` on macOS only. The first poll runs at start. +The poll runs every 15 s for 10 minutes after a network change, a wake, or the end of a recovery. +Each of these three events starts the fast cadence at once, even while the poll waits. +At rest the poll runs every 5 minutes. Each changed resolver of a poll gives one `DNS configuration changed` event. + +### Change-only lines + +These lines log at debug level on change only. Each one carries `repeats`, the number of equal results before it: + +- `Got system nameservers` +- `Final available nameservers` +- `pf anchor intact` +- `discovered active tunnel interfaces` +- `VPN DNS refresh completed` + +### PF anchor list (macOS) + +A `PF anchor list changed` event reaches the journal when its reason, its anchor list, or `pf_enabled` differs from the event before it. +The field `pf_since` is no part of that test, because the uptime of pf grows between two reads of one state. +A failed read of the anchor names gives `anchors_known` false and no anchor list. + +### Host woke + +ctrld writes one `Host woke` event per wake. Two sources that report the same wake inside 30 s give one event. +The macOS wake detector runs in DNS intercept mode only. Outside that mode, and on Windows and Linux, the netmon delta is the only source. +The netmon source knows no gap, so its report waits 8 s for the detector. A detector report inside that hold gives the event, with `gap_known` true and the length of the sleep in `gap_ms`. Without a detector report, the netmon report goes out with `gap_known` false and no `gap_ms`. Every later report of the same wake is dropped. + +### Query health + +ctrld grades the query path over a window of 15 minutes and reads the window every minute. +Each failed client query counts one failure, whatever number of upstreams that query tried. +`failures_by_class` keeps one count per failure class. The class `internal_domain` holds the failures of an Internal Domain resolver. +The grade leaves the class `internal_domain` out, and `upstreams_down` counts no Internal Domain resolver. +A laptop away from the company network cannot reach these resolvers. Without this rule the grade stays degraded for the whole day. +A change of the class reports after it holds for two reads of the window. + ## Transitions and recovery -`Network transition` debug events include a `transition_id`, affected interface, default route, and stored source addresses before and after the callback. -The outcome identifies an accepted, ignored, superseded, or unusable change. +`Network transition` events include a `transition_id`, affected interface, default route, and stored source addresses before and after the callback. +The outcome identifies an accepted, ignored, superseded, snapshot_superseded, or unusable change. +A callback whose state read a newer snapshot gives the outcome `snapshot_superseded` at debug level with `transition_id` 0, and it gives no interface event. Newer accepted changes supersede pending recovery. Ignored changes do not cancel accepted recovery. Source commits validate addresses against the current network state. Netmon does not order minor callbacks within one major cache epoch. @@ -22,8 +151,9 @@ This read does not change host DNS. A failed read defers source writes, but reco The warning gives the reason. Every warning enters the journal, so it stays on disk after a debug rotation and after a restart. `Recovery begin` and `Recovery end` connect the transition to a `recovery_generation`, reason, outcome, and elapsed time. Only the first upstream failure in each recovery produces a bounded summary. -If a recovery encounters a failure, its final outcome also enters the journal. -Normal successful recovery stays at debug level. +Every recovery outcome enters the journal. +A canceled recovery and a recovery with a failure log at warn level. +For the outcome `superseded`, `intercept_target_action` is `unchanged` and `bypass_active` is false. The recovery that follows owns both of them. These summaries exclude upstream URLs and queried names. Each PF probe records the recovery generation at its start. diff --git a/docs/pf-dns-intercept.md b/docs/pf-dns-intercept.md index 488f56e..c241d5e 100644 --- a/docs/pf-dns-intercept.md +++ b/docs/pf-dns-intercept.md @@ -289,6 +289,13 @@ The monitor probes with exponential backoff (0, 0.5, 1, 2, 4s) to win the race against async pf reloads. Only one monitor runs at a time (singleton). The watchdog also runs the probe every 30s as a safety net. +A network delta that touches AirDrop or virtual adapters alone is noise. The +handler returns at once for it, so no probe monitor starts and no pf read runs. +A delta that moves the default route, that flips the v4 or the v6 family, or +that carries a time jump is not noise, whatever its interfaces are. +The watchdog keeps its own 30 s cadence, so it still tests the rules during a +delta storm. + The full pf reload is VPN-safe: it reassembles from `pfctl -sr` + `pfctl -sn` (the current running state), preserving all existing anchors and rules. diff --git a/docs/runtime-internal-logging.md b/docs/runtime-internal-logging.md index 8a52cad..26fad07 100644 --- a/docs/runtime-internal-logging.md +++ b/docs/runtime-internal-logging.md @@ -45,12 +45,25 @@ The message of the line is `Log header`. The line carries these fields: - Mode: `intercept_mode`, `listeners`, `upstream_count`, and `upstream_types` - Identity: `resolver_uid` - Files: `log_files` -- Network: `network` with `default_route_interface`, `have_v4`, `have_v6`, and `interfaces` +- Network: `network`, the network snapshot of the host The header holds no provisioning token, no upstream endpoint URL, and no query name. `upstream_types` holds the type words, for example `doh` or `os`. `listeners` holds the listen addresses. +The `network` object holds the routes, the gateways, the interfaces, the resolvers, the link type, and the intercept state. It has no `default_route_interface` field. The runbook [Network-recovery diagnostics](network-recovery-diagnostics.md) lists every field of the object. + After each rotation ctrld logs one `Log rotated` event at info level, with the fields `file`, `bytes_written`, `first_event_at`, `last_event_at`, and `backups`. This event is a journal event, so it lands in the debug stream and in the journal. +## Network state events + +The journal holds the events that describe the host network. `Network snapshot` renders the `network` object at start, at each accepted transition, and at each begin and end of a recovery. +A snapshot equal to the snapshot written last reaches no journal, and one snapshot per 60 s is the limit. A new default route and a new resolver set pass that limit. `Network interface changed`, `Network transition`, `Host woke`, `OS resolver set changed`, and `DNS configuration changed` name each change of the network. `Recovery begin`, `Recovery end`, `PF anchor list changed`, and `Tunnel interface changed` name the repair work. + +AirDrop and virtual adapters produce a storm of network callbacks. ctrld puts these interfaces in a noise class. A delta that touches noise interfaces alone skips the handler. The first delta of a storm writes one `Network delta noise` journal line, and one more line covers each 10 minutes of the storm. + +`Query health` grades the query path over a window of 15 minutes. It logs at each change between the classes `healthy`, `degraded`, and `failing`, and once every 15 minutes as a heartbeat. A class change reports after it holds for two reads of the window. + +The runbook [Network-recovery diagnostics](network-recovery-diagnostics.md) lists every event with its trigger, its level, and its fields. + ## Sampling of per-query errors A broken network produces one error line per query. ctrld bounds this flood for each error class and each upstream. The first 5 lines in a 60 s window stay at error level. Later lines in the same window drop to debug level, so the debug stream stays complete. A run with `-v` holds every stream at info, and the debug stream then loses these lines. @@ -79,7 +92,7 @@ Both keys also apply to the `log_path` file, which rotates the same way. The jou - The marker `=== LOG_END ===` - Every journal file, oldest first -The `network` object of the send-time header comes from a fresh read of the interfaces and the default route. `ctrld log send --full` sends every debug file in place of the newest 10 MB. The reported size equals the number of bytes uploaded. +The `network` object of the send-time header comes from a fresh read of the interfaces and of the route table. The resolvers come from the list that ctrld stored at its last resolver initialization, not from a fresh read. `ctrld log send --full` sends every debug file in place of the newest 10 MB. The reported size equals the number of bytes uploaded. `ctrld log view` prints the same composition on the terminal and takes the same `--full` flag. ctrld accepts one upload per minute. The body is plain text without gzip. diff --git a/log.go b/log.go index 14c82e8..28ac770 100644 --- a/log.go +++ b/log.go @@ -32,3 +32,12 @@ func Log(ctx context.Context, e *zerolog.Event, format string, v ...any) { return fmt.Sprintf("[%s] %s", id, fmt.Sprintf(format, v...)) }) } + +// JournalField marks an event for the retained journal stream. +const JournalField = "journal" + +// Journal marks an event for the retained journal stream. The journal writer +// keeps every line that carries the field, whatever the level of the line is. +func Journal(e *zerolog.Event) *zerolog.Event { + return e.Bool(JournalField, true) +} diff --git a/net.go b/net.go index 7bbf54b..81a7557 100644 --- a/net.go +++ b/net.go @@ -6,8 +6,6 @@ import ( "sync/atomic" "time" - "tailscale.com/net/netmon" - ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) @@ -25,26 +23,16 @@ func HasIPv6() bool { val := ctrldnet.IPv6Available(ctx) ipv6Available.Store(val) ProxyLogger.Load().Debug().Msgf("ipv6 availability: %v", val) - mon, err := netmon.New(func(format string, args ...any) {}) - if err != nil { - ProxyLogger.Load().Debug().Err(err).Msg("failed to monitor IPv6 state") - return - } - mon.RegisterChangeCallback(func(delta *netmon.ChangeDelta) { - old := ipv6Available.Load() - cur := delta.Monitor.InterfaceState().HaveV6 - if old != cur { - ProxyLogger.Load().Warn().Msgf("ipv6 availability changed, old: %v, new: %v", old, cur) - } else { - ProxyLogger.Load().Debug().Msg("ipv6 availability does not changed") - } - ipv6Available.Store(cur) - }) - mon.Start() }) return ipv6Available.Load() } +// SetIPv6Available stores the IPv6 state that the network monitor reports, so +// one monitor serves the whole process. +func SetIPv6Available(v bool) { + ipv6Available.Store(v) +} + // DisableIPv6 marks IPv6 as unavailable if enabled. func DisableIPv6() { if ipv6Available.CompareAndSwap(true, false) { diff --git a/net_ipv6_test.go b/net_ipv6_test.go new file mode 100644 index 0000000..37ca2ea --- /dev/null +++ b/net_ipv6_test.go @@ -0,0 +1,37 @@ +package ctrld + +import ( + "sync" + "testing" +) + +// resetIPv6Probe clears the one-time IPv6 probe. Test only: the process runs +// the probe once, so a test cannot see it without this reset. +func resetIPv6Probe(t *testing.T) { + t.Helper() + reset := func() { + hasIPv6Once = sync.Once{} + ipv6Available.Store(false) + } + reset() + t.Cleanup(reset) +} + +func TestHasIPv6KeepsTheProbeThenTakesTheStoredFlag(t *testing.T) { + resetIPv6Probe(t) + + probed := HasIPv6() + if got := HasIPv6(); got != probed { + t.Fatalf("HasIPv6 = %v on the second call, want the probe value %v", got, probed) + } + + SetIPv6Available(!probed) + if got := HasIPv6(); got == probed { + t.Fatalf("HasIPv6 = %v after SetIPv6Available(%v), want the stored value", got, !probed) + } + + SetIPv6Available(false) + if HasIPv6() { + t.Fatal("HasIPv6 = true after SetIPv6Available(false), want false") + } +} diff --git a/resolver.go b/resolver.go index 08353a3..959d57a 100644 --- a/resolver.go +++ b/resolver.go @@ -65,12 +65,64 @@ func init() { } var ( - resolverMutex sync.Mutex - or *osResolver + resolverMutex sync.Mutex + or *osResolver + // osResolverPtr holds the same resolver as or, for readers alone. An + // initialization holds resolverMutex across a scutil read of several + // seconds, and a log header render must not wait for it. + osResolverPtr atomic.Pointer[osResolver] defaultLocalIPv4 atomic.Value // holds net.IP (IPv4) defaultLocalIPv6 atomic.Value // holds net.IP (IPv6) ) +// storeOsResolver keeps the resolver of the process and the copy that readers +// take. The caller holds resolverMutex. +func storeOsResolver(resolver *osResolver) { + or = resolver + osResolverPtr.Store(resolver) +} + +// NameserversFn reads the nameservers of the system. A test of this package +// or of a package that drives the OS resolver replaces it to stay off the host. +var NameserversFn = nameservers + +// osResolverReasonUnspecified marks a resolver read that no caller explained. +const osResolverReasonUnspecified = "unspecified" + +// nameserverReads remembers the last nameserver list of one read. The system +// reports the same list many times, and only a change is news. +type nameserverReads struct { + seen bool + last []string + repeats int +} + +// record stores a new read. It reports whether the list changed, how many +// reads repeated the list before it, and the list that this read replaces. +func (n *nameserverReads) record(list []string) (changed bool, repeats int, before []string) { + if n.seen && slices.Equal(n.last, list) { + n.repeats++ + return false, n.repeats, n.last + } + before, repeats = n.last, n.repeats + n.seen = true + n.last = slices.Clone(list) + n.repeats = 0 + return true, repeats, before +} + +// osResolverLogState keeps the last nameserver reads and the reason of the +// initialization that runs. The read sits deep in the call chain, so the +// reason cannot travel as a parameter. +type osResolverLogState struct { + mu sync.Mutex + system nameserverReads + final nameserverReads + reason string +} + +var osResolverLog = osResolverLogState{reason: osResolverReasonUnspecified} + func newLocalResolver() Resolver { var nss []string for _, addr := range Rfc1918Addresses() { @@ -117,9 +169,8 @@ func availableNameservers() []string { "Added local IP to OS resolverexclusion map: %s", ipStr) } - systemNameservers := nameservers() - Log(context.Background(), logger.Debug(), - "Got system nameservers: %v", systemNameservers) + systemNameservers := NameserversFn() + logSystemNameservers(systemNameservers) for _, ns := range systemNameservers { if _, ok := machineIPsMap[ns]; ok { @@ -132,19 +183,89 @@ func availableNameservers() []string { "Added non-local nameserver: %s", ns) } - Log(context.Background(), logger.Debug(), - "Final available nameservers: %v", nss) + logFinalNameservers(nss) return nss } +// logSystemNameservers logs the system read at debug level on change only. +func logSystemNameservers(list []string) { + osResolverLog.mu.Lock() + changed, repeats, _ := osResolverLog.system.record(list) + osResolverLog.mu.Unlock() + if !changed { + return + } + logger := *ProxyLogger.Load() + Log(context.Background(), logger.Debug().Int("repeats", repeats), + "Got system nameservers: %v", list) +} + +// logFinalNameservers logs the effective read at debug level on change only, +// and sends each change to the journal, where an outage report needs it. +func logFinalNameservers(list []string) { + osResolverLog.mu.Lock() + changed, repeats, before := osResolverLog.final.record(list) + reason := osResolverLog.reason + osResolverLog.mu.Unlock() + if !changed { + return + } + logger := *ProxyLogger.Load() + Log(context.Background(), logger.Debug().Int("repeats", repeats), + "Final available nameservers: %v", list) + Journal(logger.Info()). + Str("source", osResolverSource()). + Strs("before", before). + Strs("after", list). + Str("default_route", defaultRouteInterfaceName()). + Str("reason", reason). + Msg("OS resolver set changed") +} + +// osResolverSource names the place where the platform holds the nameservers. +func osResolverSource() string { + switch runtime.GOOS { + case "darwin": + return "scutil" + case "windows": + return "dhcp" + case "linux", "dragonfly", "freebsd", "netbsd", "openbsd": + return "resolv.conf" + } + return "unknown" +} + +// defaultRouteInterfaceName returns the interface of the default route, and an +// empty string when the route table does not answer. +func defaultRouteInterfaceName() string { + name, err := netmon.DefaultRouteInterface() + if err != nil { + return "" + } + return name +} + +// setOsResolverReason keeps the reason of the initialization that runs. +func setOsResolverReason(reason string) { + osResolverLog.mu.Lock() + defer osResolverLog.mu.Unlock() + osResolverLog.reason = reason +} + // InitializeOsResolver initializes OS resolver using the current system DNS settings. // It returns the nameservers that is going to be used by the OS resolver. // // It's the caller's responsibility to ensure the system DNS is in a clean state before // calling this function. func InitializeOsResolver(guardAgainstNoNameservers bool) []string { - ns, _ := InitializeOsResolverWithSystemNameservers(guardAgainstNoNameservers) - return ns + return InitializeOsResolverWithReason(guardAgainstNoNameservers, osResolverReasonUnspecified) +} + +// InitializeOsResolverWithReason is InitializeOsResolver with the reason that +// the journal reports when the resolver set changes. +func InitializeOsResolverWithReason(guardAgainstNoNameservers bool, reason string) []string { + effective, _ := InitializeOsResolverWithSystemNameserversReason(guardAgainstNoNameservers, reason) + return effective } // InitializeOsResolverWithSystemNameservers initializes the OS resolver and @@ -152,9 +273,19 @@ func InitializeOsResolver(guardAgainstNoNameservers bool) []string { // discovered from the system. The latter deliberately excludes synthetic // fallbacks added by initializeOsResolver. func InitializeOsResolverWithSystemNameservers(guardAgainstNoNameservers bool) (effective, system []string) { + return InitializeOsResolverWithSystemNameserversReason(guardAgainstNoNameservers, osResolverReasonUnspecified) +} + +// InitializeOsResolverWithSystemNameserversReason is +// InitializeOsResolverWithSystemNameservers with the reason that the journal +// reports when the resolver set changes. +func InitializeOsResolverWithSystemNameserversReason(guardAgainstNoNameservers bool, reason string) (effective, system []string) { resolverMutex.Lock() defer resolverMutex.Unlock() + setOsResolverReason(reason) + defer setOsResolverReason(osResolverReasonUnspecified) + system = availableNameservers() if system == nil { // A non-nil empty slice means discovery completed and found no DNS. @@ -165,7 +296,7 @@ func InitializeOsResolverWithSystemNameservers(guardAgainstNoNameservers bool) ( if skip { return effective, system } - or = newResolverWithNameserver(effective) + storeOsResolver(newResolverWithNameserver(effective)) return effective, system } @@ -230,7 +361,7 @@ func NewResolver(uc *UpstreamConfig) (Resolver, error) { resolverMutex.Lock() if or == nil { ProxyLogger.Load().Debug().Msgf("Initialize new OS resolver") - or = newResolverWithNameserver(defaultNameservers()) + storeOsResolver(newResolverWithNameserver(defaultNameservers())) } resolverMutex.Unlock() return or, nil @@ -269,10 +400,10 @@ type publicResponse struct { // OsResolverNameservers returns the current OS resolver nameservers (host:port format). // Returns nil if the OS resolver has not been initialized. +// The read takes no lock, so a log header render never waits for a resolver +// initialization. func OsResolverNameservers() []string { - resolverMutex.Lock() - r := or - resolverMutex.Unlock() + r := osResolverPtr.Load() if r == nil { return nil } @@ -727,7 +858,7 @@ func initDefaultOsResolver() []string { defer resolverMutex.Unlock() if or == nil { ProxyLogger.Load().Debug().Msgf("Initialize new OS resolver with default nameservers") - or = newResolverWithNameserver(defaultNameservers()) + storeOsResolver(newResolverWithNameserver(defaultNameservers())) } nss := *or.lanServers.Load() nss = append(nss, *or.publicServers.Load()...) diff --git a/resolver_test.go b/resolver_test.go index 1840cb3..674bef3 100644 --- a/resolver_test.go +++ b/resolver_test.go @@ -4,9 +4,13 @@ import ( "context" "crypto/rand" "encoding/hex" + "encoding/json" "errors" "net" "os" + "runtime" + "slices" + "strings" "sync" "sync/atomic" "syscall" @@ -14,6 +18,7 @@ import ( "time" "github.com/miekg/dns" + "github.com/rs/zerolog" ) func Test_osResolver_Resolve(t *testing.T) { @@ -859,3 +864,231 @@ func generateEdns0ServerCookie(clientCookie string) string { } return clientCookie + hex.EncodeToString(cookie) } + +// syncLogBuffer collects the log lines that any goroutine writes. +type syncLogBuffer struct { + mu sync.Mutex + sb strings.Builder +} + +func (b *syncLogBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.Write(p) +} + +func (b *syncLogBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.String() +} + +// captureProxyLog sends the proxy log to a buffer at debug level. It puts the +// logger and the global level back when the test ends. +func captureProxyLog(t *testing.T) *syncLogBuffer { + t.Helper() + buf := &syncLogBuffer{} + logger := zerolog.New(buf).Level(zerolog.DebugLevel) + previousLogger := ProxyLogger.Load() + previousLevel := zerolog.GlobalLevel() + ProxyLogger.Store(&logger) + zerolog.SetGlobalLevel(zerolog.DebugLevel) + t.Cleanup(func() { + ProxyLogger.Store(previousLogger) + zerolog.SetGlobalLevel(previousLevel) + }) + return buf +} + +// logEventsWithPrefix returns the log events whose message starts with prefix. +func logEventsWithPrefix(t *testing.T, logs, prefix string) []map[string]any { + t.Helper() + var events []map[string]any + for _, line := range strings.Split(logs, "\n") { + if strings.TrimSpace(line) == "" { + continue + } + event := map[string]any{} + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("log line is not JSON: %q: %v", line, err) + } + message, _ := event["message"].(string) + if !strings.HasPrefix(message, prefix) { + continue + } + events = append(events, event) + } + return events +} + +// wantLogField fails the test when an event misses a field or holds another value. +func wantLogField(t *testing.T, event map[string]any, field string, want any) { + t.Helper() + got, ok := event[field] + if !ok { + t.Fatalf("log event has no field %q: %v", field, event) + } + if got != want { + t.Fatalf("log event field %q = %v, want %v", field, got, want) + } +} + +// wantLogStrings fails the test when a list field does not hold want. +func wantLogStrings(t *testing.T, event map[string]any, field string, want []string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("log event field %q is not a list: %v", field, event[field]) + } + got := make([]string, 0, len(values)) + for _, value := range values { + text, ok := value.(string) + if !ok { + t.Fatalf("log event field %q holds a value that is not a string: %v", field, value) + } + got = append(got, text) + } + if !slices.Equal(got, want) { + t.Fatalf("log event field %q = %v, want %v", field, got, want) + } +} + +// stubNameservers answers each system nameserver read from lists, in order. +// The last list answers every read after it. +func stubNameservers(t *testing.T, lists ...[]string) { + t.Helper() + previous := NameserversFn + reads := 0 + NameserversFn = func() []string { + list := lists[min(reads, len(lists)-1)] + reads++ + return slices.Clone(list) + } + t.Cleanup(func() { NameserversFn = previous }) +} + +// resetOsResolverLog clears the change-only state, so one test does not see +// the nameserver reads of another. +func resetOsResolverLog(t *testing.T) { + t.Helper() + reset := func() { + osResolverLog.mu.Lock() + defer osResolverLog.mu.Unlock() + osResolverLog.system = nameserverReads{} + osResolverLog.final = nameserverReads{} + osResolverLog.reason = osResolverReasonUnspecified + } + reset() + t.Cleanup(reset) +} + +// keepOsResolver puts back the OS resolver that the process had before the test. +func keepOsResolver(t *testing.T) { + t.Helper() + resolverMutex.Lock() + previous := or + resolverMutex.Unlock() + t.Cleanup(func() { + resolverMutex.Lock() + storeOsResolver(previous) + resolverMutex.Unlock() + }) +} + +// TestOsResolverNameserversReadsWithoutTheResolverLock covers a log header +// render during a resolver initialization. That initialization holds +// resolverMutex across a scutil read of several seconds, and the header must +// not wait for it. +func TestOsResolverNameserversReadsWithoutTheResolverLock(t *testing.T) { + keepOsResolver(t) + resolverMutex.Lock() + storeOsResolver(newResolverWithNameserver([]string{"192.0.2.1:53"})) + + read := make(chan []string, 1) + go func() { read <- OsResolverNameservers() }() + + select { + case nameservers := <-read: + resolverMutex.Unlock() + if want := []string{"192.0.2.1:53"}; !slices.Equal(nameservers, want) { + t.Fatalf("nameservers = %v, want %v", nameservers, want) + } + case <-time.After(5 * time.Second): + resolverMutex.Unlock() + t.Fatal("OsResolverNameservers waited for resolverMutex") + } +} + +func TestJournalMarksTheEventAndKeepsItsLevel(t *testing.T) { + logs := captureProxyLog(t) + + Journal(ProxyLogger.Load().Info()).Msg("Journal test event") + + events := logEventsWithPrefix(t, logs.String(), "Journal test event") + if len(events) != 1 { + t.Fatalf("got %d events, want 1: %s", len(events), logs.String()) + } + wantLogField(t, events[0], JournalField, true) + wantLogField(t, events[0], "level", "info") +} + +func TestOsResolverNameserverReadsLogOnChange(t *testing.T) { + logs := captureProxyLog(t) + resetOsResolverLog(t) + keepOsResolver(t) + first := []string{"192.0.2.1", "192.0.2.2"} + second := []string{"192.0.2.3"} + stubNameservers(t, first, first, second) + + for range 3 { + InitializeOsResolverWithReason(false, "transition") + } + + systemReads := logEventsWithPrefix(t, logs.String(), "Got system nameservers") + if len(systemReads) != 2 { + t.Fatalf("got %d system nameserver lines, want 2: %s", len(systemReads), logs.String()) + } + wantLogField(t, systemReads[0], "repeats", float64(0)) + wantLogField(t, systemReads[1], "repeats", float64(1)) + wantLogField(t, systemReads[1], "message", "Got system nameservers: [192.0.2.3]") + + finalReads := logEventsWithPrefix(t, logs.String(), "Final available nameservers") + if len(finalReads) != 2 { + t.Fatalf("got %d final nameserver lines, want 2: %s", len(finalReads), logs.String()) + } + wantLogField(t, finalReads[1], "repeats", float64(1)) + + changes := logEventsWithPrefix(t, logs.String(), "OS resolver set changed") + if len(changes) != 2 { + t.Fatalf("got %d resolver change events, want one for each changed list: %s", len(changes), logs.String()) + } + last := changes[1] + wantLogStrings(t, last, "before", first) + wantLogStrings(t, last, "after", second) + wantLogField(t, last, JournalField, true) + wantLogField(t, last, "level", "info") + wantLogField(t, last, "reason", "transition") + if _, ok := last["default_route"]; !ok { + t.Fatalf("resolver change event has no default_route field: %v", last) + } + wantSource := map[string]string{"darwin": "scutil", "windows": "dhcp", "linux": "resolv.conf"}[runtime.GOOS] + if wantSource != "" { + wantLogField(t, last, "source", wantSource) + } +} + +func TestInitializeOsResolverReportsTheUnspecifiedReason(t *testing.T) { + logs := captureProxyLog(t) + resetOsResolverLog(t) + keepOsResolver(t) + stubNameservers(t, []string{"192.0.2.10"}) + + InitializeOsResolver(false) + + changes := logEventsWithPrefix(t, logs.String(), "OS resolver set changed") + if len(changes) != 1 { + t.Fatalf("got %d resolver change events, want 1: %s", len(changes), logs.String()) + } + wantLogField(t, changes[0], "reason", "unspecified") + wantLogStrings(t, changes[0], "after", []string{"192.0.2.10"}) +}