diff --git a/.gitignore b/.gitignore index e9121bf..290ea92 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ ctrld.exe # Local planning artifacts - never commit /SPEC.md /tasks/ +/SPEC-*.md diff --git a/cmd/cli/change_logger.go b/cmd/cli/change_logger.go new file mode 100644 index 0000000..52601d8 --- /dev/null +++ b/cmd/cli/change_logger.go @@ -0,0 +1,205 @@ +package cli + +import ( + "maps" + "slices" + "sync" + "tailscale.com/net/netmon" + "time" +) + +// noiseSummaryInterval bounds a storm of noise deltas to one journal line, +// because AirDrop and virtual adapters can churn for hours. +const noiseSummaryInterval = 10 * time.Minute + +// wakeReportInterval merges the reports of the wake sources. netmon polls for +// a time jump every 15 s, so one wake can reach ctrld over half a minute. +const wakeReportInterval = 30 * time.Second + +// repeatLogger holds the last value of each named line, so a line that +// repeats every few seconds reaches the log on change only. +type repeatLogger struct { + mu sync.Mutex + last map[string]string + repeats map[string]uint64 +} + +// changed reports whether the value of a key differs from the value before it. +// The repeat count tells how many lines the caller held back, so the next line +// it writes can carry them. +func (r *repeatLogger) changed(key, value string) (changed bool, repeats uint64) { + r.mu.Lock() + defer r.mu.Unlock() + if r.last == nil { + r.last = make(map[string]string) + r.repeats = make(map[string]uint64) + } + if previous, seen := r.last[key]; seen && previous == value { + r.repeats[key]++ + return false, r.repeats[key] + } + repeats = r.repeats[key] + r.last[key] = value + r.repeats[key] = 0 + return true, repeats +} + +// wakeHoldInterval is the time that a wake report without a gap waits for the +// detector. The detector ticks every two seconds, so it reports well inside +// this hold when it runs. +const wakeHoldInterval = 8 * time.Second + +// wakeReport is one wake as a source saw it. +type wakeReport struct { + source string + gap time.Duration + state *netmon.State +} + +// wakeReporter keeps one report per wake. netmon measures no gap, so its +// report waits for the detector, which measures one. When no detector report +// comes, the held report goes out as it is. +type wakeReporter struct { + mu sync.Mutex + last time.Time + held *wakeReport + // after arms the hold timer. nil means time.AfterFunc, and a test fires + // the hold itself. + after func(time.Duration, func()) +} + +// note takes one report of a wake and hands the report that the journal gets +// to emit. The first source of a window reports. A report with a gap replaces +// a held report without one. Every other report of the window is the same +// wake and drops. +func (w *wakeReporter) note(now time.Time, report wakeReport, emit func(wakeReport)) { + w.mu.Lock() + defer w.mu.Unlock() + gapKnown := report.gap > 0 + if !w.last.IsZero() && now.Sub(w.last) < wakeReportInterval { + if w.held != nil && gapKnown { + w.held = nil + emit(report) + } + return + } + w.last = now + if gapKnown { + w.held = nil + emit(report) + return + } + held := &wakeReport{source: report.source, gap: report.gap, state: report.state} + w.held = held + w.afterFunc()(wakeHoldInterval, func() { w.release(held, emit) }) +} + +// release emits a held report when no source with a gap replaced it. +func (w *wakeReporter) release(held *wakeReport, emit func(wakeReport)) { + w.mu.Lock() + defer w.mu.Unlock() + if w.held != held { + return + } + w.held = nil + emit(*held) +} + +func (w *wakeReporter) afterFunc() func(time.Duration, func()) { + if w.after != nil { + return w.after + } + return func(d time.Duration, fn func()) { time.AfterFunc(d, fn) } +} + +// noiseSummary describes the noise deltas that one journal line covers. +type noiseSummary struct { + Count int + First time.Time + Last time.Time + Interfaces []string +} + +// noiseCoalescer counts the noise deltas of a storm and reports when the +// journal needs a line for them. The count, the first time, and the names +// cover the open window; last covers the storm, so a quiet period ends it. +type noiseCoalescer struct { + mu sync.Mutex + count int + first time.Time + last time.Time + emitted time.Time + names map[string]struct{} +} + +// add records one noise delta and reports whether to log a summary now. The +// summary covers the deltas since the last one, so a storm of any length +// keeps to one line per interval. +func (c *noiseCoalescer) add(interfaces []string, now time.Time) (summary noiseSummary, emit bool) { + c.mu.Lock() + defer c.mu.Unlock() + newRun := c.stormEnded(now) + if newRun { + c.clearWindow() + } + c.record(interfaces, now) + if !newRun && now.Sub(c.emitted) < noiseSummaryInterval { + return noiseSummary{}, false + } + summary = c.summary() + c.emitted = now + c.clearWindow() + return summary, true +} + +// flush reports the open window and closes it. A storm ends with a delta that +// the daemon acts on, so the last window of the storm reaches the journal +// there instead of waiting for the next storm. +func (c *noiseCoalescer) flush(now time.Time) (summary noiseSummary, emit bool) { + c.mu.Lock() + defer c.mu.Unlock() + if c.count == 0 { + return noiseSummary{}, false + } + summary = c.summary() + c.emitted = now + c.clearWindow() + return summary, true +} + +// stormEnded is true when the deltas stopped for longer than the interval, so +// a new storm does not inherit the count of the storm before it. +func (c *noiseCoalescer) stormEnded(now time.Time) bool { + return c.last.IsZero() || now.Sub(c.last) > noiseSummaryInterval +} + +func (c *noiseCoalescer) record(interfaces []string, now time.Time) { + if c.count == 0 { + c.first = now + } + c.count++ + c.last = now + if c.names == nil { + c.names = make(map[string]struct{}) + } + for _, name := range interfaces { + c.names[name] = struct{}{} + } +} + +// summary sorts the names, because a log line that keeps its order reads as +// one value across a capture. +func (c *noiseCoalescer) summary() noiseSummary { + return noiseSummary{ + Count: c.count, + First: c.first, + Last: c.last, + Interfaces: slices.Sorted(maps.Keys(c.names)), + } +} + +func (c *noiseCoalescer) clearWindow() { + c.count = 0 + c.first = time.Time{} + c.names = nil +} diff --git a/cmd/cli/change_logger_test.go b/cmd/cli/change_logger_test.go new file mode 100644 index 0000000..4292c4c --- /dev/null +++ b/cmd/cli/change_logger_test.go @@ -0,0 +1,198 @@ +package cli + +import ( + "slices" + "testing" + "time" +) + +func Test_repeatLogger(t *testing.T) { + var repeats repeatLogger + for _, step := range []struct { + name string + key string + value string + wantChanged bool + wantRepeats uint64 + }{ + {"the first value of a key is a change", "pf anchor", "intact", true, 0}, + {"the same value repeats", "pf anchor", "intact", false, 1}, + {"the same value repeats again", "pf anchor", "intact", false, 2}, + {"a new value reports the run before it", "pf anchor", "missing", true, 2}, + {"the new value repeats", "pf anchor", "missing", false, 1}, + {"the value before is a change again", "pf anchor", "intact", true, 1}, + {"a second key starts on its own", "tunnels", "utun0", true, 0}, + {"the second key repeats", "tunnels", "utun0", false, 1}, + {"the first key keeps its own count", "pf anchor", "intact", false, 1}, + } { + t.Run(step.name, func(t *testing.T) { + changed, count := repeats.changed(step.key, step.value) + if changed != step.wantChanged || count != step.wantRepeats { + t.Errorf("changed(%q, %q) = %v, %d, want %v, %d", + step.key, step.value, changed, count, step.wantChanged, step.wantRepeats) + } + }) + } +} + +func Test_wakeReporter(t *testing.T) { + base := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + var wake wakeReporter + var armed []func() + wake.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + var emitted []wakeReport + emit := func(report wakeReport) { emitted = append(emitted, report) } + fireHeld := func() { + for _, fn := range armed { + fn() + } + armed = nil + } + + // netmon reports first and knows no gap, so its report waits. + wake.note(base, wakeReport{source: "netmon"}, emit) + if len(emitted) != 0 || len(armed) != 1 { + t.Fatalf("a report without a gap must wait: emitted %d, timers %d", len(emitted), len(armed)) + } + // The detector reports inside the hold with the gap, so it replaces the + // held report. + wake.note(base.Add(2*time.Second), wakeReport{source: "detector", gap: 90 * time.Second}, emit) + fireHeld() + if len(emitted) != 1 || emitted[0].source != "detector" { + t.Fatalf("emitted %+v, want the detector report alone", emitted) + } + // Every other report of this wake is the same wake. + wake.note(base.Add(10*time.Second), wakeReport{source: "netmon"}, emit) + wake.note(base.Add(12*time.Second), wakeReport{source: "detector", gap: 91 * time.Second}, emit) + if len(emitted) != 1 { + t.Fatalf("emitted %d reports inside the window, want 1", len(emitted)) + } + + // A wake that only netmon sees reports without a gap when the hold ends. + wake.note(base.Add(40*time.Second), wakeReport{source: "netmon"}, emit) + if len(emitted) != 1 || len(armed) != 1 { + t.Fatalf("a new wake must arm one hold: emitted %d, timers %d", len(emitted), len(armed)) + } + fireHeld() + if len(emitted) != 2 || emitted[1].source != "netmon" || emitted[1].gap != 0 { + t.Fatalf("emitted %+v, want the netmon report after the hold", emitted) + } + // A gap that arrives after the hold is too late: one event per wake. + wake.note(base.Add(50*time.Second), wakeReport{source: "detector", gap: 92 * time.Second}, emit) + if len(emitted) != 2 { + t.Fatalf("emitted %d reports, want no second report of one wake", len(emitted)) + } + + // The detector first reports at once, and netmon adds nothing. + wake.note(base.Add(80*time.Second), wakeReport{source: "detector", gap: 93 * time.Second}, emit) + wake.note(base.Add(82*time.Second), wakeReport{source: "netmon"}, emit) + fireHeld() + if len(emitted) != 3 || emitted[2].source != "detector" || len(armed) != 0 { + t.Fatalf("emitted %+v with %d timers, want one detector report and no hold", emitted, len(armed)) + } +} + +func Test_noiseCoalescer(t *testing.T) { + base := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + + t.Run("a storm reports once every ten minutes", func(t *testing.T) { + const deltas = 100 + const cadence = 15 * time.Second + var noise noiseCoalescer + var got []noiseSummary + for i := 0; i < deltas; i++ { + interfaces := []string{"awdl0"} + if i%2 == 1 { + interfaces = []string{"llw0", "awdl0"} + } + summary, emit := noise.add(interfaces, base.Add(time.Duration(i)*cadence)) + if !emit { + continue + } + got = append(got, summary) + } + want := []noiseSummary{{ + Count: 1, + First: base, + Last: base, + Interfaces: []string{"awdl0"}, + }, { + Count: 40, + First: base.Add(cadence), + Last: base.Add(10 * time.Minute), + Interfaces: []string{"awdl0", "llw0"}, + }, { + Count: 40, + First: base.Add(10*time.Minute + cadence), + Last: base.Add(20 * time.Minute), + Interfaces: []string{"awdl0", "llw0"}, + }} + assertSummaries(t, got, want) + }) + + t.Run("a flush reports the open window", func(t *testing.T) { + var noise noiseCoalescer + if _, emit := noise.flush(base); emit { + t.Fatal("an empty coalescer reported a summary") + } + for _, offset := range []time.Duration{0, time.Minute, 2 * time.Minute} { + noise.add([]string{"awdl0"}, base.Add(offset)) + } + summary, emit := noise.flush(base.Add(3 * time.Minute)) + if !emit { + t.Fatal("the flush reported no summary for an open window") + } + assertSummaries(t, []noiseSummary{summary}, []noiseSummary{{ + Count: 2, + First: base.Add(time.Minute), + Last: base.Add(2 * time.Minute), + Interfaces: []string{"awdl0"}, + }}) + if _, emit = noise.flush(base.Add(4 * time.Minute)); emit { + t.Fatal("the flush reported the same window twice") + } + }) + + t.Run("a gap longer than the interval starts a new run", func(t *testing.T) { + var noise noiseCoalescer + var got []noiseSummary + for _, offset := range []time.Duration{0, time.Minute, 2 * time.Minute, 17 * time.Minute} { + summary, emit := noise.add([]string{"awdl0"}, base.Add(offset)) + if !emit { + continue + } + got = append(got, summary) + } + want := []noiseSummary{{ + Count: 1, + First: base, + Last: base, + Interfaces: []string{"awdl0"}, + }, { + Count: 1, + First: base.Add(17 * time.Minute), + Last: base.Add(17 * time.Minute), + Interfaces: []string{"awdl0"}, + }} + assertSummaries(t, got, want) + }) +} + +func assertSummaries(t *testing.T, got, want []noiseSummary) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("got %d summaries, want %d: %+v", len(got), len(want), got) + } + for i := range want { + if !sameSummary(got[i], want[i]) { + t.Errorf("summary %d = %+v, want %+v", i, got[i], want[i]) + } + } +} + +func sameSummary(got, want noiseSummary) bool { + return got.Count == want.Count && + got.First.Equal(want.First) && + got.Last.Equal(want.Last) && + slices.Equal(got.Interfaces, want.Interfaces) +} diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 94a7e8c..7d99251 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -34,7 +34,6 @@ import ( "github.com/pelletier/go-toml/v2" "github.com/spf13/cobra" "github.com/spf13/viper" - "go.uber.org/zap" "tailscale.com/logtail/backoff" "tailscale.com/net/netmon" @@ -48,7 +47,6 @@ const selfCheckInternalTestDomain = "ctrld" + loopTestDomain const ( windowsForwardersFilename = ".forwarders.txt" oldBinSuffix = "_previous" - oldLogSuffix = ".1" msgExit = "$$EXIT$$" ) @@ -378,6 +376,7 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { oldLogPath := cfg.Service.LogPath if uid := cdUIDFromProvToken(); uid != "" { cdUID = uid + p.initLoggingAfterProvisioning() } if cdUID != "" { if !validateCdUpstreamProtocol(p.notifyExitToLogServer) { @@ -462,18 +461,7 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { } if newLogPath := cfg.Service.LogPath; newLogPath != "" && oldLogPath != newLogPath { - // After processCDFlags, log config may change, so reset mainLog and re-init logging. - l := zap.NewNop() - mainLog.Store(&ctrld.Logger{Logger: l}) - - // Copy logs written so far to new log file if possible. - if buf, err := os.ReadFile(oldLogPath); err == nil { - if err := os.WriteFile(newLogPath, buf, os.FileMode(0o600)); err != nil { - p.Warn().Err(err).Msg("Could not copy old log file") - } - } - initLoggingWithBackup(false) - p.logger.Store(mainLog.Load()) + p.switchLogPath(oldLogPath, newLogPath) } if err := validateConfig(&cfg); err != nil { @@ -893,7 +881,9 @@ func apiRejectionSummary(statusCode int) string { // provisionSecrets lists every secret-bearing value to strip from provisioning // artifacts, including both parts of a composite "/" --cd -// value, which the API may echo back separately. +// value, which the API may echo back separately. redactSecrets drops the +// values that are too short to redact, so a client ID such as "os" leaves the +// words of the message alone. func provisionSecrets() []string { uid, clientID := controld.ParseRawUID(cdUID) return []string{cdUID, cdOrg, uid, clientID} @@ -1378,11 +1368,7 @@ func selfCheckResolveDomain(ctx context.Context, addr, scope string, domain stri mainLog.Load().Debug().Msgf("Self-check against %q failed", domain) loggerCtx := ctrld.LoggerCtx(ctx, mainLog.Load()) // Ping all upstreams to provide better error message to users. - for name, uc := range cfg.Upstream { - if err := uc.ErrorPing(loggerCtx); err != nil { - mainLog.Load().Err(err).Msgf("Failed to connect to upstream.%s, endpoint: %s", name, uc.Endpoint) - } - } + logUpstreamPingFailures(loggerCtx, cfg.Upstream) marker := strings.Repeat("=", 32) mainLog.Load().Debug().Msg(marker) @@ -1398,6 +1384,28 @@ func selfCheckResolveDomain(ctx context.Context, addr, scope string, domain stri return errSelfCheckNoAnswer } +// upstreamPingFn is a var so tests can drive the self-check report without a +// network probe. +var upstreamPingFn = (*ctrld.UpstreamConfig).ErrorPing + +// logUpstreamPingFailures names the upstreams that do not answer after a +// failed self-check, so the user learns which one is unreachable. The config +// key and the endpoint are operator text that can hold a token, so both stay +// at debug and the retained line names the upstream by its bounded name. The +// self-check runs before the program exists, so a bare prog carries the main +// logger into the probe report. +func logUpstreamPingFailures(ctx context.Context, upstreams map[string]*ctrld.UpstreamConfig) { + reporter := &prog{} + reporter.logger.Store(mainLog.Load()) + for name, uc := range upstreams { + err := upstreamPingFn(uc, ctx) + if err == nil { + continue + } + reporter.logUpstreamProbeFailure(upstreamPrefix+name, uc, err, reporter.Error, "Failed to connect to the upstream") + } +} + // userHomeDir returns the user's home directory func userHomeDir() (string, error) { // Mobile platform should provide a rw dir path for this. @@ -1576,6 +1584,8 @@ func fieldErrorMsg(fe validator.FieldError) string { return fmt.Sprintf("minimum len: %q", fe.Param()) case "gte": return fmt.Sprintf("must be greater than or equal to: %s", fe.Param()) + case "lte": + return fmt.Sprintf("must be less than or equal to: %s", fe.Param()) case "cidr": return fmt.Sprintf("invalid value: %s", fe.Value()) case "required_unless", "required": diff --git a/cmd/cli/commands_cleanup_test.go b/cmd/cli/commands_cleanup_test.go new file mode 100644 index 0000000..36b50ab --- /dev/null +++ b/cmd/cli/commands_cleanup_test.go @@ -0,0 +1,71 @@ +package cli + +import ( + "os" + "path/filepath" + "testing" +) + +// writeCleanupTestFile creates a file that a cleanup test can watch. +func writeCleanupTestFile(t *testing.T, path string) string { + t.Helper() + if err := os.WriteFile(path, []byte("line\n"), 0600); err != nil { + t.Fatalf("write %s: %v", path, err) + } + return path +} + +func assertRemoved(t *testing.T, paths []string) { + t.Helper() + for _, path := range paths { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("%s is still there: %v", path, err) + } + } +} + +func Test_removeLogFiles(t *testing.T) { + dir := t.TempDir() + logPath := writeCleanupTestFile(t, filepath.Join(dir, "syslog")) + chain := []string{ + writeCleanupTestFile(t, filepath.Join(dir, "syslog.1")), + writeCleanupTestFile(t, filepath.Join(dir, "syslog.2")), + } + // A log_path can name a file in a directory of another program, so a + // number above the configured count belongs to that program. + foreign := writeCleanupTestFile(t, filepath.Join(dir, "syslog.5")) + home := t.TempDir() + internal := []string{ + writeCleanupTestFile(t, filepath.Join(home, logFileName)), + writeCleanupTestFile(t, filepath.Join(home, journalLogFileName)), + } + internalBackups := []string{ + writeCleanupTestFile(t, filepath.Join(home, logFileName+".3")), + writeCleanupTestFile(t, filepath.Join(home, journalLogFileName+".1")), + } + + if errs := removeLogFiles(logPath, len(chain), internal); len(errs) > 0 { + t.Fatalf("removeLogFiles: %v", errs) + } + + assertRemoved(t, append([]string{logPath}, chain...)) + assertRemoved(t, append(internal, internalBackups...)) + if _, err := os.Stat(foreign); err != nil { + t.Fatalf("the cleanup removed %s, which sits above the backup count: %v", foreign, err) + } +} + +func Test_removeLogFilesWithoutALogPath(t *testing.T) { + home := t.TempDir() + internal := []string{writeCleanupTestFile(t, filepath.Join(home, logFileName))} + kept := writeCleanupTestFile(t, filepath.Join(home, "other.log")) + + if errs := removeLogFiles("", 4, internal); len(errs) > 0 { + t.Fatalf("removeLogFiles: %v", errs) + } + + assertRemoved(t, internal) + if _, err := os.Stat(kept); err != nil { + t.Fatalf("the cleanup removed %s, which is not a ctrld log: %v", kept, err) + } +} diff --git a/cmd/cli/commands_log.go b/cmd/cli/commands_log.go index 094fd45..cc599d9 100644 --- a/cmd/cli/commands_log.go +++ b/cmd/cli/commands_log.go @@ -35,6 +35,15 @@ func NewLogCommand() (*LogCommand, error) { }, nil } +// logRequestPath adds the query that makes the control server read every log +// file, not only the newest debug bytes. +func logRequestPath(path string, full bool) string { + if !full { + return path + } + return path + "?full=1" +} + // warnRuntimeLoggingNotEnabled logs a warning about runtime logging not being enabled func (lc *LogCommand) warnRuntimeLoggingNotEnabled() { mainLog.Load().Warn().Msg("Runtime debug logging is not enabled") @@ -59,7 +68,8 @@ func (lc *LogCommand) SendLogs(cmd *cobra.Command, args []string) error { return nil } - resp, err := lc.controlClient.post(sendLogsPath, nil) + full, _ := cmd.Flags().GetBool("full") + resp, err := lc.controlClient.post(logRequestPath(sendLogsPath, full), nil) if err != nil { return fmt.Errorf("failed to send logs: %w", err) } @@ -105,7 +115,8 @@ func (lc *LogCommand) ViewLogs(cmd *cobra.Command, args []string) error { return nil } - resp, err := lc.controlClient.post(viewLogsPath, nil) + full, _ := cmd.Flags().GetBool("full") + resp, err := lc.controlClient.post(logRequestPath(viewLogsPath, full), nil) if err != nil { return fmt.Errorf("failed to get logs: %w", err) } @@ -221,6 +232,7 @@ func InitLogCmd(rootCmd *cobra.Command) *cobra.Command { }, RunE: lc.SendLogs, } + logSendCmd.Flags().Bool("full", false, "Send every log file, not only the newest 10 MB of debug") logViewCmd := &cobra.Command{ Use: "view", @@ -231,6 +243,7 @@ func InitLogCmd(rootCmd *cobra.Command) *cobra.Command { }, RunE: lc.ViewLogs, } + logViewCmd.Flags().Bool("full", false, "Show every log file, not only the newest 10 MB of debug") logTailCmd := &cobra.Command{ Use: "tail", diff --git a/cmd/cli/commands_service_uninstall.go b/cmd/cli/commands_service_uninstall.go index c3c1941..f1801fc 100644 --- a/cmd/cli/commands_service_uninstall.go +++ b/cmd/cli/commands_service_uninstall.go @@ -1,6 +1,7 @@ package cli import ( + "fmt" "net" "os" "path/filepath" @@ -47,14 +48,15 @@ func (sc *ServiceCommand) Uninstall(cmd *cobra.Command, args []string) error { var files []string // Config file. files = append(files, v.ConfigFileUsed()) - // Log file and backup log file. - // For safety, only process if log file path is absolute. - if logFile := normalizeLogFilePath(cfg.Service.LogPath); filepath.IsAbs(logFile) { - files = append(files, logFile) - oldLogFile := logFile + oldLogSuffix - if _, err := os.Stat(oldLogFile); err == nil { - files = append(files, oldLogFile) - } + // Log files. For safety, only remove the log_path chain if that path + // is absolute. + logFile := normalizeLogFilePath(cfg.Service.LogPath) + if !filepath.IsAbs(logFile) { + logFile = "" + } + internalLogs := []string{absHomeDir(logFileName), absHomeDir(journalLogFileName)} + for _, err := range removeLogFiles(logFile, debugLogBudget(&cfg.Service).backups, internalLogs) { + logger.Warn().Err(err).Msg("Failed to remove log file") } // Socket files. if dir, _ := socketDir(); dir != "" { @@ -104,3 +106,34 @@ func (sc *ServiceCommand) Uninstall(cmd *cobra.Command, args []string) error { logger.Debug().Msg("Service uninstall command completed") return nil } + +// removeLogFiles deletes the log files of this installation and returns what +// it could not remove. A missing file is not an error. +func removeLogFiles(logPath string, backups int, internalPaths []string) []error { + for _, path := range internalPaths { + pruneNumberedBackups(path, 0) + } + var errs []error + for _, path := range logFilesToRemove(logPath, backups, internalPaths) { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + errs = append(errs, fmt.Errorf("remove %s: %w", path, err)) + } + } + return errs +} + +// logFilesToRemove names the log files of this installation. The log_path +// backups come from the configured count, because log_path can name a file in +// a directory that holds the files of other programs, and a scan of that +// directory would take their files too. An empty logPath leaves the internal +// files alone. +func logFilesToRemove(logPath string, backups int, internalPaths []string) []string { + var paths []string + if logPath != "" { + paths = append(paths, logPath) + for index := 1; index <= backups; index++ { + paths = append(paths, fmt.Sprintf("%s.%d", logPath, index)) + } + } + return append(paths, internalPaths...) +} diff --git a/cmd/cli/control_client.go b/cmd/cli/control_client.go index 8f174f2..7a5693b 100644 --- a/cmd/cli/control_client.go +++ b/cmd/cli/control_client.go @@ -5,6 +5,8 @@ import ( "io" "net" "net/http" + "net/url" + "strings" "time" ) @@ -27,13 +29,25 @@ func newControlClient(addr string) *controlClient { } func (c *controlClient) post(path string, data io.Reader) (*http.Response, error) { - // for log/send, set the timeout to 5 minutes - if path == sendLogsPath { + // A log send uploads many megabytes, so it needs more time than the other + // calls. + if isSendLogsPath(path) { c.c.Timeout = time.Minute * 5 } return c.c.Post("http://unix"+path, contentTypeJson, data) } +// isSendLogsPath reports whether path addresses the log send handler. The path +// can carry a query, so the comparison takes the path alone. A path that does +// not parse keeps the prefix rule, because it still reaches that handler. +func isSendLogsPath(path string) bool { + parsed, err := url.Parse(path) + if err != nil { + return strings.HasPrefix(path, sendLogsPath) + } + return parsed.Path == sendLogsPath +} + // postStream sends a POST request with no timeout, suitable for long-lived streaming connections. func (c *controlClient) postStream(path string, data io.Reader) (*http.Response, error) { c.c.Timeout = 0 diff --git a/cmd/cli/control_server.go b/cmd/cli/control_server.go index 0e8195a..8bf12fc 100644 --- a/cmd/cli/control_server.go +++ b/cmd/cli/control_server.go @@ -11,6 +11,7 @@ import ( "reflect" "sort" "strconv" + "sync/atomic" "time" "github.com/kardianos/service" @@ -323,48 +324,31 @@ func (p *prog) registerControlServerHandler() { return } })) - p.cs.register(viewLogsPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { - lr, err := p.logReaderRaw() - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - defer lr.r.Close() - if lr.size == 0 { - w.WriteHeader(http.StatusMovedPermanently) - return - } - data, err := io.ReadAll(lr.r) - if err != nil { - http.Error(w, fmt.Sprintf("could not read log: %v", err), http.StatusInternalServerError) - return - } - if err := json.NewEncoder(w).Encode(&logViewResponse{Data: string(data)}); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - http.Error(w, fmt.Sprintf("could not marshal log data: %v", err), http.StatusInternalServerError) - return - } - })) + p.cs.register(viewLogsPath, http.HandlerFunc(p.handleLogView)) p.cs.register(sendLogsPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { if time.Since(p.internalLogSent) < logWriterSentInterval { w.WriteHeader(http.StatusServiceUnavailable) return } - r, err := p.logReaderNoColor() + r, err := p.logReader(wantFullLogs(request), true) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } + defer r.r.Close() if r.size == 0 { w.WriteHeader(http.StatusMovedPermanently) return } + // The upload loses its color codes on the way, so the answer reports + // the bytes that went out, not the bytes that the files hold. + upload := &countingReadCloser{rc: r.r} req := &controld.LogsRequest{ UID: cdUID, - Data: r.r, + Data: upload, } p.Debug().Msg("Sending log file to ControlD server") - resp := logSentResponse{Size: r.size} + resp := logSentResponse{} loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) if err := controld.SendLogs(loggerCtx, req, cdDev); err != nil { p.Error().Msgf("Could not send log file to ControlD server: %v", err) @@ -374,6 +358,7 @@ func (p *prog) registerControlServerHandler() { p.Debug().Msg("Sending log file successfully") w.WriteHeader(http.StatusOK) } + resp.Size = upload.count() if err := json.NewEncoder(w).Encode(&resp); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } @@ -445,50 +430,138 @@ func (p *prog) registerControlServerHandler() { } } else { // File-based logging mode: tail the log file. - logFile := normalizeLogFilePath(p.cfg.Service.LogPath) - f, err := os.Open(logFile) - if err != nil { - // Already committed 200, just return. - return - } - defer f.Close() - - // Seek to show last N lines. - if numLines > 0 { - if tail := tailFileLastLines(f, numLines); len(tail) > 0 { - w.Write(tail) - flusher.Flush() - } - } else { - // Seek to end. - f.Seek(0, io.SeekEnd) - } - - // Poll for new data. - buf := make([]byte, 4096) - ticker := time.NewTicker(200 * time.Millisecond) - defer ticker.Stop() - for { - select { - case <-ticker.C: - n, err := f.Read(buf) - if n > 0 { - if _, werr := w.Write(buf[:n]); werr != nil { - return - } - flusher.Flush() - } - if err != nil && err != io.EOF { - return - } - case <-request.Context().Done(): - return - } - } + followLogFile(request.Context(), w, flusher, normalizeLogFilePath(p.cfg.Service.LogPath), numLines) } })) } +// jsonStringChunkSize bounds the memory that one log view answer needs. A log +// of tens of megabytes must not get a second copy in the answer. +const jsonStringChunkSize = 32 * 1024 + +// wantFullLogs reports whether the request asks for every log file instead of +// the newest debug bytes. +func wantFullLogs(r *http.Request) bool { + return r.URL.Query().Get("full") == "1" +} + +// handleLogView answers with the log bytes in the data field. It writes the +// body while it reads the log, so the answer needs no copy of the whole log. +func (p *prog) handleLogView(w http.ResponseWriter, r *http.Request) { + lr, err := p.logReader(wantFullLogs(r), false) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + defer lr.r.Close() + if lr.size == 0 { + w.WriteHeader(http.StatusMovedPermanently) + return + } + w.Header().Set("Content-Type", contentTypeJson) + if _, err := io.WriteString(w, `{"data":`); err != nil { + return + } + if err := writeJSONString(w, lr.r); err != nil { + // The answer is on its way, so it can only end short. This line tells + // the operator why. + p.Error().Err(err).Msg("Could not send log view answer") + return + } + _, _ = io.WriteString(w, `}`) +} + +// writeJSONString writes what r yields as one JSON string, in chunks. +func writeJSONString(w io.Writer, r io.Reader) error { + if _, err := io.WriteString(w, `"`); err != nil { + return err + } + chunk := make([]byte, jsonStringChunkSize) + for { + n, readErr := r.Read(chunk) + if n > 0 { + if err := writeJSONStringChunk(w, chunk[:n]); err != nil { + return err + } + } + if readErr == io.EOF { + break + } + if readErr != nil { + return readErr + } + } + _, err := io.WriteString(w, `"`) + return err +} + +// writeJSONStringChunk writes one chunk and escapes the bytes that a JSON +// string cannot carry. It writes the bytes between two escapes in one call. +func writeJSONStringChunk(w io.Writer, chunk []byte) error { + start := 0 + for i, b := range chunk { + escaped := jsonStringEscape(b) + if escaped == "" { + continue + } + if _, err := w.Write(chunk[start:i]); err != nil { + return err + } + if _, err := io.WriteString(w, escaped); err != nil { + return err + } + start = i + 1 + } + _, err := w.Write(chunk[start:]) + return err +} + +// jsonStringEscape returns the escape of one byte, or an empty string for a +// byte that a JSON string carries as it is. Every byte of a multi-byte +// character is above 0x1f, so a chunk that ends inside a character still +// writes the character whole. +func jsonStringEscape(b byte) string { + switch b { + case '"': + return `\"` + case '\\': + return `\\` + case '\n': + return `\n` + case '\r': + return `\r` + case '\t': + return `\t` + } + if b < 0x20 { + return fmt.Sprintf(`\u%04x`, b) + } + return "" +} + +// countingReadCloser counts the bytes that the HTTP client reads from an +// upload. It closes the log files behind the upload, because the client only +// closes the body it was given. +type countingReadCloser struct { + rc io.ReadCloser + n atomic.Int64 +} + +func (c *countingReadCloser) Read(p []byte) (int, error) { + n, err := c.rc.Read(p) + c.n.Add(int64(n)) + return n, err +} + +func (c *countingReadCloser) Close() error { + return c.rc.Close() +} + +// count reports the bytes read so far. +func (c *countingReadCloser) count() int64 { + return c.n.Load() +} + // tailFileLastLines reads the last n lines from a file and returns them. // The file position is left at the end of the file after this call. func tailFileLastLines(f *os.File, n int) []byte { diff --git a/cmd/cli/control_server_tail_test.go b/cmd/cli/control_server_tail_test.go new file mode 100644 index 0000000..ec678e5 --- /dev/null +++ b/cmd/cli/control_server_tail_test.go @@ -0,0 +1,104 @@ +package cli + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// streamRecorder collects a streamed response under a lock, because the +// handler writes from its own goroutine while the test reads. +type streamRecorder struct { + mu sync.Mutex + header http.Header + body strings.Builder +} + +func (r *streamRecorder) Header() http.Header { return r.header } + +func (r *streamRecorder) Write(p []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + return r.body.Write(p) +} + +func (r *streamRecorder) WriteHeader(int) {} + +func (r *streamRecorder) Flush() {} + +func (r *streamRecorder) String() string { + r.mu.Lock() + defer r.mu.Unlock() + return r.body.String() +} + +// waitForText polls the recorder until want arrives or the deadline passes. +func waitForText(rec *streamRecorder, want string, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if strings.Contains(rec.String(), want) { + return true + } + time.Sleep(20 * time.Millisecond) + } + return false +} + +// Test_logTailFollowsTheRotatedLogPath keeps one tail request open while the +// log_path file rotates. The lines after the rotation must reach the client. +func Test_logTailFollowsTheRotatedLogPath(t *testing.T) { + origUID := cdUID + cdUID = "" + t.Cleanup(func() { cdUID = origUID }) + + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, logBudget{maxSize: 64, backups: 1}, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + if _, err := rf.Write([]byte("before rotation\n")); err != nil { + t.Fatal(err) + } + + p := &prog{ + cfg: &ctrld.Config{Service: ctrld.ServiceConfig{LogPath: path}}, + cs: &controlServer{mux: http.NewServeMux()}, + } + p.registerControlServerHandler() + + ctx, cancel := context.WithCancel(context.Background()) + req := httptest.NewRequest(http.MethodGet, tailLogsPath+"?lines=1", nil).WithContext(ctx) + rec := &streamRecorder{header: http.Header{}} + done := make(chan struct{}) + go func() { + defer close(done) + p.cs.mux.ServeHTTP(rec, req) + }() + t.Cleanup(func() { + cancel() + <-done + }) + + if !waitForText(rec, "before rotation", 2*time.Second) { + t.Fatalf("the first line did not arrive: %q", rec.String()) + } + for i := 0; i < 4; i++ { + if _, err := rf.Write([]byte("filler line that crosses the budget\n")); err != nil { + t.Fatal(err) + } + } + if _, err := rf.Write([]byte("after rotation sentinel\n")); err != nil { + t.Fatal(err) + } + if !waitForText(rec, "after rotation sentinel", 3*time.Second) { + t.Fatalf("the sentinel did not arrive after the rotation: %q", rec.String()) + } +} diff --git a/cmd/cli/control_server_test.go b/cmd/cli/control_server_test.go index 297b37d..bdafe7b 100644 --- a/cmd/cli/control_server_test.go +++ b/cmd/cli/control_server_test.go @@ -2,10 +2,14 @@ package cli import ( "bytes" + "encoding/json" "io" "net/http" "os" + "strings" "testing" + + "github.com/Control-D-Inc/ctrld" ) func TestControlServer(t *testing.T) { @@ -52,3 +56,107 @@ func TestControlServer(t *testing.T) { t.Fatal(err) } } + +// jsonStringTestPayload holds every byte class that a JSON string must escape +// and splits a multi-byte character across the chunk boundary, so a chunk that +// ends inside a character is covered. +func jsonStringTestPayload() []byte { + var payload bytes.Buffer + payload.WriteString("quotes \" backslash \\ tab\t return\r newline\n") + payload.Write([]byte{0x00, 0x01, 0x1f}) + for payload.Len() < jsonStringChunkSize-1 { + payload.WriteByte('x') + } + payload.WriteString("é日本語 and an emoji 🙂\n") + payload.WriteString("the tail after the boundary\n") + return payload.Bytes() +} + +func Test_writeJSONString(t *testing.T) { + payload := jsonStringTestPayload() + var written bytes.Buffer + + if err := writeJSONString(&written, bytes.NewReader(payload)); err != nil { + t.Fatalf("writeJSONString: %v", err) + } + + var got string + if err := json.Unmarshal(written.Bytes(), &got); err != nil { + t.Fatalf("unmarshal %d bytes: %v", written.Len(), err) + } + if got != string(payload) { + t.Fatalf("the round trip returned %d bytes, want the %d bytes of the payload", len(got), len(payload)) + } +} + +// newLogViewTestServer serves the log view handler of p on a temporary socket +// and returns a client of it. +func newLogViewTestServer(t *testing.T, p *prog) *controlClient { + t.Helper() + f, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("temporary socket: %v", err) + } + f.Close() + t.Cleanup(func() { os.Remove(f.Name()) }) + s, err := newControlServer(f.Name()) + if err != nil { + t.Fatalf("newControlServer: %v", err) + } + s.register(viewLogsPath, http.HandlerFunc(p.handleLogView)) + if err := s.start(); err != nil { + t.Fatalf("start: %v", err) + } + t.Cleanup(func() { _ = s.stop() }) + return newControlClient(f.Name()) +} + +func Test_logViewHandler(t *testing.T) { + t.Run("answers the log as one JSON string", func(t *testing.T) { + p := setupMemoryLogReaderTest(t) + line := "a line with \"quotes\", a tab\there and 日本語\n" + if _, err := p.internalLogWriter.Write([]byte(line)); err != nil { + t.Fatalf("write the debug buffer: %v", err) + } + + resp, err := newLogViewTestServer(t, p).post(viewLogsPath, nil) + + if err != nil { + t.Fatalf("post: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if ct := resp.Header.Get("content-type"); ct != contentTypeJson { + t.Fatalf("content type = %s, want %s", ct, contentTypeJson) + } + var view logViewResponse + if err := json.NewDecoder(resp.Body).Decode(&view); err != nil { + t.Fatalf("decode the answer: %v", err) + } + if !strings.Contains(view.Data, line) { + t.Fatalf("the answer holds %.64q, want the line %q in it", view.Data, line) + } + if !strings.Contains(view.Data, logWriterLogEndMarker) { + t.Fatalf("the answer holds no log end marker: %.64q", view.Data) + } + }) + + t.Run("answers 301 without a log", func(t *testing.T) { + origSilent, origCdUID := silent, cdUID + t.Cleanup(func() { silent, cdUID = origSilent, origCdUID }) + silent, cdUID = false, "" + p := &prog{cfg: &ctrld.Config{}} + + resp, err := newLogViewTestServer(t, p).post(viewLogsPath, nil) + + if err != nil { + t.Fatalf("post: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusMovedPermanently { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusMovedPermanently) + } + }) +} diff --git a/cmd/cli/dns64.go b/cmd/cli/dns64.go index aa5af19..1856bad 100644 --- a/cmd/cli/dns64.go +++ b/cmd/cli/dns64.go @@ -293,6 +293,15 @@ func (p *prog) activeDNS64Prefix() (netip.Prefix, bool) { return p.dns64.prefix, p.dns64.prefix.IsValid() } +// dns64PrefixSnapshot reports the stored NAT64 prefix and whether synthesis +// runs. It reads the state only, because a report of the network must not +// reclassify it or start a discovery query. +func (p *prog) dns64PrefixSnapshot() (netip.Prefix, bool) { + p.dns64.mu.Lock() + defer p.dns64.mu.Unlock() + return p.dns64.prefix, p.dns64.active +} + func dns64CacheVariant(prefix netip.Prefix) string { return "dns64:" + prefix.String() } @@ -399,13 +408,40 @@ func (p *prog) discoverNAT64Prefix(generation uint64) { } prefix, ok := nat64PrefixFromAnswer(answer) if !ok { - mainLog.Load().Debug().Msg("dns64: no NAT64 prefix present (not a DNS64 network)") + p.logNAT64Result("") return } if !p.storeDiscoveredNAT64Prefix(generation, prefix) { return } - mainLog.Load().Info().Msgf("dns64: discovered NAT64 prefix %s; enabling AAAA synthesis for IPv6-only network without CLAT", prefix) + p.logNAT64Result(prefix.String()) +} + +// logNAT64Result reports one discovery result. An empty prefix means that the +// network has none. Both results report a change of the network, so both go to +// the journal at info. +func (p *prog) logNAT64Result(prefix string) { + if !p.noteNAT64Result(prefix) { + return + } + if prefix == "" { + journal(mainLog.Load().Info()).Msg("dns64: no NAT64 prefix present (not a DNS64 network)") + return + } + journal(mainLog.Load().Info()).Msgf("dns64: discovered NAT64 prefix %s; enabling AAAA synthesis for IPv6-only network without CLAT", prefix) +} + +// noteNAT64Result reports whether this NAT64 result differs from the one that +// the journal already holds. An empty prefix means no prefix. Discovery runs +// every recheck window, and the journal keeps the changes only. +func (p *prog) noteNAT64Result(prefix string) bool { + p.lastNAT64Mu.Lock() + defer p.lastNAT64Mu.Unlock() + if p.lastNAT64Logged == prefix { + return false + } + p.lastNAT64Logged = prefix + return true } // maybeDNS64 applies DNS64 synthesis to an already-filtered answer when the diff --git a/cmd/cli/dns64_test.go b/cmd/cli/dns64_test.go index 292450e..3985aea 100644 --- a/cmd/cli/dns64_test.go +++ b/cmd/cli/dns64_test.go @@ -364,3 +364,61 @@ func TestMaybeDNS64DropsStaleInFlightPrefix(t *testing.T) { t.Fatal("in-flight synthesis used a prefix invalidated by a network change") } } + +// TestDNS64PrefixSnapshotReadsTheStoredStateOnly covers the read that a report +// of the network uses. The active read reclassifies the network and can start +// a discovery query, and a report must change nothing. +func TestDNS64PrefixSnapshotReadsTheStoredStateOnly(t *testing.T) { + classReads := 0 + old := dns64NetworkClassFn + dns64NetworkClassFn = func() (bool, bool, error) { + classReads++ + return false, false, nil + } + t.Cleanup(func() { dns64NetworkClassFn = old }) + + p := &prog{} + p.dns64.prefix = netip.MustParsePrefix("64:ff9b::/96") + p.dns64.active = true + + prefix, active := p.dns64PrefixSnapshot() + + if !active || prefix != p.dns64.prefix { + t.Fatalf("snapshot = %v, %v, want %v, true", prefix, active, p.dns64.prefix) + } + if classReads != 0 { + t.Fatalf("the snapshot classified the network %d times, want 0", classReads) + } + if p.dns64.discovering { + t.Fatal("the snapshot started a NAT64 prefix discovery") + } +} + +// TestDNS64ResultLinesReachTheJournalAtInfo covers the level of the two NAT64 +// result lines. Both report a change, and a debug line never leaves a host +// that runs at the normal level, so the journal lost the result of a network +// that has no prefix. +func TestDNS64ResultLinesReachTheJournalAtInfo(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + + // The first result of a run reports nothing when the network has no + // prefix, so the prefix has to arrive before it goes away. + p.logNAT64Result("64:ff9b::/96") + p.logNAT64Result("64:ff9b::/96") + p.logNAT64Result("") + + absent := jsonLogEvents(t, logs, "dns64: no NAT64 prefix present (not a DNS64 network)") + if len(absent) != 1 { + t.Fatalf("got %d no-prefix lines, want 1", len(absent)) + } + wantField(t, absent[0], "level", "info") + wantField(t, absent[0], "journal", true) + + found := jsonLogEvents(t, logs, "dns64: discovered NAT64 prefix 64:ff9b::/96; enabling AAAA synthesis for IPv6-only network without CLAT") + if len(found) != 1 { + t.Fatalf("got %d discovered lines for two identical results, want 1", len(found)) + } + wantField(t, found[0], "level", "info") + wantField(t, found[0], "journal", true) +} diff --git a/cmd/cli/dns_config.go b/cmd/cli/dns_config.go new file mode 100644 index 0000000..22c44ef --- /dev/null +++ b/cmd/cli/dns_config.go @@ -0,0 +1,360 @@ +package cli + +import ( + "bufio" + "errors" + "io" + "slices" + "strconv" + "strings" + "sync" + "time" +) + +const ( + // dnsConfigFastInterval is the poll period that follows a network event. + dnsConfigFastInterval = 15 * time.Second + + // dnsConfigSlowInterval is the poll period of a quiet host. + dnsConfigSlowInterval = 5 * time.Minute + + // dnsConfigFastWindow is how long the fast period lasts. A resolver change + // that follows a network event arrives late, after the OS settles. + dnsConfigFastWindow = 10 * time.Minute +) + +// scutilTablePrefix starts a resolver table. macOS prints the table for scoped +// queries after the main one, with the same line. +const scutilTablePrefix = "DNS configuration" + +// scutilResolverPrefix starts one resolver block inside a table. +const scutilResolverPrefix = "resolver #" + +// scutilDNSOutputLimit bounds the text of one read. A resolver table of a host +// is a few kilobytes, so a larger output does not come from a table. +const scutilDNSOutputLimit = 1 << 20 + +// errSCUtilDNSTooLarge ends a read that passed the bound. +var errSCUtilDNSTooLarge = errors.New("scutil --dns output is too large") + +// dnsResolverEntry is one resolver of the DNS configuration of the host. +type dnsResolverEntry struct { + Order int + Nameservers []string + IfIndex int + Interface string + Flags string + SearchDomains []string + Reachable string + // Scoped marks a resolver of the table for scoped queries, which serves one + // interface. A second service keeps its own resolver there while the + // unscoped table stays the same. + Scoped bool + // Action tells what the diff saw: added, changed, or removed. A resolver + // without nameservers looks the same whether it came or went, so the event + // needs the word. + Action string +} + +// The actions of a changed resolver. +const ( + resolverActionAdded = "added" + resolverActionChanged = "changed" + resolverActionRemoved = "removed" +) + +// parseSCUtilDNS reads the resolver tables of "scutil --dns": the unscoped +// table that answers a query of the host, then the table for scoped queries +// with one resolver per interface. Both tables count, because a second +// service can change its resolver while the unscoped table stays the same. +// +// A read that ends early or passes the bound gives an error and no table. A +// part of a table looks like resolvers that went away, and the caller must +// keep the table of the read before instead. +func parseSCUtilDNS(r io.Reader) ([]dnsResolverEntry, error) { + var entries []dnsResolverEntry + scoped := false + var entry *dnsResolverEntry + closeEntry := func() { + if entry == nil { + return + } + entry.Scoped = scoped + entries = append(entries, *entry) + entry = nil + } + read := 0 + scanner := bufio.NewScanner(io.LimitReader(r, scutilDNSOutputLimit+1)) + for scanner.Scan() { + read += len(scanner.Bytes()) + 1 + if read > scutilDNSOutputLimit { + return nil, errSCUtilDNSTooLarge + } + line := strings.TrimSpace(scanner.Text()) + switch { + case strings.HasPrefix(line, scutilTablePrefix): + closeEntry() + scoped = strings.Contains(line, "scoped") + case strings.HasPrefix(line, scutilResolverPrefix): + closeEntry() + entry = &dnsResolverEntry{} + case entry != nil: + applySCUtilField(entry, line) + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + closeEntry() + return entries, nil +} + +func applySCUtilField(entry *dnsResolverEntry, line string) { + name, value, ok := strings.Cut(line, ":") + if !ok { + return + } + name, value = strings.TrimSpace(name), strings.TrimSpace(value) + switch { + case strings.HasPrefix(name, "nameserver["): + entry.Nameservers = append(entry.Nameservers, value) + case strings.HasPrefix(name, "search domain["): + entry.SearchDomains = append(entry.SearchDomains, value) + case name == "if_index": + entry.IfIndex, entry.Interface = parseSCUtilIfIndex(value) + case name == "flags": + entry.Flags = value + case name == "reach": + entry.Reachable = scutilBracketText(value) + case name == "order": + entry.Order, _ = strconv.Atoi(value) + } +} + +// parseSCUtilIfIndex reads an "if_index" value of the form "14 (en0)". +func parseSCUtilIfIndex(value string) (int, string) { + number, name, _ := strings.Cut(value, " ") + index, _ := strconv.Atoi(number) + return index, scutilBracketText(name) +} + +// scutilBracketText returns the text in brackets, where scutil puts the +// readable form of a number. +func scutilBracketText(value string) string { + start := strings.Index(value, "(") + end := strings.LastIndex(value, ")") + if start < 0 || end < start { + return "" + } + return value[start+1 : end] +} + +// resolverKey identifies one resolver across two reads. An unscoped resolver +// has an order, and several can share it, so the count of earlier entries +// with that order completes the key. A scoped resolver has no order line and +// serves one interface, so its interface index is its identity. The two +// tables are two key spaces. +type resolverKey struct { + scoped bool + order int + ifIndex int + occurrence int +} + +func resolverKeys(entries []dnsResolverEntry) []resolverKey { + counts := make(map[resolverKey]int, len(entries)) + keys := make([]resolverKey, len(entries)) + for i, entry := range entries { + base := resolverKey{scoped: entry.Scoped, order: entry.Order} + if entry.Scoped { + base = resolverKey{scoped: true, ifIndex: entry.IfIndex} + } + key := base + key.occurrence = counts[base] + keys[i] = key + counts[base]++ + } + return keys +} + +func equalResolverEntry(a, b dnsResolverEntry) bool { + return a.IfIndex == b.IfIndex && + a.Interface == b.Interface && + a.Flags == b.Flags && + a.Reachable == b.Reachable && + slices.Equal(a.Nameservers, b.Nameservers) && + slices.Equal(a.SearchDomains, b.SearchDomains) +} + +// diffResolverTables returns the resolvers that differ between two reads. A +// resolver that vanished comes back without nameservers, so the journal shows +// that the host dropped it. +func diffResolverTables(before, after []dnsResolverEntry) []dnsResolverEntry { + beforeKeys, afterKeys := resolverKeys(before), resolverKeys(after) + previous := make(map[resolverKey]dnsResolverEntry, len(before)) + for i, key := range beforeKeys { + previous[key] = before[i] + } + current := make(map[resolverKey]struct{}, len(after)) + for _, key := range afterKeys { + current[key] = struct{}{} + } + var changed []dnsResolverEntry + for i, key := range afterKeys { + known, ok := previous[key] + if ok && equalResolverEntry(known, after[i]) { + continue + } + entry := after[i] + entry.Action = resolverActionChanged + if !ok { + entry.Action = resolverActionAdded + } + changed = append(changed, entry) + } + for i, key := range beforeKeys { + if _, ok := current[key]; ok { + continue + } + vanished := before[i] + vanished.Nameservers = nil + vanished.Action = resolverActionRemoved + changed = append(changed, vanished) + } + return changed +} + +// dnsConfigPoller reads the DNS configuration of the host on a timer and +// reports the resolvers that changed since the read before. +type dnsConfigPoller struct { + run func() ([]dnsResolverEntry, error) + now func() time.Time + mu sync.Mutex + last []dnsResolverEntry + lastActivity time.Time + started bool + readFailed bool + // kick wakes a loop that waits on the slow delay. One pending signal is + // enough, so a storm of events does not queue polls. + kick chan struct{} +} + +func newDNSConfigPoller(run func() ([]dnsResolverEntry, error)) *dnsConfigPoller { + return &dnsConfigPoller{run: run, now: time.Now, kick: make(chan struct{}, 1)} +} + +// noteActivity keeps the poll fast for a while after the network moved, and +// wakes a loop that waits on the slow delay. +func (p *dnsConfigPoller) noteActivity(now time.Time) { + p.markActivity(now) + select { + case p.kick <- struct{}{}: + default: + } +} + +// markActivity records the time of the last network event. +func (p *dnsConfigPoller) markActivity(now time.Time) { + p.mu.Lock() + defer p.mu.Unlock() + p.lastActivity = now +} + +func (p *dnsConfigPoller) nextDelay(now time.Time) time.Duration { + p.mu.Lock() + defer p.mu.Unlock() + if p.lastActivity.IsZero() || now.Sub(p.lastActivity) >= dnsConfigFastWindow { + return dnsConfigSlowInterval + } + return dnsConfigFastInterval +} + +// pollOnce returns the resolvers that changed. The first table is the baseline +// of the process and reports no change. A failed read keeps the table of the +// read before, so a short failure does not report the whole table twice. The +// first failure and the read that ends it log once, so a host whose read +// never works says so. +func (p *dnsConfigPoller) pollOnce(now time.Time) []dnsResolverEntry { + table, err := p.run() + p.mu.Lock() + defer p.mu.Unlock() + if err != nil { + if !p.readFailed { + mainLog.Load().Warn().Err(err).Msg("DNS configuration read failed; the journal keeps the last table") + } + p.readFailed = true + return nil + } + if p.readFailed { + mainLog.Load().Info().Msg("DNS configuration read works again") + p.readFailed = false + } + if !p.started { + p.started = true + p.last = table + return nil + } + changed := diffResolverTables(p.last, table) + p.last = table + return changed +} + +// dnsConfigDelayFn lets a test shorten the wait between two polls. +var dnsConfigDelayFn = (*dnsConfigPoller).nextDelay + +// startLoop runs the poll loop and returns a channel that closes when the loop +// returns. A caller joins that channel before it reads the log. +func (p *dnsConfigPoller) startLoop(stop <-chan struct{}, emit func([]dnsResolverEntry)) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) + p.loop(stop, emit) + }() + return done +} + +// loop polls until stop closes. The read starts a subprocess on macOS, so the +// delay follows the last activity and a quiet host pays little. The start of a +// run counts as activity, because the table of the host settles after the +// daemon comes up, and the read at entry is the baseline of the process. +func (p *dnsConfigPoller) loop(stop <-chan struct{}, emit func([]dnsResolverEntry)) { + now := p.now() + p.markActivity(now) + p.pollOnce(now) + delay := dnsConfigDelayFn(p, now) + deadline := now.Add(delay) + timer := time.NewTimer(delay) + defer timer.Stop() + for { + select { + case <-stop: + return + case <-p.kick: + // noteActivity set the time just now, so the delay is the fast one. + // A kick only moves a deadline closer; a storm of kicks must not + // push the read back again and again. + now := p.now() + wanted := now.Add(dnsConfigDelayFn(p, now)) + if !wanted.Before(deadline) { + continue + } + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timer.Reset(wanted.Sub(now)) + deadline = wanted + continue + case <-timer.C: + } + now := p.now() + if changed := p.pollOnce(now); len(changed) > 0 { + emit(changed) + } + delay := dnsConfigDelayFn(p, now) + deadline = now.Add(delay) + timer.Reset(delay) + } +} diff --git a/cmd/cli/dns_config_darwin.go b/cmd/cli/dns_config_darwin.go new file mode 100644 index 0000000..60d0c78 --- /dev/null +++ b/cmd/cli/dns_config_darwin.go @@ -0,0 +1,22 @@ +package cli + +import ( + "bytes" + "context" + "os/exec" + "time" +) + +// scutilDNSTimeout bounds the read. scutil talks to the configuration daemon, +// and that daemon stalls while the network moves. +const scutilDNSTimeout = 5 * time.Second + +func runSCUtilDNS() ([]dnsResolverEntry, error) { + ctx, cancel := context.WithTimeout(context.Background(), scutilDNSTimeout) + defer cancel() + out, err := exec.CommandContext(ctx, "scutil", "--dns").Output() + if err != nil { + return nil, err + } + return parseSCUtilDNS(bytes.NewReader(out)) +} diff --git a/cmd/cli/dns_config_others.go b/cmd/cli/dns_config_others.go new file mode 100644 index 0000000..9905dd5 --- /dev/null +++ b/cmd/cli/dns_config_others.go @@ -0,0 +1,9 @@ +//go:build !darwin + +package cli + +// runSCUtilDNS has no source of a resolver table outside macOS, so the poller +// finds no change and logs nothing. +func runSCUtilDNS() ([]dnsResolverEntry, error) { + return nil, nil +} diff --git a/cmd/cli/dns_config_test.go b/cmd/cli/dns_config_test.go new file mode 100644 index 0000000..b389ad0 --- /dev/null +++ b/cmd/cli/dns_config_test.go @@ -0,0 +1,598 @@ +package cli + +import ( + "errors" + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +const scutilDNSFixture = `DNS configuration + +resolver #1 + search domain[0] : corp.example + nameserver[0] : 192.168.50.200 + nameserver[1] : 192.168.50.1 + if_index : 16 (en0) + flags : Request A records, Request AAAA records + reach : 0x00020002 (Reachable,Directly Reachable Address) + order : 200000 + +resolver #2 + nameserver[0] : 100.100.100.100 + if_index : 28 (utun4) + flags : Supplemental, Request A records, Request AAAA records + reach : 0x00000003 (Reachable,Transient Connection) + order : 101600 + +resolver #3 + domain : local + options : mdns + timeout : 5 + flags : Request A records + reach : 0x00000000 (Not Reachable) + order : 300000 + +DNS configuration (for scoped queries) + +resolver #1 + nameserver[0] : 192.168.50.200 + if_index : 16 (en0) + flags : Scoped, Request A records + reach : 0x00020002 (Reachable,Directly Reachable Address) +` + +const scutilDNSScopedOnlyFixture = `DNS configuration + +DNS configuration (for scoped queries) + +resolver #1 + nameserver[0] : 192.168.50.200 + if_index : 16 (en0) + flags : Scoped, Request A records + reach : 0x00020002 (Reachable,Directly Reachable Address) +` + +var errStubResolverRun = errors.New("scutil failed") + +func wifiResolverEntry() dnsResolverEntry { + return dnsResolverEntry{ + Order: 200000, + Nameservers: []string{"192.168.50.200", "192.168.50.1"}, + IfIndex: 16, + Interface: "en0", + Flags: "Request A records, Request AAAA records", + SearchDomains: []string{"corp.example"}, + Reachable: "Reachable,Directly Reachable Address", + } +} + +func vpnResolverEntry() dnsResolverEntry { + return dnsResolverEntry{ + Order: 101600, + Nameservers: []string{"100.100.100.100"}, + IfIndex: 28, + Interface: "utun4", + Flags: "Supplemental, Request A records, Request AAAA records", + Reachable: "Reachable,Transient Connection", + } +} + +func mdnsResolverEntry() dnsResolverEntry { + return dnsResolverEntry{ + Order: 300000, + Flags: "Request A records", + Reachable: "Not Reachable", + } +} + +func Test_parseSCUtilDNS(t *testing.T) { + tests := []struct { + name string + input string + want []dnsResolverEntry + }{ + { + name: "both tables", + input: scutilDNSFixture, + want: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry(), mdnsResolverEntry(), { + Nameservers: []string{"192.168.50.200"}, + IfIndex: 16, + Interface: "en0", + Flags: "Scoped, Request A records", + Reachable: "Reachable,Directly Reachable Address", + Scoped: true, + }}, + }, + { + name: "scoped table when the first is empty", + input: scutilDNSScopedOnlyFixture, + want: []dnsResolverEntry{{ + Nameservers: []string{"192.168.50.200"}, + IfIndex: 16, + Interface: "en0", + Flags: "Scoped, Request A records", + Reachable: "Reachable,Directly Reachable Address", + Scoped: true, + }}, + }, + { + name: "no output", + input: "", + want: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := parseSCUtilDNS(strings.NewReader(tc.input)) + if err != nil { + t.Fatalf("parseSCUtilDNS() error = %v", err) + } + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("parseSCUtilDNS() = %+v, want %+v", got, tc.want) + } + }) + } +} + +func Test_diffResolverTables(t *testing.T) { + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + vanishedVPN := vpnResolverEntry() + vanishedVPN.Nameservers = nil + vanishedVPN.Action = resolverActionRemoved + appearedVPN := vpnResolverEntry() + appearedVPN.Action = resolverActionAdded + + tests := []struct { + name string + before []dnsResolverEntry + after []dnsResolverEntry + want []dnsResolverEntry + }{ + { + name: "no change", + before: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + after: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + want: nil, + }, + { + name: "one resolver changed", + before: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + after: []dnsResolverEntry{changedWiFi, vpnResolverEntry()}, + want: []dnsResolverEntry{changedWiFi}, + }, + { + name: "one resolver vanished", + before: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + after: []dnsResolverEntry{wifiResolverEntry()}, + want: []dnsResolverEntry{vanishedVPN}, + }, + { + name: "one resolver appeared", + before: []dnsResolverEntry{wifiResolverEntry()}, + after: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + want: []dnsResolverEntry{appearedVPN}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := diffResolverTables(tc.before, tc.after) + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("diffResolverTables() = %+v, want %+v", got, tc.want) + } + }) + } +} + +func Test_diffResolverTablesKeepsTheInput(t *testing.T) { + before := []dnsResolverEntry{vpnResolverEntry()} + diffResolverTables(before, nil) + if !reflect.DeepEqual(before[0], vpnResolverEntry()) { + t.Fatalf("before entry = %+v, want %+v", before[0], vpnResolverEntry()) + } +} + +// resolverRunResult is one answer of the stub runner. +type resolverRunResult struct { + table []dnsResolverEntry + err error +} + +// stubResolverRunner answers one result per call and repeats the last one. +type stubResolverRunner struct { + mu sync.Mutex + results []resolverRunResult + calls int +} + +func (s *stubResolverRunner) run() ([]dnsResolverEntry, error) { + s.mu.Lock() + defer s.mu.Unlock() + result := s.results[min(s.calls, len(s.results)-1)] + s.calls++ + return result.table, result.err +} + +func Test_dnsConfigPollerNextDelay(t *testing.T) { + base := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + tests := []struct { + name string + activity bool + elapsed time.Duration + want time.Duration + }{ + {name: "nine minutes after activity", activity: true, elapsed: 9 * time.Minute, want: dnsConfigFastInterval}, + {name: "eleven minutes after activity", activity: true, elapsed: 11 * time.Minute, want: dnsConfigSlowInterval}, + {name: "no activity yet", elapsed: time.Minute, want: dnsConfigSlowInterval}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + poller := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil }) + if tc.activity { + poller.noteActivity(base) + } + if got := poller.nextDelay(base.Add(tc.elapsed)); got != tc.want { + t.Fatalf("nextDelay() = %s, want %s", got, tc.want) + } + }) + } +} + +func Test_dnsConfigPollerPollOnce(t *testing.T) { + now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + baseline := []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()} + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + runner := &stubResolverRunner{results: []resolverRunResult{ + {table: baseline}, + {err: errStubResolverRun}, + {table: baseline}, + {table: []dnsResolverEntry{changedWiFi, vpnResolverEntry()}}, + }} + poller := newDNSConfigPoller(runner.run) + + if got := poller.pollOnce(now); got != nil { + t.Fatalf("baseline poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(time.Minute)); got != nil { + t.Fatalf("failed poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(2 * time.Minute)); got != nil { + t.Fatalf("poll after a failure = %+v, want nil", got) + } + want := []dnsResolverEntry{changedWiFi} + if got := poller.pollOnce(now.Add(3 * time.Minute)); !reflect.DeepEqual(got, want) { + t.Fatalf("changed poll = %+v, want %+v", got, want) + } +} + +func Test_dnsConfigPollerLoop(t *testing.T) { + previous := dnsConfigDelayFn + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return time.Millisecond } + t.Cleanup(func() { dnsConfigDelayFn = previous }) + + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + runner := &stubResolverRunner{results: []resolverRunResult{ + {table: []dnsResolverEntry{wifiResolverEntry()}}, + {table: []dnsResolverEntry{changedWiFi}}, + }} + poller := newDNSConfigPoller(runner.run) + + emitted := make(chan []dnsResolverEntry, 4) + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + poller.loop(stop, func(changed []dnsResolverEntry) { emitted <- changed }) + }() + + select { + case got := <-emitted: + want := []dnsResolverEntry{changedWiFi} + if !reflect.DeepEqual(got, want) { + t.Fatalf("emitted = %+v, want %+v", got, want) + } + case <-time.After(5 * time.Second): + t.Fatal("the loop emitted no change") + } + + close(stop) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the loop did not stop") + } +} + +// Test_dnsConfigPollerLoopWakesOnActivity proves that a network event does not +// wait for a sleeping timer. The delay is one hour until the event arrives, so +// a loop without the kick would poll after the test deadline. +func Test_dnsConfigPollerLoopWakesOnActivity(t *testing.T) { + var kicked atomic.Bool + parked := make(chan struct{}, 1) + previous := dnsConfigDelayFn + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { + if kicked.Load() { + return time.Millisecond + } + select { + case parked <- struct{}{}: + default: + } + return time.Hour + } + t.Cleanup(func() { dnsConfigDelayFn = previous }) + + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + runner := &stubResolverRunner{results: []resolverRunResult{ + {table: []dnsResolverEntry{wifiResolverEntry()}}, + {table: []dnsResolverEntry{changedWiFi}}, + }} + poller := newDNSConfigPoller(runner.run) + + emitted := make(chan []dnsResolverEntry, 4) + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + poller.loop(stop, func(changed []dnsResolverEntry) { emitted <- changed }) + }() + t.Cleanup(func() { + close(stop) + <-done + }) + + // The kick matters only once the loop waits on the long delay. + select { + case <-parked: + case <-time.After(5 * time.Second): + t.Fatal("the loop did not park on the long delay") + } + kicked.Store(true) + poller.noteActivity(time.Now()) + + select { + case got := <-emitted: + want := []dnsResolverEntry{changedWiFi} + if !reflect.DeepEqual(got, want) { + t.Fatalf("emitted = %+v, want %+v", got, want) + } + case <-time.After(5 * time.Second): + t.Fatal("the loop polled no table after the activity") + } +} + +// Test_dnsConfigPollerLoopTakesABaseline proves that the loop reads the table +// once at entry. Without that read the first change after the start reports +// the whole table. +func Test_dnsConfigPollerLoopTakesABaseline(t *testing.T) { + previous := dnsConfigDelayFn + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return time.Hour } + t.Cleanup(func() { dnsConfigDelayFn = previous }) + + runner := &stubResolverRunner{results: []resolverRunResult{{table: []dnsResolverEntry{wifiResolverEntry()}}}} + poller := newDNSConfigPoller(runner.run) + + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + poller.loop(stop, func([]dnsResolverEntry) { t.Error("the baseline read reported a change") }) + }() + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + runner.mu.Lock() + calls := runner.calls + runner.mu.Unlock() + if calls > 0 { + break + } + time.Sleep(time.Millisecond) + } + close(stop) + <-done + + runner.mu.Lock() + defer runner.mu.Unlock() + if runner.calls != 1 { + t.Fatalf("reads at the loop entry: got %d, want 1", runner.calls) + } + if !poller.started { + t.Fatal("the loop took no baseline") + } +} + +// oversizedSCUtilDNS is a table with one line that no scanner buffer holds. A +// stalled or hostile process produces output of this shape. +func oversizedSCUtilDNS() string { + return "DNS configuration\n\nresolver #1\n nameserver[0] : " + strings.Repeat("9", 70*1024) + "\n" +} + +// Test_parseSCUtilDNSRejectsAnOversizedLine proves that a line the scanner +// cannot hold ends the read with an error. A part of a table must not pass for +// the table of the host. +func Test_parseSCUtilDNSRejectsAnOversizedLine(t *testing.T) { + got, err := parseSCUtilDNS(strings.NewReader(oversizedSCUtilDNS())) + if err == nil { + t.Fatal("parseSCUtilDNS accepted a 70 KiB line") + } + if got != nil { + t.Fatalf("parseSCUtilDNS() = %+v, want nil", got) + } +} + +// Test_parseSCUtilDNSRejectsAnOversizedOutput proves that output past the +// bound ends the read with an error instead of a table cut in the middle. +func Test_parseSCUtilDNSRejectsAnOversizedOutput(t *testing.T) { + block := "resolver #1\n nameserver[0] : 192.168.50.200\n order : 200000\n" + repeats := scutilDNSOutputLimit/len(block) + 2 + got, err := parseSCUtilDNS(strings.NewReader("DNS configuration\n\n" + strings.Repeat(block, repeats))) + if err == nil { + t.Fatal("parseSCUtilDNS accepted output past the bound") + } + if got != nil { + t.Fatalf("parseSCUtilDNS() = %+v, want nil", got) + } +} + +// Test_dnsConfigPollerKeepsTheTableOnAnOversizedRead proves that a read the +// parser rejects reports nothing and keeps the table of the read before. +func Test_dnsConfigPollerKeepsTheTableOnAnOversizedRead(t *testing.T) { + now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + outputs := []string{scutilDNSFixture, oversizedSCUtilDNS(), scutilDNSFixture} + calls := 0 + poller := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { + output := outputs[min(calls, len(outputs)-1)] + calls++ + return parseSCUtilDNS(strings.NewReader(output)) + }) + + if got := poller.pollOnce(now); got != nil { + t.Fatalf("baseline poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(time.Minute)); got != nil { + t.Fatalf("oversized poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(2 * time.Minute)); got != nil { + t.Fatalf("poll after the oversized read = %+v, want nil", got) + } +} + +// scutilTwoTables renders a host with one unscoped resolver and one scoped +// resolver, the shape of a Mac with two active services. +func scutilTwoTables(unscoped, scoped string) string { + return "DNS configuration\n\nresolver #1\n nameserver[0] : " + unscoped + "\n if_index : 14 (en0)\n" + + " flags : Request A records\n reach : 0x00020002 (Reachable,Directly Reachable Address)\n\n" + + "DNS configuration (for scoped queries)\n\nresolver #1\n nameserver[0] : " + scoped + "\n if_index : 15 (en1)\n" + + " flags : Scoped, Request A records\n reach : 0x00020002 (Reachable,Directly Reachable Address)\n" +} + +// Test_diffResolverTablesSeesAScopedChange changes the scoped resolver of a +// second service while the unscoped resolver stays. The diff must report the +// scoped change, because the two tables are not copies of each other. +func Test_diffResolverTablesSeesAScopedChange(t *testing.T) { + before, err := parseSCUtilDNS(strings.NewReader(scutilTwoTables("192.168.1.1", "10.0.0.1"))) + if err != nil { + t.Fatal(err) + } + after, err := parseSCUtilDNS(strings.NewReader(scutilTwoTables("192.168.1.1", "9.9.9.9"))) + if err != nil { + t.Fatal(err) + } + changed := diffResolverTables(before, after) + if len(changed) != 1 || len(changed[0].Nameservers) != 1 || changed[0].Nameservers[0] != "9.9.9.9" { + t.Fatalf("changed resolvers = %+v, want the scoped resolver with 9.9.9.9", changed) + } + if !changed[0].Scoped || before[0].Scoped || len(before) != 2 { + t.Fatalf("scope marks: changed %v, before %+v; want the scoped resolver marked and the unscoped one not", changed[0].Scoped, before) + } +} + +// Test_dnsConfigPollerPollsUnderContinuousActivity kicks the poller faster +// than its fast interval. A kick must not push the read back again and again, +// or a host in a storm of transitions never reports a resolver change. +func Test_dnsConfigPollerPollsUnderContinuousActivity(t *testing.T) { + var reads atomic.Int64 + changed := wifiResolverEntry() + changed.Nameservers = []string{"9.9.9.9"} + p := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { + if reads.Add(1) == 1 { + return []dnsResolverEntry{wifiResolverEntry()}, nil + } + return []dnsResolverEntry{changed}, nil + }) + origDelay := dnsConfigDelayFn + t.Cleanup(func() { dnsConfigDelayFn = origDelay }) + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return 50 * time.Millisecond } + + var emits atomic.Int64 + stop := make(chan struct{}) + done := p.startLoop(stop, func([]dnsResolverEntry) { emits.Add(1) }) + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + p.noteActivity(time.Now()) + time.Sleep(5 * time.Millisecond) + } + close(stop) + <-done + + if emits.Load() == 0 { + t.Fatalf("no resolver event during 500 ms of activity (reads %d)", reads.Load()) + } +} + +// scutilScopedTable renders the scoped table the way macOS prints it: one +// resolver per interface and no order line. +func scutilScopedTable(resolvers ...[2]string) string { + var b strings.Builder + b.WriteString("DNS configuration\n\nresolver #1\n nameserver[0] : 100.100.100.100\n if_index : 28 (utun4)\n flags : Request A records, Request AAAA records\n reach : 0x00000003 (Reachable,Transient Connection)\n order : 200000\n\n") + b.WriteString("DNS configuration (for scoped queries)\n\n") + for i, resolver := range resolvers { + fmt.Fprintf(&b, "resolver #%d\n nameserver[0] : %s\n if_index : %s\n flags : Scoped, Request A records\n reach : 0x00020002 (Reachable,Directly Reachable Address)\n\n", i+1, resolver[1], resolver[0]) + } + return b.String() +} + +// Test_diffResolverTablesKeysScopedResolversByInterface inserts a scoped +// resolver between two others and removes it again. The scoped table has no +// order line, so the interface is the identity, and the diff must name the +// resolver that came and went, not its neighbors. +func Test_diffResolverTablesKeysScopedResolversByInterface(t *testing.T) { + en0 := [2]string{"16 (en0)", "192.168.50.200"} + en7 := [2]string{"30 (en7)", "10.9.9.1"} + utun := [2]string{"28 (utun4)", "100.100.100.100"} + before, err := parseSCUtilDNS(strings.NewReader(scutilScopedTable(en0, utun))) + if err != nil { + t.Fatal(err) + } + after, err := parseSCUtilDNS(strings.NewReader(scutilScopedTable(en0, en7, utun))) + if err != nil { + t.Fatal(err) + } + + inserted := diffResolverTables(before, after) + if len(inserted) != 1 || inserted[0].Interface != "en7" || inserted[0].Nameservers == nil { + t.Fatalf("insert of en7 reported %+v, want en7 alone with its nameserver", inserted) + } + removed := diffResolverTables(after, before) + if len(removed) != 1 || removed[0].Interface != "en7" || removed[0].Nameservers != nil { + t.Fatalf("removal of en7 reported %+v, want en7 alone without nameservers", removed) + } +} + +// Test_diffResolverTablesNamesTheAction tells a resolver that came from one +// that went. An mDNS resolver has no nameservers, so without the action the +// two events read the same. +func Test_diffResolverTablesNamesTheAction(t *testing.T) { + mdns := "DNS configuration\n\nresolver #1\n domain : local\n options : mdns\n timeout : 5\n flags : Request A records, Request AAAA records\n reach : 0x00000000 (Not Reachable)\n order : 300000\n" + table, err := parseSCUtilDNS(strings.NewReader(mdns)) + if err != nil { + t.Fatal(err) + } + appeared := diffResolverTables(nil, table) + vanished := diffResolverTables(table, nil) + if len(appeared) != 1 || appeared[0].Action != resolverActionAdded { + t.Fatalf("appeared = %+v, want one resolver with action added", appeared) + } + if len(vanished) != 1 || vanished[0].Action != resolverActionRemoved { + t.Fatalf("vanished = %+v, want one resolver with action removed", vanished) + } + moved := wifiResolverEntry() + moved.Nameservers = []string{"9.9.9.9"} + changed := diffResolverTables([]dnsResolverEntry{wifiResolverEntry()}, []dnsResolverEntry{moved}) + if len(changed) != 1 || changed[0].Action != resolverActionChanged { + t.Fatalf("changed = %+v, want one resolver with action changed", changed) + } +} diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go index 807ec31..95748b8 100644 --- a/cmd/cli/dns_intercept_darwin.go +++ b/cmd/cli/dns_intercept_darwin.go @@ -267,8 +267,9 @@ func (p *prog) startDNSIntercept() error { } // Store the initial set of tunnel interfaces so we can detect changes later. + tunnels := p.activeTunnelInterfaces() p.mu.Lock() - p.lastTunnelIfaces = discoverTunnelInterfacesForReconcile() + p.lastTunnelIfaces = tunnels p.pendingTunnelIfaces = nil p.hasPendingTunnelIfaces = false p.mu.Unlock() @@ -764,14 +765,29 @@ func discoverTunnelInterfaces() []string { } } - if len(tunnels) > 0 { - mainLog.Load().Debug().Msgf("DNS intercept: discovered active tunnel interfaces: %v", tunnels) - } return tunnels } var discoverTunnelInterfacesForReconcile = discoverTunnelInterfaces +// tunnelInterfacesKey holds the last discovered tunnel set, so the discovery +// line reports a change and not every reconcile. +const tunnelInterfacesKey = "tunnel_interfaces" + +// activeTunnelInterfaces returns the live tunnel interfaces and names them on +// change. Every pf reconcile discovers them again, and the set is the same in +// almost every one of those reads. +func (p *prog) activeTunnelInterfaces() []string { + tunnels := discoverTunnelInterfacesForReconcile() + changed, repeats := p.repeats.changed(tunnelInterfacesKey, fmt.Sprintf("%v", tunnels)) + if !changed || len(tunnels) == 0 { + return tunnels + } + mainLog.Load().Debug().Uint64("repeats", repeats). + Msgf("DNS intercept: discovered active tunnel interfaces: %v", tunnels) + return tunnels +} + // dnsInterceptSupported reports whether DNS intercept mode is supported on this platform. func dnsInterceptSupported() bool { _, err := exec.LookPath("pfctl") @@ -843,7 +859,7 @@ func (p *prog) buildPFAnchorRules(vpnExemptions []vpnDNSExemption) string { // had already taken effect. Passing the set in lets each path record exactly what pf // accepted. func (p *prog) buildPFAnchorRulesWith(vpnExemptions []vpnDNSExemption, forwardedSources []forwardedSource) string { - return p.buildPFAnchorRulesForState(vpnExemptions, discoverTunnelInterfacesForReconcile(), forwardedSources) + return p.buildPFAnchorRulesForState(vpnExemptions, p.activeTunnelInterfaces(), forwardedSources) } func (p *prog) buildPFAnchorRulesForState(vpnExemptions []vpnDNSExemption, tunnelIfaces []string, forwardedSources []forwardedSource) string { @@ -1200,11 +1216,12 @@ var ( restorePFAnchorForReconcile = func(p *prog, reason string) pfAnchorCheckResult { return p.restorePFAnchor(reason) } + verifyPFStateFn = (*prog).verifyPFState pfShutdownStateRevokedForTest = func() {} ) func (p *prog) rebuildPFAnchorRules(vpnExemptions []vpnDNSExemption) ([]string, error) { - tunnelIfaces := append([]string(nil), discoverTunnelInterfacesForReconcile()...) + tunnelIfaces := append([]string(nil), p.activeTunnelInterfaces()...) forwarded := p.currentForwardedSources() rulesStr := p.buildPFAnchorRulesForState(vpnExemptions, tunnelIfaces, forwarded) if err := writePFAnchorFile(rulesStr); err != nil { @@ -1257,7 +1274,7 @@ func (p *prog) restorePFAnchorWithTransportReset(reason string, resetTransports if resetTransports { p.resetUpstreamTransports() } - if !p.verifyPFState() { + if !verifyPFStateFn(p) { mainLog.Load().Error().Str("reason", reason).Msg("DNS intercept: rebuilt anchor failed post-load verification") return pfAnchorCheckFailed } @@ -1267,6 +1284,7 @@ func (p *prog) restorePFAnchorWithTransportReset(reason string, resetTransports p.commitPFReconcileState(tunnelIfaces) p.pfLastRestoreTime.Store(time.Now().UnixMilli()) mainLog.Load().Info().Str("reason", reason).Msg("DNS intercept: pf anchor restored successfully") + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{}) return pfAnchorCheckRestored } @@ -1317,7 +1335,8 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return false } - current := append([]string(nil), discoverTunnelInterfacesForReconcile()...) + current := append([]string(nil), p.activeTunnelInterfaces()...) + p.noteTunnelInterfaceSet(current) p.mu.Lock() prev := append([]string(nil), p.lastTunnelIfaces...) @@ -1352,18 +1371,9 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return true } - // Detect NEW tunnel interfaces (not just any change). - prevSet := make(map[string]bool, len(prev)) - for _, iface := range prev { - prevSet[iface] = true - } - hasNewTunnel := false - for _, iface := range current { - if !prevSet[iface] { - hasNewTunnel = true - mainLog.Load().Info().Msgf("DNS intercept: new tunnel interface detected: %s", iface) - break - } + added, _ := tunnelSetDiff(prev, current) + if len(added) > 0 { + mainLog.Load().Info().Msgf("DNS intercept: new tunnel interface detected: %s", added[0]) } if stabilizing { @@ -1371,14 +1381,13 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return newObservation } - if hasNewTunnel { + if len(added) > 0 { // A new VPN tunnel appeared. The dedicated post-stabilization repair path // rebuilds the anchor if no successful exemption update applies it first. p.pfStartStabilization() return newObservation } - mainLog.Load().Info().Msgf("DNS intercept: tunnel interfaces changed (was %v, now %v) — rebuilding pf anchor rules", prev, current) result := p.reconcilePFAnchorForTunnelChange() if result != pfAnchorCheckRestored { // Keep the desired tunnel set pending. The next event, delayed check, or @@ -1388,6 +1397,70 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return newObservation || result == pfAnchorCheckRestored } +// tunnelChangedMessage names the tunnel event of the journal. +const tunnelChangedMessage = "Tunnel interface changed" + +// noteTunnelInterfaceSet logs one event for each change of the tunnel set. The +// reconcile retries a set until it succeeds, and a set that returns to the one +// before it is news as well, so the diff runs against the set of the last +// event and not against the applied set. +func (p *prog) noteTunnelInterfaceSet(current []string) { + p.mu.Lock() + // Before the first event the applied set is the baseline, because no event + // named a set yet. + previous := p.lastLoggedTunnelIfaces + if previous == nil { + previous = p.lastTunnelIfaces + } + previous = append([]string(nil), previous...) + if stringSlicesEqual(previous, current) { + p.mu.Unlock() + return + } + p.lastLoggedTunnelIfaces = append([]string{}, current...) + p.mu.Unlock() + + added, removed := tunnelSetDiff(previous, current) + p.logTunnelInterfaceChange(added, removed) +} + +// logTunnelInterfaceChange names the tunnels that appeared and the ones that +// went away. The owner names the program behind a new tunnel, because its pf +// rules compete with the ctrld anchor for the same DNS packets. +func (p *prog) logTunnelInterfaceChange(added, removed []string) { + owner := "" + if len(added) > 0 { + owner = tunnelOwner(added[0]) + } + journal(mainLog.Load().Info()). + Strs("added", added). + Strs("removed", removed). + Str("owner", owner). + Msg(tunnelChangedMessage) +} + +// tunnelSetDiff reports the tunnels that appeared and the ones that went away, +// each in the order of the set it comes from. +func tunnelSetDiff(prev, current []string) (added, removed []string) { + before := make(map[string]bool, len(prev)) + for _, iface := range prev { + before[iface] = true + } + live := make(map[string]bool, len(current)) + for _, iface := range current { + live[iface] = true + if !before[iface] { + added = append(added, iface) + } + } + for _, iface := range prev { + if !live[iface] { + removed = append(removed, iface) + } + } + return added, removed +} + // stringSlicesEqual reports whether two string slices have the same elements in the same order. func stringSlicesEqual(a, b []string) bool { if len(a) != len(b) { @@ -1416,7 +1489,8 @@ func (p *prog) pfStartStabilization() { stableRequired = 45 * time.Second } - mainLog.Load().Info().Msgf("DNS intercept: VPN connecting — entering stabilization mode (waiting %s for pf to settle)", stableRequired) + journal(mainLog.Load().Info()).Msgf("DNS intercept: VPN connecting — entering stabilization mode (waiting %s for pf to settle)", stableRequired) + p.logPFAnchorList(pfAnchorReasonStabilizationStart, mainLog.Load().Info(), pfRuleDump{}) ctx, cancel := context.WithCancel(context.Background()) p.mu.Lock() @@ -1495,7 +1569,8 @@ func (p *prog) pfStabilizationLoopWithMaxWait(ctx context.Context, stableRequire // held still for the required window. The active loop retains ownership throughout: it // never re-enters stabilization recursively. func (p *prog) finishPFStabilization(stableRequired time.Duration) { - mainLog.Load().Info().Msgf("DNS intercept: pf stable for %s — reconciling anchor rules", stableRequired) + journal(mainLog.Load().Info()).Msgf("DNS intercept: pf stable for %s — reconciling anchor rules", stableRequired) + p.logPFAnchorList(pfAnchorReasonStabilizationEnd, mainLog.Load().Info(), pfRuleDump{}) result := p.reconcilePFAnchorAfterStabilization() if result != pfAnchorCheckRestored && result != pfAnchorCheckIntact { p.scheduleDelayedRechecks() @@ -1662,6 +1737,11 @@ func (p *prog) verifyInterceptAfterWake(gap time.Duration) { return } mainLog.Load().Info().Msgf("DNS intercept: host resumed after a %s gap - verifying interception is still translating", gap.Round(time.Second)) + p.noteHostWoke("detector", gap, freshNetworkState()) + // A resume moves the resolvers of the host, so the poll goes fast again. + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(networkEventsNowFn()) + } repairs := 0 probed := 0 @@ -1734,7 +1814,7 @@ func (p *prog) waitBeforeWakeProbe(delay time.Duration) bool { // could not run does not count: nothing was changed, so the next attempt may try again. // The caller holds functional-probe ownership for the call. func (p *prog) runWakeProbeAttempt(attempt, repairs int) (restored, repaired bool) { - initializeOsResolver(ctrld.LoggerCtx(context.Background(), mainLog.Load()), true) + initializeOsResolver(ctrld.LoggerCtx(context.Background(), mainLog.Load()), true, osResolverReasonWakeProbe) result := probePFInterceptFn(p) if result.result == pfProbeIndeterminate { mainLog.Load().Debug().Msg("DNS intercept: post-wake probe indeterminate; repair deferred") @@ -1775,6 +1855,77 @@ var runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { return exec.Command("pfctl", args...).CombinedOutput() } +const ( + // pfAnchorListMessage names the anchor state event of the journal. + pfAnchorListMessage = "PF anchor list changed" + + pfAnchorReasonStabilizationStart = "stabilization_start" + pfAnchorReasonStabilizationEnd = "stabilization_end" + pfAnchorReasonMissing = "missing" + pfAnchorReasonRestored = "restored" + + // pfAnchorStateKey holds the last anchor state, so an anchor that stays + // missing for a whole storm reaches the journal once. + pfAnchorStateKey = "pf_anchor_state" + + // pfAnchorWatchdogKey holds the last state the watchdog saw, so the healthy + // line reports the recovery and not every tick between two of them. + pfAnchorWatchdogKey = "pf_anchor_intact" + + pfAnchorIntactState = "pf anchor intact" + pfAnchorMissingState = "pf anchor missing" + + pfAnchorIntactMessage = "DNS intercept watchdog: " + pfAnchorIntactState +) + +// logPFAnchorList reports which anchors the running ruleset refers to. A capture +// of an outage has to tell whether ctrld or another program owned pf, and the +// anchor names are the only evidence of that. +func (p *prog) logPFAnchorList(reason string, level *ctrld.LogEvent, dump pfRuleDump) { + anchors := p.pfAnchorNames(dump) + enabled, since := p.pfStatus() + if changed, _ := p.repeats.changed(pfAnchorStateKey, pfAnchorStateValue(reason, anchors, enabled)); !changed { + return + } + event := journal(level). + Str("reason", reason). + Bool("pf_enabled", enabled). + Str("pf_since", since) + if anchors == nil { + event.Bool("anchors_known", false).Msg(pfAnchorListMessage) + return + } + event.Strs("anchors", anchors).Msg(pfAnchorListMessage) +} + +// pfAnchorStateValue leaves the pf uptime out, because it grows between two +// reads of the same state. +func pfAnchorStateValue(reason string, anchors []string, enabled bool) string { + if anchors == nil { + return fmt.Sprintf("%s|%t|unknown", reason, enabled) + } + return fmt.Sprintf("%s|%t|%s", reason, enabled, strings.Join(anchors, ",")) +} + +// logPFAnchorIntact reports a healthy anchor. The watchdog finds it healthy on +// almost every tick for the life of the daemon, so only a change belongs in the +// log. +func (p *prog) logPFAnchorIntact() { + changed, repeats := p.repeats.changed(pfAnchorWatchdogKey, pfAnchorIntactState) + if !changed { + return + } + mainLog.Load().Debug().Uint64("repeats", repeats).Msg(pfAnchorIntactMessage) +} + +// notePFAnchorMissing reports the wiped anchor with the anchors that replaced +// it. The record of the missing state makes the next healthy check reach the +// log, so a capture holds both ends of the outage. +func (p *prog) notePFAnchorMissing(dump pfRuleDump) { + p.repeats.changed(pfAnchorWatchdogKey, pfAnchorMissingState) + p.logPFAnchorList(pfAnchorReasonMissing, mainLog.Load().Warn(), dump) +} + // ensurePFAnchorActive checks live PF state and returns an explicit outcome for // callers that must distinguish intact, restored, deferred, skipped, and failed. func (p *prog) ensurePFAnchorActive() pfAnchorCheckResult { @@ -1816,6 +1967,9 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu rdrAnchorRef := fmt.Sprintf("rdr-anchor \"%s\"", pfAnchorName) anchorRef := fmt.Sprintf("anchor \"%s\"", pfAnchorName) needsRestore := false + // The anchor event parses the rules this check reads, so it starts no + // second pair of pfctl processes. + var dump pfRuleDump // Check 1: anchor references in the main ruleset. natOut, err := runPFAnchorCheckCommand("-sn") @@ -1824,7 +1978,9 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump NAT rules") return pfAnchorCheckFailed } - if !strings.Contains(string(natOut), rdrAnchorRef) { + dump.nat = natOut + natStr := string(natOut) + if !strings.Contains(natStr, rdrAnchorRef) { mainLog.Load().Warn().Msg("DNS intercept watchdog: rdr-anchor reference missing from running ruleset") needsRestore = true } @@ -1836,6 +1992,7 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump filter rules") return pfAnchorCheckFailed } + dump.rules = filterOut if !strings.Contains(string(filterOut), anchorRef) { mainLog.Load().Warn().Msg("DNS intercept watchdog: anchor reference missing from running filter rules") needsRestore = true @@ -1895,6 +2052,7 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu p.pfBackoffMultiplier.Add(1) mainLog.Load().Warn().Msgf("DNS intercept: rules wiped %s after restore — entering stabilization (backoff multiplier: %d)", elapsed, p.pfBackoffMultiplier.Load()) + p.notePFAnchorMissing(dump) p.pfStartStabilization() return pfAnchorCheckDeferred } @@ -1904,7 +2062,7 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu } if !needsRestore && !forceRebuild { - mainLog.Load().Debug().Msg("DNS intercept watchdog: pf anchor intact") + p.logPFAnchorIntact() return pfAnchorCheckIntact } @@ -1912,6 +2070,9 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu if forceRebuild && !needsRestore { reason = "post-stabilization rebuild" } + if needsRestore { + p.notePFAnchorMissing(dump) + } return restorePFAnchorForReconcile(p, reason) } @@ -2036,9 +2197,9 @@ func (p *prog) scheduleDelayedRechecks() { p.checkTunnelInterfaceChanges() // Refresh OS resolver — VPN may have finished DNS cleanup since the // immediate handler ran. This clears stale LAN nameservers (e.g., - // Windscribe's 10.255.255.3 lingering in scutil --dns). + // a VPN's DNS IP (e.g., 10.255.255.3) lingering in scutil --dns). ctx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) - ctrld.InitializeOsResolver(ctx, true) + ctrld.InitializeOsResolverWithReason(ctx, true, osResolverReasonDelayedRecheck) if p.vpnDNS != nil { p.vpnDNS.Refresh(ctx, true) } @@ -2188,7 +2349,7 @@ func (p *prog) exemptVPNDNSServers(exemptions []vpnDNSExemption) error { defer p.pfEnsureRunning.Store(false) forwarded := p.currentForwardedSources() - tunnelIfaces := append([]string(nil), discoverTunnelInterfacesForReconcile()...) + tunnelIfaces := append([]string(nil), p.activeTunnelInterfaces()...) rulesStr := p.buildPFAnchorRulesForState(exemptions, tunnelIfaces, forwarded) if err := writePFAnchorFile(rulesStr); err != nil { diff --git a/cmd/cli/dns_intercept_darwin_test.go b/cmd/cli/dns_intercept_darwin_test.go index 0faa154..a01e669 100644 --- a/cmd/cli/dns_intercept_darwin_test.go +++ b/cmd/cli/dns_intercept_darwin_test.go @@ -6,9 +6,12 @@ import ( "context" "errors" "fmt" + "net" "os" "path/filepath" + "slices" "strings" + "sync" "testing" "time" @@ -987,7 +990,7 @@ func TestFinishPFStabilizationRunsFunctionalVerification(t *testing.T) { "-a com.controld.ctrld -sn": "rdr on lo0", }) originalResolver := initializeOsResolver - initializeOsResolver = func(context.Context, bool) []string { return nil } + initializeOsResolver = func(context.Context, bool, string) []string { return nil } t.Cleanup(func() { initializeOsResolver = originalResolver }) probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) @@ -1092,7 +1095,7 @@ func stubWakeProbeSchedule(t *testing.T, attempts int) (refreshes *int) { pfWakeProbeOwnerWait = 10 * time.Millisecond calls := 0 - initializeOsResolver = func(context.Context, bool) []string { + initializeOsResolver = func(context.Context, bool, string) []string { calls++ return []string{"10.0.0.1:53"} } @@ -1275,7 +1278,7 @@ func TestWakeProbeStopsWhenServiceStops(t *testing.T) { pfWakeProbeDelays = []time.Duration{0, time.Hour} t.Cleanup(func() { pfWakeProbeDelays = originalDelays }) originalResolver := initializeOsResolver - initializeOsResolver = func(context.Context, bool) []string { return []string{"10.0.0.1:53"} } + initializeOsResolver = func(context.Context, bool, string) []string { return []string{"10.0.0.1:53"} } t.Cleanup(func() { initializeOsResolver = originalResolver }) probes, _ := stubStabilizationProbe(t, nil, false) // first probe fails, reload refuses @@ -1433,3 +1436,531 @@ func TestStartDNSInterceptSkipsBackgroundWorkWhenInstallFails(t *testing.T) { // wiring a moment to report rather than racing the test's end. time.Sleep(50 * time.Millisecond) } + +// ============================================================================= +// pf anchor state, tunnel, and wake event tests +// ============================================================================= + +// pfStateCommandsMu guards the answer of the pf and tunnel state reads, because +// a watchdog or stabilization goroutine can read while a test replaces it. +var ( + pfStateCommandsMu sync.Mutex + pfStateCommandFn func(string, ...string) ([]byte, error) +) + +// init keeps the whole test binary away from pfctl and ifconfig. A test that +// needs values stubs them with stubPFStateCommands. +func init() { + pfStateRunCommand = func(name string, args ...string) ([]byte, error) { + pfStateCommandsMu.Lock() + run := pfStateCommandFn + pfStateCommandsMu.Unlock() + if run == nil { + return nil, errors.New("pf state read is not stubbed") + } + return run(name, args...) + } +} + +// stubPFStateCommands answers the pf and tunnel state reads from fixture text, +// keyed by the whole command line. +func stubPFStateCommands(t *testing.T, outputs map[string]string) { + t.Helper() + stubPFStateCommandFn(t, func(name string, args ...string) ([]byte, error) { + key := strings.Join(append([]string{name}, args...), " ") + output, ok := outputs[key] + if !ok { + return nil, fmt.Errorf("no fixture for %q", key) + } + return []byte(output), nil + }) +} + +// stubPFStateCommandFn answers the pf and tunnel state reads from run. +func stubPFStateCommandFn(t *testing.T, run func(string, ...string) ([]byte, error)) { + t.Helper() + pfStateCommandsMu.Lock() + original := pfStateCommandFn + pfStateCommandFn = run + pfStateCommandsMu.Unlock() + t.Cleanup(func() { + pfStateCommandsMu.Lock() + pfStateCommandFn = original + pfStateCommandsMu.Unlock() + }) +} + +// pfIntactAnchorRules answers the anchor check of a healthy ruleset. +var pfIntactAnchorRules = map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", +} + +// pfStateFixtures answers the anchor list and status reads with a real ruleset. +var pfStateFixtures = map[string]string{ + "pfctl -sr": pfShowRules, + "pfctl -sn": pfShowNAT, + "pfctl -si": pfStatusEnabled, +} + +// waitForStabilizationExit waits for the stabilization loop to release pf. +func waitForStabilizationExit(t *testing.T, p *prog) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for p.pfStabilizing.Load() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if p.pfStabilizing.Load() { + t.Fatal("the stabilization loop did not observe the closed stop channel") + } +} + +// Test_logPFAnchorListLogsStabilizationStartAndEnd covers the anchor list at the +// two stabilization events. A capture of a VPN connect has to name the anchors +// that were loaded before ctrld waited, and the ones that were loaded after. +func Test_logPFAnchorListLogsStabilizationStartAndEnd(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + stubPFAnchorCheckCommand(t, pfIntactAnchorRules) + originalResolver := initializeOsResolver + initializeOsResolver = func(context.Context, bool, string) []string { return nil } + t.Cleanup(func() { initializeOsResolver = originalResolver }) + stubStabilizationProbe(t, []bool{true}, true) + + stopCh := make(chan struct{}) + close(stopCh) + p := &prog{dnsInterceptState: &pfState{}, stopCh: stopCh} + p.pfStartStabilization() + waitForStabilizationExit(t, p) + p.pfStabilizing.Store(true) + p.finishPFStabilization(time.Millisecond) + stopPFTestTimers(p) + + events := jsonLogEvents(t, logs, "PF anchor list changed") + if len(events) != 2 { + t.Fatalf("got %d anchor list events for one stabilization, want 2", len(events)) + } + wantField(t, events[0], "reason", "stabilization_start") + wantField(t, events[0], "journal", true) + wantField(t, events[0], "pf_enabled", true) + wantField(t, events[0], "pf_since", "0 days 02:11:05") + wantAnchors(t, events[0], []string{"com.apple/*", "com.controld.ctrld"}) + wantField(t, events[1], "reason", "stabilization_end") + wantField(t, events[1], "journal", true) + + for _, prefix := range []string{ + "DNS intercept: VPN connecting", + "DNS intercept: pf stable for", + } { + lines := eventsWithPrefix(t, logs, prefix) + if len(lines) != 1 { + t.Fatalf("got %d lines with prefix %q, want 1", len(lines), prefix) + } + wantField(t, lines[0], "journal", true) + } +} + +// Test_logPFAnchorListReportsAMissingAnchorOnce covers the watchdog state. The +// anchor stays missing for as long as another program holds pf, and one line per +// tick would fill the journal with the same fact. +func Test_logPFAnchorListReportsAMissingAnchorOnce(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + outputs := pfIntactAnchorRules + original := runPFAnchorCheckCommand + runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { + return []byte(outputs[strings.Join(args, " ")]), nil + } + originalRestore := restorePFAnchorForReconcile + restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult { return pfAnchorCheckRestored } + t.Cleanup(func() { + runPFAnchorCheckCommand = original + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{dnsInterceptState: &pfState{}} + for range 3 { + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckIntact { + t.Fatalf("healthy check result = %v, want intact", result) + } + } + intact := eventsWithPrefix(t, logs, "DNS intercept watchdog: pf anchor intact") + if len(intact) != 1 { + t.Fatalf("got %d intact lines for three healthy checks, want 1", len(intact)) + } + wantField(t, intact[0], "repeats", float64(0)) + + outputs = map[string]string{} + for range 2 { + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckRestored { + t.Fatalf("wiped check result = %v, want restored", result) + } + } + missing := jsonLogEvents(t, logs, "PF anchor list changed") + if len(missing) != 1 { + t.Fatalf("got %d anchor list events for two wiped checks, want 1", len(missing)) + } + wantField(t, missing[0], "reason", "missing") + wantField(t, missing[0], "level", "warn") + wantField(t, missing[0], "journal", true) + + outputs = pfIntactAnchorRules + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckIntact { + t.Fatalf("recovered check result = %v, want intact", result) + } + intact = eventsWithPrefix(t, logs, "DNS intercept watchdog: pf anchor intact") + if len(intact) != 2 { + t.Fatalf("got %d intact lines after the anchor came back, want 2", len(intact)) + } + // The second wiped check repeated the state of the first one, and that held + // line is the count the recovered line carries. + wantField(t, intact[1], "repeats", float64(1)) +} + +// Test_logPFAnchorListReportsARestoredAnchor covers the restore report. The +// missing event alone leaves a capture without the anchors ctrld ended with. +func Test_logPFAnchorListReportsARestoredAnchor(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + originalReference := ensurePFAnchorReferenceForRestore + originalRebuild := rebuildPFAnchorRulesForReconcile + originalVerify := verifyPFStateFn + ensurePFAnchorReferenceForRestore = func(*prog) error { return nil } + rebuildPFAnchorRulesForReconcile = func(*prog, []vpnDNSExemption) ([]string, error) { return nil, nil } + verifyPFStateFn = func(*prog) bool { return true } + t.Cleanup(func() { + ensurePFAnchorReferenceForRestore = originalReference + rebuildPFAnchorRulesForReconcile = originalRebuild + verifyPFStateFn = originalVerify + }) + + p := &prog{dnsInterceptState: &pfState{}} + if result := p.restorePFAnchorWithTransportReset("test", false); result != pfAnchorCheckRestored { + t.Fatalf("restore result = %v, want restored", result) + } + + events := jsonLogEvents(t, logs, "PF anchor list changed") + if len(events) != 1 { + t.Fatalf("got %d anchor list events for one restore, want 1", len(events)) + } + wantField(t, events[0], "reason", "restored") + wantField(t, events[0], "level", "info") + wantField(t, events[0], "journal", true) + wantAnchors(t, events[0], []string{"com.apple/*", "com.controld.ctrld"}) +} + +// Test_logPFAnchorListMarksAnUnknownAnchorList covers a failed read. pfctl fails +// on a host that revoked the privileges of ctrld, and an empty list there reads +// as a ruleset that holds no anchor at all. +func Test_logPFAnchorListMarksAnUnknownAnchorList(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, nil) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{}) + + events := jsonLogEvents(t, logs, "PF anchor list changed") + if len(events) != 1 { + t.Fatalf("got %d anchor list events, want 1", len(events)) + } + wantField(t, events[0], "anchors_known", false) + if _, ok := events[0]["anchors"]; ok { + t.Fatalf("a failed read reported an anchor list: %v", events[0]) + } +} + +// TestTunnelInterfaceChangedNamesTheOwner covers the tunnel event. The #611 +// capture could not tell a VPN tunnel from a mesh client, and the owner is the +// field that names the program whose pf rules compete with the ctrld anchor. +func TestTunnelInterfaceChangedNamesTheOwner(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, map[string]string{"ifconfig -v utun9": ifconfigWindscribeTunnel}) + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return []string{"utun9"} } + t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover }) + + p := &prog{dnsInterceptState: &pfState{}, lastTunnelIfaces: []string{"utun7"}} + // Stabilization owns the rebuild, so the event is all this drives. + p.pfStabilizing.Store(true) + if !p.checkTunnelInterfaceChanges() { + t.Fatal("the tunnel set change was not observed") + } + + events := jsonLogEvents(t, logs, "Tunnel interface changed") + if len(events) != 1 { + t.Fatalf("got %d tunnel events for one set change, want 1", len(events)) + } + wantField(t, events[0], "owner", "Windscribe VPN") + wantField(t, events[0], "journal", true) + wantStrings(t, events[0], "added", []string{"utun9"}) + wantStrings(t, events[0], "removed", []string{"utun7"}) + + discovered := eventsWithPrefix(t, logs, "DNS intercept: discovered active tunnel interfaces") + if len(discovered) != 1 { + t.Fatalf("got %d discovery lines for one tunnel set, want 1", len(discovered)) + } + if p.checkTunnelInterfaceChanges() { + t.Fatal("the identical pending tunnel set was not coalesced") + } + discovered = eventsWithPrefix(t, logs, "DNS intercept: discovered active tunnel interfaces") + if len(discovered) != 1 { + t.Fatalf("got %d discovery lines for an unchanged tunnel set, want 1", len(discovered)) + } +} + +// TestVerifyInterceptAfterWakeReportsTheWake covers the detector source of the +// wake event. The 2026-09-14 capture has no wake line at all, and the resolver +// table after a resume is the state the investigation needs next. +func TestVerifyInterceptAfterWakeReportsTheWake(t *testing.T) { + logs := captureDebugMainLog(t) + stubWakeProbeSchedule(t, 4) + stubStabilizationProbe(t, []bool{true}, true) + now := time.Date(2026, 9, 14, 7, 14, 0, 0, time.UTC) + originalNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return now } + t.Cleanup(func() { networkEventsNowFn = originalNow }) + + p := &prog{ + dnsInterceptState: &pfState{}, + dnsConfig: newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil }), + } + p.verifyInterceptAfterWake(18 * time.Second) + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 { + t.Fatalf("got %d Host woke events for one resume, want 1", len(events)) + } + wantField(t, events[0], "source", "detector") + wantField(t, events[0], "gap_ms", float64(18000)) + wantField(t, events[0], "journal", true) + if got := p.dnsConfig.nextDelay(now); got != dnsConfigFastInterval { + t.Fatalf("DNS configuration poll delay after a wake = %s, want %s", got, dnsConfigFastInterval) + } +} + +func wantAnchors(t *testing.T, event map[string]any, want []string) { + t.Helper() + wantStrings(t, event, "anchors", want) +} + +func wantStrings(t *testing.T, event map[string]any, field string, want []string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("field %q: got %v, want a list", field, event[field]) + } + if len(values) != len(want) { + t.Fatalf("field %q: got %v, want %v", field, values, want) + } + for i, value := range values { + if value != want[i] { + t.Fatalf("field %q: got %v, want %v", field, values, want) + } + } +} + +// TestIgnoredNetworkChangeJournalsTheInterfaceChange proves that an interface +// that appears or disappears on the ignored path enters the journal with its +// class and its action, so a hypervisor adapter storm is readable after the fact. +func TestIgnoredNetworkChangeJournalsTheInterfaceChange(t *testing.T) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + stubSnapshotVirtualSet(t, "bridge101") + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return nil } + t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover }) + + p := &prog{dnsInterceptState: &pfState{}, vpnDNS: newVPNDNSManager(&mainLog, nil)} + p.pfStabilizing.Store(true) + t.Cleanup(func() { + p.pfDelayedRecheckMu.Lock() + defer p.pfDelayedRecheckMu.Unlock() + for _, timer := range p.pfDelayedRecheckTimers { + if timer != nil { + timer.Stop() + } + } + }) + stateWith := func(names ...string) *netmon.State { + state := &netmon.State{Interface: map[string]netmon.Interface{}} + for _, name := range names { + state.Interface[name] = netmon.Interface{Interface: &net.Interface{Name: name, Flags: net.FlagUp}} + } + return state + } + now := time.Unix(1_000_000, 0) + + p.handleDNSInterceptIgnoredNetworkChange(&netmon.ChangeDelta{Old: stateWith("en0"), New: stateWith("en0", "bridge101")}, now) + p.handleDNSInterceptIgnoredNetworkChange(&netmon.ChangeDelta{Old: stateWith("en0", "bridge101"), New: stateWith("en0")}, now.Add(time.Second)) + + events := jsonLogEvents(t, logs, "DNS intercept: interface appeared/disappeared — starting interception probe monitor") + if len(events) != 2 { + t.Fatalf("interface events: got %d, want 2: %s", len(events), logs.String()) + } + for i, action := range []string{"added", "removed"} { + wantField(t, events[i], "journal", true) + wantField(t, events[i], "interface", "bridge101") + wantField(t, events[i], "class", "virtual") + wantField(t, events[i], "action", action) + } +} + +// TestTunnelInterfaceChangedLogsOncePerChange covers the retry path and the +// flap back. The reconcile keeps a new tunnel set pending until it succeeds, +// so every retry repeated the same event, and a set that returned to the one +// before it reported nothing at all. +func TestTunnelInterfaceChangedLogsOncePerChange(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + originalDiscover := discoverTunnelInterfacesForReconcile + originalRestore := restorePFAnchorForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return []string{"utun7"} } + restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult { return pfAnchorCheckFailed } + t.Cleanup(func() { + discoverTunnelInterfacesForReconcile = originalDiscover + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{dnsInterceptState: &pfState{}, lastTunnelIfaces: []string{"utun7", "utun9"}} + for range 3 { + p.checkTunnelInterfaceChanges() + } + + events := jsonLogEvents(t, logs, tunnelChangedMessage) + if len(events) != 1 { + t.Fatalf("got %d tunnel events for three retries of one change, want 1", len(events)) + } + wantStrings(t, events[0], "removed", []string{"utun9"}) + + // The tunnel comes back before a reconcile applied its removal. + discoverTunnelInterfacesForReconcile = func() []string { return []string{"utun7", "utun9"} } + p.checkTunnelInterfaceChanges() + + events = jsonLogEvents(t, logs, tunnelChangedMessage) + if len(events) != 2 { + t.Fatalf("got %d tunnel events after the tunnel came back, want 2", len(events)) + } + wantStrings(t, events[1], "added", []string{"utun9"}) +} + +// TestPFAnchorWipeAfterRestoreReportsTheMissingAnchor covers the wipe that +// follows a restore within ten seconds. That path enters stabilization instead +// of a rebuild, and it left the journal without the wipe that caused it. +func TestPFAnchorWipeAfterRestoreReportsTheMissingAnchor(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + stubPFAnchorCheckCommand(t, map[string]string{"-sn": ""}) + stubStabilizationProbe(t, []bool{true}, true) + stopCh := make(chan struct{}) + close(stopCh) + + p := &prog{dnsInterceptState: &pfState{}, stopCh: stopCh} + p.pfLastRestoreTime.Store(time.Now().UnixMilli()) + + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckDeferred { + t.Fatalf("wipe after restore result = %v, want deferred", result) + } + waitForStabilizationExit(t, p) + stopPFTestTimers(p) + + events := jsonLogEvents(t, logs, pfAnchorListMessage) + if len(events) != 2 { + t.Fatalf("got %d anchor list events for one wipe, want 2", len(events)) + } + wantField(t, events[0], "reason", pfAnchorReasonMissing) + wantField(t, events[0], "level", "warn") + wantField(t, events[1], "reason", pfAnchorReasonStabilizationStart) +} + +// TestPFAnchorListParsesTheRulesTheCallerHolds covers the anchor read of the +// watchdog. The watchdog already read both rule sets, and one more pair of +// pfctl processes on the network change path is work the host does not need. +func TestPFAnchorListParsesTheRulesTheCallerHolds(t *testing.T) { + logs := captureDebugMainLog(t) + var reads []string + stubPFStateCommandFn(t, func(name string, args ...string) ([]byte, error) { + key := strings.Join(append([]string{name}, args...), " ") + reads = append(reads, key) + output, ok := pfStateFixtures[key] + if !ok { + return nil, fmt.Errorf("no fixture for %q", key) + } + return []byte(output), nil + }) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonMissing, mainLog.Load().Warn(), pfRuleDump{ + rules: []byte(pfShowRules), + nat: []byte(pfShowNAT), + }) + + if want := []string{"pfctl -si"}; !slices.Equal(reads, want) { + t.Fatalf("pf state reads = %v, want %v", reads, want) + } + events := jsonLogEvents(t, logs, pfAnchorListMessage) + if len(events) != 1 { + t.Fatalf("got %d anchor list events, want 1", len(events)) + } + wantAnchors(t, events[0], []string{"com.apple/*", "com.controld.ctrld"}) + wantField(t, events[0], "pf_enabled", true) +} + +// TestPFAnchorListBacksOffOnExhaustedResources covers a failed anchor read. +// Every other pfctl read feeds the backoff, so the anchor read kept starting +// processes on a host that had none left. +func TestPFAnchorListBacksOffOnExhaustedResources(t *testing.T) { + captureDebugMainLog(t) + stubPFStateCommandFn(t, func(string, ...string) ([]byte, error) { + return nil, errors.New("fork/exec /sbin/pfctl: resource temporarily unavailable") + }) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{}) + + if !p.pfExecBackoffActive() { + t.Fatal("a failed anchor read did not start the pfctl backoff") + } +} + +// TestPFAnchorListMarksATruncatedRuleListUnknown covers an oversized read. A +// reader stops at a line it cannot hold, and the anchors it collected before +// that line read as a ruleset that lost the rest of them. +func TestPFAnchorListMarksATruncatedRuleListUnknown(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{ + rules: []byte(pfShowRules + strings.Repeat("a", 70<<10) + "\n"), + nat: []byte(pfShowNAT), + }) + + events := jsonLogEvents(t, logs, pfAnchorListMessage) + if len(events) != 1 { + t.Fatalf("got %d anchor list events, want 1", len(events)) + } + wantField(t, events[0], "anchors_known", false) + if _, ok := events[0]["anchors"]; ok { + t.Fatalf("a truncated read reported an anchor list: %v", events[0]) + } +} + +// TestPFStateOutputIsBounded covers the memory a state read can take. A pfctl +// that prints without end must not grow the daemon. +func TestPFStateOutputIsBounded(t *testing.T) { + out := &boundedBuffer{limit: 8} + if _, err := out.Write([]byte("12345")); err != nil { + t.Fatalf("write below the limit: %v", err) + } + if _, err := out.Write([]byte("67890")); err != nil { + t.Fatalf("write above the limit: %v", err) + } + if !out.exceeded { + t.Fatal("the writer did not report the dropped bytes") + } + if got := out.Len(); got > 8 { + t.Fatalf("the writer kept %d bytes, want at most 8", got) + } +} diff --git a/cmd/cli/dns_intercept_settle.go b/cmd/cli/dns_intercept_settle.go index 5aa9355..25a8022 100644 --- a/cmd/cli/dns_intercept_settle.go +++ b/cmd/cli/dns_intercept_settle.go @@ -6,12 +6,21 @@ import ( "github.com/Control-D-Inc/ctrld" ) -var initializeOsResolver = ctrld.InitializeOsResolver +var initializeOsResolver = ctrld.InitializeOsResolverWithReason + +// Reasons that the journal reports when the resolver set changes. Each one +// names the code path that read the resolvers of the host. +const ( + osResolverReasonStart = "start" + osResolverReasonWakeProbe = "wake_probe" + osResolverReasonVPNSettle = "vpn_settle" + osResolverReasonDelayedRecheck = "delayed_recheck" +) func (p *prog) refreshDNSAfterVPNSettle(reason string) (routes, domainlessServers, exemptions int) { mainLog.Load().Info().Msgf("DNS intercept: refreshing OS/VPN DNS route state after VPN settle (%s)", reason) ctx := ctrld.LoggerCtx(context.Background(), mainLog.Load()) - ns := initializeOsResolver(ctx, true) + ns := initializeOsResolver(ctx, true, osResolverReasonVPNSettle) mainLog.Load().Debug().Msgf("DNS intercept: post-settle OS resolver nameservers: %v", ns) if p.vpnDNS == nil { diff --git a/cmd/cli/dns_intercept_settle_test.go b/cmd/cli/dns_intercept_settle_test.go index 6a49dc7..ccc9504 100644 --- a/cmd/cli/dns_intercept_settle_test.go +++ b/cmd/cli/dns_intercept_settle_test.go @@ -12,8 +12,10 @@ func TestRefreshDNSAfterVPNSettleRefreshesOSResolverAndVPNRoutes(t *testing.T) { defer func() { initializeOsResolver = oldInitialize }() var initialized []bool - initializeOsResolver = func(ctx context.Context, force bool) []string { + var reasons []string + initializeOsResolver = func(ctx context.Context, force bool, reason string) []string { initialized = append(initialized, force) + reasons = append(reasons, reason) return []string{"10.102.26.10:53"} } @@ -40,6 +42,9 @@ func TestRefreshDNSAfterVPNSettleRefreshesOSResolverAndVPNRoutes(t *testing.T) { if len(initialized) != 1 || !initialized[0] { t.Fatalf("expected forced OS resolver refresh once, got %v", initialized) } + if len(reasons) != 1 || reasons[0] != osResolverReasonVPNSettle { + t.Fatalf("OS resolver reasons = %v, want [%s]", reasons, osResolverReasonVPNSettle) + } if got := p.vpnDNS.UpstreamForDomain("jira.cc.bmwgroup.net."); len(got) != 1 || got[0] != "10.102.26.10" { t.Fatalf("expected refreshed VPN DNS route, got %v", got) } diff --git a/cmd/cli/dns_intercept_windows.go b/cmd/cli/dns_intercept_windows.go index 8f3fa81..6fb2b78 100644 --- a/cmd/cli/dns_intercept_windows.go +++ b/cmd/cli/dns_intercept_windows.go @@ -3075,7 +3075,7 @@ func (p *prog) scheduleDelayedRechecks() { // Refresh OS resolver — VPN may have finished DNS cleanup since the // immediate handler ran. ctx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) - ctrld.InitializeOsResolver(ctx, true) + ctrld.InitializeOsResolverWithReason(ctx, true, osResolverReasonDelayedRecheck) if p.vpnDNS != nil { p.vpnDNS.Refresh(ctx, true) } diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index e3655d8..3b6213e 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -369,6 +369,8 @@ func (p *prog) processStandardQuery(req *standardQueryRequest) { ctrld.Log(req.ctx, p.Debug(), "Query refused, %s does not match any network policy", remoteAddr.String()) answer = new(dns.Msg) answer.SetRcode(req.msg, dns.RcodeRefused) + // The grade of the window must not depend on the goroutine below. + p.health.countQuery() // Process the refused query go p.postProcessStandardQuery(ci, req.listenerConfig, q, &proxyResponse{answer: answer, refused: true}) } else { @@ -391,12 +393,14 @@ func (p *prog) processStandardQuery(req *standardQueryRequest) { p.firewallRecordResolvedIPs(pr.answer, canonicalName(q.Name)) } + // The grade of the window must not depend on the goroutine below. + p.health.countQuery() go p.postProcessStandardQuery(ci, req.listenerConfig, q, pr) answer = pr.answer } if err := req.writer.WriteMsg(answer); err != nil { - ctrld.Log(req.ctx, p.Error().Err(err), "serveDNS: failed to send DNS response to client") + ctrld.Log(req.ctx, p.querySampler.event(sampleClassSendResponseFailed, "").Err(err), "serveDNS: failed to send DNS response to client") } ctrld.Log(req.ctx, p.Debug(), "Standard query processing completed") @@ -761,6 +765,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { if cachedRes := p.tryCache(ctx, req, upstreams); cachedRes != nil { ctrld.Log(ctx, p.Debug(), "Cache hit, returning cached response") + p.health.countCacheHit() return cachedRes } @@ -800,6 +805,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { if gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, vpnServers) { ctrld.Log(ctx, p.Debug(), "All VPN DNS servers had transport failures for %s; returning SERVFAIL while retained VPN DNS state is active", domain) + p.countFailedClientQuery(upstreams) answer := new(dns.Msg) answer.SetRcode(req.msg, dns.RcodeServerFailure) return &proxyResponse{answer: answer, cached: false} @@ -859,6 +865,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { if !gotDNSAnswer && gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, dlServers) { ctrld.Log(ctx, p.Debug(), "All domain-less VPN DNS servers had transport failures for %s; returning SERVFAIL while retained VPN DNS state is active", domain) + p.countFailedClientQuery(upstreams) answer := new(dns.Msg) answer.SetRcode(req.msg, dns.RcodeServerFailure) return &proxyResponse{answer: answer, cached: false} @@ -964,6 +971,7 @@ func (p *prog) updateCache(ctx context.Context, req *proxyRequest, answer *dns.M // serveStaleResponse serves a stale cached DNS response when an upstream query fails, updating TTL for cached records. func (p *prog) serveStaleResponse(ctx context.Context, staleAnswer *dns.Msg) *proxyResponse { ctrld.Log(ctx, p.Debug(), "Serving stale cached response") + p.health.countCacheHit() now := time.Now() setCachedAnswerTTL(staleAnswer, now, now.Add(staleTTL)) return &proxyResponse{answer: staleAnswer, cached: true} @@ -971,7 +979,7 @@ func (p *prog) serveStaleResponse(ctx context.Context, staleAnswer *dns.Msg) *pr // handleAllUpstreamsFailure handles the failure scenario when all upstream resolvers fail to respond or process the request. func (p *prog) handleAllUpstreamsFailure(ctx context.Context, req *proxyRequest, upstreams []string) *proxyResponse { - ctrld.Log(ctx, p.Error(), "All %v endpoints failed", upstreams) + ctrld.Log(ctx, p.querySampler.event(sampleClassAllEndpointsFailed, ""), "All %v endpoints failed", journalUpstreamNames(upstreams)) // An unavailable internal resolver must not trigger endpoint-wide recovery // or send the private name through the OS-resolver catch-all. @@ -996,11 +1004,23 @@ func (p *prog) handleAllUpstreamsFailure(ctx context.Context, req *proxyRequest, } ctrld.Log(ctx, p.Debug(), "Returning server failure response") + p.countFailedClientQuery(upstreams) answer := new(dns.Msg) answer.SetRcode(req.msg, dns.RcodeServerFailure) return &proxyResponse{answer: answer} } +// countFailedClientQuery grades one client query that ended without an answer. +// A query that only Internal Domain resolvers serve stays out of the grade: an +// endpoint away from the organization network never reaches them, and that is +// not an outage of the query path. +func (p *prog) countFailedClientQuery(upstreams []string) { + if internalDomainExplicitUpstreams(upstreams) { + return + } + p.health.countFailedQuery() +} + // shouldContinueWithNextUpstream determines whether processing should continue with the next upstream based on response conditions. func (p *prog) shouldContinueWithNextUpstream(ctx context.Context, req *proxyRequest, answer *dns.Msg, upstream string, lastUpstream bool) bool { if answer.Rcode == dns.RcodeSuccess { @@ -1176,21 +1196,22 @@ func (p *prog) queryUpstream(ctx context.Context, req *proxyRequest, upstream st answer, err := dnsResolver.Resolve(resolveCtx, req.msg) if answer != nil { ctrld.Log(ctx, p.Debug(), "Upstream resolution successful") - p.um.mu.Lock() - p.um.failureReq[upstream] = 0 - p.um.down[upstream] = false - p.um.mu.Unlock() + // reset is not used here, because it also stops the failure count for + // one second, and the failures already in flight must still count. + p.um.noteSuccess(upstream) return answer } + // A failing minute must not push the state events out of the retained log, + // so the sampler sets the level of these lines. // Transport errors contain the organization's private resolver address. // Keep that detail at debug while retaining a visible failure classification. if isInternalDomainUpstream(upstream) { ctrld.Log(ctx, p.Debug().Err(err), "Failed to resolve query") - ctrld.Log(ctx, p.Error().Str("failure", internalDomainFailureReason(err)), + ctrld.Log(ctx, p.querySampler.event(sampleClassInternalDomain, upstream).Str("failure", internalDomainFailureReason(err)), "Failed to resolve query using an Internal Domain resolver") } else { - ctrld.Log(ctx, p.Error().Err(err), "Failed to resolve query") + ctrld.Log(ctx, p.querySampler.event(sampleClassResolveFailed, upstream).Err(err), "Failed to resolve query") } // Increasing the failure count when there is no answer regardless of what kind of error we get p.um.increaseFailureCount(upstream) @@ -1866,12 +1887,66 @@ func (p *prog) handleNetworkChange(ctx context.Context, delta *netmon.ChangeDelt // delta.Major flag after a time jump does not change the cache. p.networkSourceMu.Lock() currentState := networkChangeCurrentStateFn(delta) - if !networkSnapshotCurrent(delta, isMajorChange, currentState) { + current := networkSnapshotCurrent(delta, isMajorChange, currentState) + if delta.TimeJumped { + // A late callback still tells that the host woke, but the network of + // the wake is the one that netmon holds now. + wakeState := delta.New + if !current { + wakeState = currentState + } + p.noteHostWoke("netmon", 0, wakeState) + } + // Only a current callback becomes the baseline of the next diff. A late + // callback that a newer snapshot replaced would hide the changes between + // the callback before it and the callback after it. + before := p.deltaBeforeState(delta, currentState, current, isMajorChange) + changes := diffNetworkDelta(before, delta.New) + if current && hasAddOrRemove(changes) { + refreshInterfaceMeta() + } + // The noise test runs before any other work, because the point of the class + // is that a storm starts no pfctl and no scutil process. + if current && noiseDelta(before, delta.New, delta.TimeJumped, changes) { p.networkSourceMu.Unlock() - p.logger.Load().Debug().Msg("Network transition skipped: snapshot superseded") + p.noteNoiseDelta(changes) return } - transitionID := p.networkTransitionGen.Add(1) + // A superseded snapshot never becomes a transition, so it takes no ID and + // reports zero. + var transitionID uint64 + changed := false + changedInterface := "" + sourceSnapshotAvailable := false + before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + after4, after6 := before4, before6 + // The newer callback of this epoch carries the same interface changes, so + // this outcome reports at debug and stays out of the journal. + outcome := transitionOutcomeSnapshotSuperseded + defer func() { + stateBefore, stateAfter := networkStateOrEmpty(before), networkStateOrEmpty(delta.New) + networkTransitionEvent(outcome).Uint64("transition_id", transitionID). + Bool("is_major_change", isMajorChange).Bool("changed", changed). + Str("interface", changedInterface). + Strs("changed_interfaces", changedInterfaceNames(changes)). + Bool("time_jumped", delta.TimeJumped). + Str("default_route_before", stateBefore.DefaultRouteInterface). + Str("default_route_after", stateAfter.DefaultRouteInterface). + Bool("have_v4_before", stateBefore.HaveV4).Bool("have_v4_after", stateAfter.HaveV4). + Bool("have_v6_before", stateBefore.HaveV6).Bool("have_v6_after", stateAfter.HaveV6). + Str("source_ipv4_before", before4.String()).Str("source_ipv6_before", before6.String()). + Str("source_ipv4_after", after4.String()). + Str("source_ipv6_after", after6.String()). + Bool("source_snapshot_available", sourceSnapshotAvailable).Str("outcome", outcome). + Msg(networkTransitionMessage) + }() + if !current { + p.networkSourceMu.Unlock() + return + } + p.flushNoiseSummary() + outcome = transitionOutcomeIgnored + transitionID = p.networkTransitionGen.Add(1) sourceState := delta.New // A late major callback and reordered minor callbacks share one cache // epoch. For these cases, source validity comes from the OS, not ordering. @@ -1895,29 +1970,25 @@ func (p *prog) handleNetworkChange(ctx context.Context, delta *netmon.ChangeDelt } } p.networkSourceState, p.networkSourceEpoch = sourceState, currentState - sourceSnapshotAvailable := sourceState != nil - before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + sourceSnapshotAvailable = sourceState != nil validateDefaultLocalIPsFromDelta(ctrld.LoggerCtx(ctx, p.logger.Load()), sourceState, transitionID) - after4, after6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + after4, after6 = ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + // The network of this callback belongs to the epoch that the lines above + // store. A store after the unlock can overwrite the network of a newer + // callback. The header render reads the hardware ports, so it runs after + // the unlock and starts no process under the lock. + p.lastNetworkState.Store(currentState) + // HasIPv6 reads the flag that this callback stores, so one monitor serves + // the whole process. + ctrld.SetIPv6Available(currentState.HaveV6) p.networkSourceMu.Unlock() + p.refreshLogHeader() + describeInterfaceChanges(changes, interfaceMetaFor) + logInterfaceChanges(transitionID, changes) p.handleDNS64NetworkChange(delta, isMajorChange) validIfaces := networkChangeValidInterfacesFn(ctrld.LoggerCtx(ctx, p.logger.Load())) - changed := false - changedInterface := "" - outcome := "ignored" - defer func() { - p.logger.Load().Debug().Uint64("transition_id", transitionID). - Bool("is_major_change", isMajorChange).Bool("changed", changed). - Str("interface", changedInterface). - Str("default_route_before", delta.Old.DefaultRouteInterface). - Str("default_route_after", delta.New.DefaultRouteInterface). - Str("source_ipv4_before", before4.String()).Str("source_ipv6_before", before6.String()). - Str("source_ipv4_after", after4.String()). - Str("source_ipv6_after", after6.String()). - Bool("source_snapshot_available", sourceSnapshotAvailable).Str("outcome", outcome).Msg("Network transition") - }() activeInterfaceExists := false var changeIPs []netip.Prefix // Check each valid interface for changes @@ -1937,6 +2008,9 @@ func (p *prog) handleNetworkChange(ctx context.Context, delta *netmon.ChangeDelt // The interface is new (was not present in the old state). usableNewIPs := filterUsableIPs(newIPs) if newIface.IsUp() && len(usableNewIPs) > 0 { + // The new interface is the active one, or the change ends as + // no_active_interface and nothing reconciles. + activeInterfaceExists = true changed = true changeIPs = usableNewIPs changedInterface = ifaceName @@ -2092,6 +2166,10 @@ func (p *prog) handleNetworkChange(ctx context.Context, delta *netmon.ChangeDelt outcome = "accepted" networkChangeReconcileFn(p, ctx, transitionID) + p.logNetworkSnapshot("transition") + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(networkEventsNowFn()) + } } // reconcileNetworkChange is the first DNS/PF mutation boundary after source updates. @@ -2152,32 +2230,16 @@ func (p *prog) handleDNSInterceptIgnoredNetworkChange(delta *netmon.ChangeDelta, // Spawn an async monitor that probes pf interception with backoff and forces // a full pf reload if broken. if delta.Old != nil { - interfaceChanged := false - var changedIface string - for ifaceName := range delta.Old.Interface { - if ifaceName == "lo0" { - continue - } - if _, exists := delta.New.Interface[ifaceName]; !exists { - interfaceChanged = true - changedIface = ifaceName - break - } + changedAction := "removed" + changedIface := interfaceOnlyIn(delta.Old.Interface, delta.New.Interface) + if changedIface == "" { + changedAction = "added" + changedIface = interfaceOnlyIn(delta.New.Interface, delta.Old.Interface) } - if !interfaceChanged { - for ifaceName := range delta.New.Interface { - if ifaceName == "lo0" { - continue - } - if _, exists := delta.Old.Interface[ifaceName]; !exists { - interfaceChanged = true - changedIface = ifaceName - break - } - } - } - if interfaceChanged { - mainLog.Load().Info().Str("interface", changedIface). + if changedIface != "" { + journal(mainLog.Load().Info()).Str("interface", changedIface). + Str("class", interfaceMetaFor(changedIface).Class). + Str("action", changedAction). Msg("DNS intercept: interface appeared/disappeared — starting interception probe monitor") go p.pfInterceptMonitor() // A VM/container bridge appearing/disappearing changes the effective @@ -2197,6 +2259,20 @@ func (p *prog) handleDNSInterceptIgnoredNetworkChange(delta *netmon.ChangeDelta, } } +// interfaceOnlyIn names one interface that have holds and lack does not. The +// loopback stays out, because it never appears and never goes away. +func interfaceOnlyIn(have, lack map[string]netmon.Interface) string { + for name := range have { + if name == "lo0" { + continue + } + if _, exists := lack[name]; !exists { + return name + } + } + return "" +} + // interfaceStatesEqual compares two interface states func interfaceStatesEqual(a, b *netmon.Interface) bool { if a == nil || b == nil { @@ -2247,15 +2323,30 @@ func interfaceIPsEqual(a, b []netip.Prefix) bool { var errOsHealthcheckSuppressed = errors.New("upstream os health check suppressed") +// upstreamFailureLog keeps one error line for each upstream of one recovery +// pass. The pass retries every two seconds until the upstream answers, and the +// first line already names the fault, so the failures that follow it go to +// debug. +type upstreamFailureLog struct{ reported bool } + +// report logs one failed check of upstream. +func (l *upstreamFailureLog) report(p *prog, upstream string, uc *ctrld.UpstreamConfig, err error, duration time.Duration) { + level := p.Error + if l.reported { + level = p.Debug + } + l.reported = true + p.logUpstreamProbeFailure(upstream, uc, err, level, "Upstream check failed after %v", duration) +} + // checkUpstreamOnce sends a test query to the specified upstream. // Returns nil if the upstream responds successfully. -func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig) error { +func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig, failures *upstreamFailureLog) error { p.Debug().Msgf("Starting check for upstream: %s", upstream) resolver, err := ctrld.NewResolver(ctrld.LoggerCtx(context.Background(), p.logger.Load()), uc) if err != nil { - p.logUpstreamProbeFailure(isGeneratedInternalDomainUpstreamRef(upstream, uc), uc, err, - p.Error, "Failed to create resolver for upstream %s", upstream) + p.logUpstreamProbeFailure(upstream, uc, err, p.Error, "Failed to create resolver") return err } @@ -2294,8 +2385,7 @@ func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig) erro p.Debug().Err(err).Msgf("Upstream %s check failed after %v (network unreachable)", upstream, duration) return err } - p.logUpstreamProbeFailure(isGeneratedInternalDomainUpstreamRef(upstream, uc), uc, err, - p.Error, "Upstream %s check failed after %v", upstream, duration) + failures.report(p, upstream, uc, err, duration) return err } p.Debug().Msgf("Upstream %s responded successfully in %v", upstream, duration) @@ -2314,6 +2404,10 @@ func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig) erro // leaves DoH transports in a stale state. const recoveryDebounceWindow = 500 * time.Millisecond +// recoveryResolverReason names this flow in the OS resolver events, so a +// reader tells a resolver read of the recovery from every other read. +const recoveryResolverReason = "recovery" + // debounceRecovery schedules a handleRecovery(NetworkChange) call after a debounce // window. If called again before the window expires, the timer is reset so that // recovery runs once with the final network state. All other state updates (IP, @@ -2347,6 +2441,48 @@ func (p *prog) debounceRecovery(transitionID uint64) { p.Debug().Msg("Recovery debounce: scheduled (500ms window)") } +// endRecovery closes one recovery pass. Support reads the end event and the +// snapshot that follows it to learn what the recovery left behind. +func (p *prog) endRecovery(diagnostic *recoveryDiagnostic, outcome, targetBefore string) { + // A superseded pass changed nothing that it owns any more. Its successor + // owns the loopback target and the bypass flag, so reading them here would + // report the state of the successor. + if outcome == recoveryOutcomeSuperseded { + diagnostic.interceptTargetAction = interceptTargetActionUnchanged + diagnostic.bypassActive = false + } else { + diagnostic.interceptTargetAction = interceptTargetAction(targetBefore, p.interceptTargetSnapshot()) + diagnostic.bypassActive = p.recoveryBypass.Load() + } + diagnostic.end(outcome) + p.logNetworkSnapshot("recovery_end") + // The resolver table of the host settles after the recovery, so the poll + // stays fast until it does. + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(networkEventsNowFn()) + } +} + +// The actions that one recovery takes on the loopback DNS target. +const ( + interceptTargetActionUnchanged = "unchanged" + interceptTargetActionRemoved = "removed" + interceptTargetActionSet = "set" +) + +// interceptTargetAction names what one recovery did to the loopback DNS +// target. The target value alone does not say whether this pass wrote it. +func interceptTargetAction(before, after string) string { + switch { + case before == after: + return interceptTargetActionUnchanged + case after == "": + return interceptTargetActionRemoved + default: + return interceptTargetActionSet + } +} + func (p *prog) handleRecovery(reason RecoveryReason) { p.handleRecoveryForTransition(reason, 0) } @@ -2367,9 +2503,11 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u return } diagnostic := recoveryDiagnostic{logger: p.logger.Load(), transitionID: transitionID, generation: gen, reason: reason, started: time.Now()} - diagnostic.event(p.logger.Load().Debug()).Bool("intercept", interceptRecovery).Msg("Recovery begin") - outcome := "superseded" - defer func() { diagnostic.end(outcome) }() + journal(diagnostic.event(p.logger.Load().Info())).Bool("intercept", interceptRecovery).Msg("Recovery begin") + p.logNetworkSnapshot("recovery_begin") + targetBefore := p.interceptTargetSnapshot() + outcome := recoveryOutcomeSuperseded + defer func() { p.endRecovery(&diagnostic, outcome, targetBefore) }() if reason == RecoveryReasonNetworkChange { p.logger.Load().Debug().Msg("Network change recovery now owns shared recovery state") } @@ -2391,26 +2529,34 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u p.Info().Msg("Force-reset upstream transports for network change recovery") } - if err := p.prepareForRecovery(reason, interceptRecovery); err != nil { + if err := p.prepareForRecovery(reason, interceptRecovery, &diagnostic); err != nil { p.Error().Err(err).Msg("Failed to prepare for recovery") + outcome = recoveryOutcomeFailed p.recoveryCanceledCleanup(gen) return } upstreams := p.buildRecoveryUpstreams(reason) - recovered, err := p.waitForUpstreamRecovery(recoveryCtx, upstreams, &diagnostic) + recovered, err := waitForUpstreamRecoveryFn(p, recoveryCtx, upstreams, &diagnostic) if err != nil { if p.recoveryOwnsState(gen) { - outcome = "canceled" + outcome = recoveryOutcomeCanceled } p.recoveryCanceledCleanup(gen) return } + diagnostic.recoveredUpstream = recovered if !p.recoveryOwnsState(gen) { p.Debug().Msgf("Recovery generation %d was superseded after upstream success; skipping stale completion", gen) return } + // The reset ends the outage, so the down time is read before it. The name + // of the line is bounded, because an upstream key is operator text. + journalName := journalUpstreamName(recovered) + journal(p.Info()).Str("upstream", journalName). + Int64("down_for_ms", p.um.downFor(recovered).Milliseconds()). + Msgf("Upstream %q recovered; re-applying DNS settings", journalName) if err := p.completeRecoveryWork(reason, recovered, interceptRecovery); err != nil { p.Error().Err(err).Msg("Failed to complete recovery") p.recoveryCanceledCleanup(gen) @@ -2421,29 +2567,36 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u return } - outcome = "completed" - p.Info().Msgf("Recovery completed successfully for upstream %q", recovered) + outcome = recoveryOutcomeCompleted + if interceptRecovery { + journal(p.Info()).Msg("DNS intercept recovery complete: disabling DHCP bypass, resuming normal flow") + } + p.Info().Msgf("Recovery completed successfully for upstream %q", journalName) } // prepareForRecovery removes DNS settings and initializes OS resolver if needed. -func (p *prog) prepareForRecovery(reason RecoveryReason, interceptRecovery bool) error { +func (p *prog) prepareForRecovery(reason RecoveryReason, interceptRecovery bool, diagnostic *recoveryDiagnostic) error { // In DNS intercept mode, don't tear down WFP/pf filters. // Instead, enable recovery bypass so proxy() forwards queries to // the OS/DHCP resolver. This handles captive portal authentication // without the overhead of filter teardown/rebuild. if interceptRecovery { - p.Info().Msg("DNS intercept recovery: enabling DHCP bypass (filters stay active)") + journal(p.Info()).Msg("DNS intercept recovery: enabling DHCP bypass (filters stay active)") // Reinitialize OS resolver to discover DHCP servers on the new network. // This is critical for captive portals — we need the network's DNS servers // to resolve the auth page. p.Debug().Msg("DNS intercept recovery: discovering DHCP nameservers") loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) - dhcpServers, systemNameservers := initializeOsResolverWithSystemNameserversFn(loggerCtx, true) - if len(dhcpServers) == 0 { - p.Warn().Msg("DNS intercept recovery: no DHCP nameservers found") + // The effective list adds a synthetic public resolver when the network + // gave none, so the journal reports the discovered list instead. + resolverNameservers, systemNameservers := initializeOsResolverWithSystemNameserversFn(loggerCtx, true, recoveryResolverReason) + diagnostic.dhcpServers = systemNameservers + if len(systemNameservers) == 0 { + journal(p.Warn()).Msg("DNS intercept recovery: no DHCP nameservers found") } else { - p.Info().Msgf("DNS intercept recovery: found DHCP nameservers: %v", dhcpServers) + journal(p.Info()).Strs("dhcp_servers", systemNameservers). + Msgf("DNS intercept recovery: found DHCP nameservers: %v", systemNameservers) } ensureInterceptDNSTargetFn(p, systemNameservers) @@ -2451,17 +2604,18 @@ func (p *prog) prepareForRecovery(reason RecoveryReason, interceptRecovery bool) // Exempt DHCP nameservers from intercept filters so the OS resolver // can actually reach them on port 53. Without this, the WFP block // or pf redirect would intercept ctrld's own recovery queries. - if len(dhcpServers) > 0 { + // The OS resolver queries the effective list, so the exemptions follow it. + if len(resolverNameservers) > 0 { // Strip :53 port suffix if present (exemptVPNDNSServers expects bare IPs). var dhcpExemptions []vpnDNSExemption - for _, s := range dhcpServers { + for _, s := range resolverNameservers { host := s if h, _, err := net.SplitHostPort(s); err == nil { host = h } dhcpExemptions = append(dhcpExemptions, vpnDNSExemption{Server: host}) } - p.Info().Msgf("DNS intercept recovery: exempting DHCP nameservers from filters: %v", dhcpServers) + p.Info().Msgf("DNS intercept recovery: exempting DHCP nameservers from filters: %v", resolverNameservers) if err := p.exemptVPNDNSServers(dhcpExemptions); err != nil { p.Warn().Err(err).Msg("DNS intercept recovery: failed to exempt DHCP nameservers — recovery queries may fail") } @@ -2490,11 +2644,11 @@ func (p *prog) prepareForRecovery(reason RecoveryReason, interceptRecovery bool) func (p *prog) reinitializeOSResolver(message string) error { p.Debug().Msg(message) loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) - ns := ctrld.InitializeOsResolver(loggerCtx, true) + ns := ctrld.InitializeOsResolverWithReason(loggerCtx, true, recoveryResolverReason) if len(ns) == 0 { p.Warn().Msg("No nameservers found for OS resolver; using existing values") } else { - p.Info().Msgf("Reinitialized OS resolver with nameservers: %v", ns) + journal(p.Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns) } return nil } @@ -2528,11 +2682,16 @@ func (p *prog) completeRecoveryWork(reason RecoveryReason, recovered string, int } } + // Apply our DNS settings back. The snapshot that follows the end event + // reports the interfaces. p.setDNS(systemNameservers) - p.logInterfacesState() return nil } +// waitForUpstreamRecoveryFn is the seam of the upstream probe. A test drives +// the whole recovery flow without a query to a real upstream. +var waitForUpstreamRecoveryFn = (*prog).waitForUpstreamRecovery + // waitForUpstreamRecovery checks the provided upstreams concurrently until one recovers. // It returns the name of the recovered upstream or an error if the check times out. func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string]*ctrld.UpstreamConfig, diagnostic *recoveryDiagnostic) (string, error) { @@ -2551,6 +2710,7 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string p.Debug().Msgf("Starting recovery check loop for upstream: %s", name) attempts := 0 unreachableStreak := 0 + var failures upstreamFailureLog for { select { case <-recoveryCtx.Done(): @@ -2559,7 +2719,7 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string default: attempts++ // checkUpstreamOnce will reset any failure counters on success. - err := p.checkUpstreamOnce(name, uc) + err := p.checkUpstreamOnce(name, uc, &failures) if err == nil || errors.Is(err, errOsHealthcheckSuppressed) { p.Debug().Msgf("Upstream %s recovered successfully", name) select { @@ -2595,11 +2755,11 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string // we should try to reinit the OS resolver to ensure we can recover if name == upstreamOS && attempts%3 == 0 { p.Debug().Msgf("UpstreamOS check failed on attempt %d, reinitializing OS resolver", attempts) - ns := ctrld.InitializeOsResolver(ctrld.LoggerCtx(ctx, p.logger.Load()), true) + ns := ctrld.InitializeOsResolverWithReason(ctrld.LoggerCtx(ctx, p.logger.Load()), true, recoveryResolverReason) if len(ns) == 0 { p.Warn().Msg("No nameservers found for OS resolver; using existing values") } else { - p.Info().Msgf("Reinitialized OS resolver with nameservers: %v", ns) + journal(p.Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns) } } } diff --git a/cmd/cli/dns_proxy_sampling_test.go b/cmd/cli/dns_proxy_sampling_test.go new file mode 100644 index 0000000..2c4fe13 --- /dev/null +++ b/cmd/cli/dns_proxy_sampling_test.go @@ -0,0 +1,175 @@ +package cli + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/miekg/dns" + + "github.com/Control-D-Inc/ctrld" +) + +// deadUpstreamEndpoint is a port that nothing listens on, so every query to it +// fails at once. +const deadUpstreamEndpoint = "127.0.0.1:1" + +// countSamplingLines counts the captured lines of one level whose message +// starts with the given text. A request id rides in front of the message. +func countSamplingLines(t *testing.T, logs *syncBuffer, level, message string) int { + t.Helper() + count := 0 + for _, event := range jsonLogEvents(t, logs, "") { + text, _ := event["message"].(string) + if !strings.HasPrefix(text, message) || event["level"] != level { + continue + } + count++ + } + return count +} + +func Test_querySamplingBoundsPerQueryErrors(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + p.logger.Store(mainLog.Load()) + // The zero value sampler arms a real timer. Close its window after the + // assertions, so no late summary line reaches another test. + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + for i := 0; i < querySampleLimit+1; i++ { + ctrld.Log(context.Background(), p.querySampler.event(sampleClassResolveFailed, "upstream.0"), samplerTestMessage) + } + + if got := countSamplingLines(t, logs, "error", samplerTestMessage); got != querySampleLimit { + t.Fatalf("error lines: got %d, want %d", got, querySampleLimit) + } + if got := countSamplingLines(t, logs, "debug", samplerTestMessage); got != 1 { + t.Fatalf("debug lines: got %d, want 1", got) + } +} + +// askThroughProxy sends one query through the real proxy path and returns the +// answer. +func askThroughProxy(t *testing.T, p *prog, ctx context.Context) *dns.Msg { + t.Helper() + msg := new(dns.Msg) + msg.SetQuestion("sampling.test.", dns.TypeA) + res := p.proxy(ctx, &proxyRequest{msg: msg, ufr: &upstreamForResult{srcAddr: "192.168.0.1:1234"}}) + if res == nil || res.answer == nil { + t.Fatal("the proxy returned no answer") + } + return res.answer +} + +// Test_querySamplingOnTheQueryPath drives the real query path with an upstream +// that answers nothing: the flood keeps its request id, the sampler bounds it, +// and one state event brackets each end of the outage. +func Test_querySamplingOnTheQueryPath(t *testing.T) { + logs := captureDebugMainLog(t) + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load())} + p.logger.Store(mainLog.Load()) + // A new monitor holds back the failure counts for one second. The test + // clears the flag instead of waiting for that timer. + p.um.clearRecovered(upstreamOS) + var armed []func() + p.um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + // The zero value sampler arms a real timer. Close its window after the + // assertions, so no late summary line reaches another test. + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + origOSUpstream := osUpstreamConfig + t.Cleanup(func() { osUpstreamConfig = origOSUpstream }) + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "dead resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 500, + } + + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, "r1") + for i := 0; i < querySampleLimit+2; i++ { + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeServerFailure { + t.Fatalf("query %d: rcode %s, want SERVFAIL", i+1, dns.RcodeToString[rcode]) + } + } + + const failedMessage = "[r1] " + samplerTestMessage + if got := countSamplingLines(t, logs, "error", failedMessage); got != querySampleLimit { + t.Fatalf("error lines: got %d, want %d", got, querySampleLimit) + } + if got := countSamplingLines(t, logs, "debug", failedMessage); got != 2 { + t.Fatalf("debug lines: got %d, want 2", got) + } + + if len(armed) != 1 { + t.Fatalf("armed timers: got %d, want 1", len(armed)) + } + armed[0]() + startOSResolverStub(t) + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeSuccess { + t.Fatalf("the answered query returned %s, want NOERROR", dns.RcodeToString[rcode]) + } + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want 2", len(events)) + } + wantField(t, events[0], "upstream", upstreamOS) + wantField(t, events[0], "state", "down") + wantField(t, events[0], "reason", "timer") + wantField(t, events[1], "upstream", upstreamOS) + wantField(t, events[1], "state", "up") +} + +// Test_allEndpointsFailedBoundsTheUpstreamName drives the query path with an +// operator-chosen upstream key. The line that reports the loss of every +// endpoint reaches the journal, so it must name the upstream by its bounded +// name. +func Test_allEndpointsFailedBoundsTheUpstreamName(t *testing.T) { + logs := captureDebugMainLog(t) + uc := &ctrld.UpstreamConfig{ + Name: probeSecret, + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 200, + } + uc.Init(context.Background()) + cfg := &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{probeSecret: uc}} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load())} + p.logger.Store(mainLog.Load()) + // The zero value sampler arms a real timer. Close its window after the + // assertions, so no late summary line reaches another test. + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + msg := new(dns.Msg) + msg.SetQuestion("private.test.", dns.TypeA) + res := p.proxy(context.Background(), &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{ + srcAddr: "192.168.0.1:1234", + upstreams: []string{upstreamPrefix + probeSecret}, + }, + }) + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeServerFailure { + t.Fatal("the proxy answered although every endpoint failed") + } + + var reported string + for _, event := range jsonLogEvents(t, logs, "") { + text, _ := event["message"].(string) + if strings.Contains(text, "endpoints failed") { + reported = text + } + } + if !strings.Contains(reported, upstreamPrefix+"custom") { + t.Fatalf("the all-endpoints line does not name the bounded upstream: %q", reported) + } + if strings.Contains(reported, probeSecret) { + t.Fatalf("the all-endpoints line holds the operator key: %q", reported) + } +} diff --git a/cmd/cli/dns_proxy_zpa_test.go b/cmd/cli/dns_proxy_zpa_test.go index a373779..0898978 100644 --- a/cmd/cli/dns_proxy_zpa_test.go +++ b/cmd/cli/dns_proxy_zpa_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - ctrld "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld" "github.com/Control-D-Inc/ctrld/internal/dnscache" "github.com/Control-D-Inc/ctrld/testhelper" ) diff --git a/cmd/cli/dns_target_darwin.go b/cmd/cli/dns_target_darwin.go index 3d411c6..b4f7938 100644 --- a/cmd/cli/dns_target_darwin.go +++ b/cmd/cli/dns_target_darwin.go @@ -61,8 +61,7 @@ func (p *prog) loadInterceptDNSTargetStateLocked() { if err := json.Unmarshal(data, &st); err != nil || st.Service == "" || st.Value == "" { return } - p.interceptDNSTargetService = st.Service - p.interceptDNSTargetSetValue = st.Value + p.setInterceptDNSTargetLocked(st.Service, st.Value) mainLog.Load().Debug().Msgf("intercept DNS target: restored tracking of %s on %q from previous run", st.Value, st.Service) } @@ -186,10 +185,11 @@ func (p *prog) ensureInterceptDNSTarget(systemDiscovery []string) { mainLog.Load().Warn().Err(err).Msgf("intercept DNS target: could not set %s on %q", target, iface.Name) return } - p.interceptDNSTargetService = iface.Name - p.interceptDNSTargetSetValue = target + p.setInterceptDNSTargetLocked(iface.Name, target) p.persistInterceptDNSTargetStateLocked() - mainLog.Load().Warn().Msgf("intercept DNS target: service %q provides no usable IPv4 DNS; set %s so macOS can emit DNS queries (removed automatically when the network provides IPv4 DNS)", iface.Name, target) + journal(mainLog.Load().Warn()).Str("service", iface.Name).Str("target", target). + Str("reason", "dns_less_network"). + Msgf("intercept DNS target: service %q provides no usable IPv4 DNS; set %s so macOS can emit DNS queries (removed automatically when the network provides IPv4 DNS)", iface.Name, target) } // removeInterceptDNSTarget removes a previously set intercept DNS target, @@ -219,8 +219,12 @@ func (p *prog) removeInterceptDNSTargetLocked(reason string) { return } if !isInterceptDNSTargetOnly(cur, val) { - mainLog.Load().Debug().Msgf("intercept DNS target: %q DNS changed externally; not removing (%s)", svc, reason) p.clearInterceptDNSTargetStateLocked() + // ctrld owns the DNS of the service no longer, and a later outage + // report needs the moment that ownership ended. + journal(mainLog.Load().Info()).Str("service", svc).Str("target", val). + Str("reason", "external_change"). + Msgf("intercept DNS target: %q DNS changed externally; not removing (%s)", svc, reason) return } if saved := interceptSavedStaticNameserversFn(iface); len(saved) > 0 { @@ -233,11 +237,20 @@ func (p *prog) removeInterceptDNSTargetLocked(reason string) { return } p.clearInterceptDNSTargetStateLocked() - mainLog.Load().Info().Msgf("intercept DNS target: removed %s from %q (%s)", val, svc, reason) + journal(mainLog.Load().Info()).Str("service", svc).Str("target", val).Str("reason", reason). + Msgf("intercept DNS target: removed %s from %q (%s)", val, svc, reason) } func (p *prog) clearInterceptDNSTargetStateLocked() { - p.interceptDNSTargetService = "" - p.interceptDNSTargetSetValue = "" + p.setInterceptDNSTargetLocked("", "") p.persistInterceptDNSTargetStateLocked() } + +// setInterceptDNSTargetLocked stores the service that ctrld owns and the value +// it wrote, and publishes the value for the readers that take no lock. Callers +// must hold interceptDNSTargetMu. +func (p *prog) setInterceptDNSTargetLocked(service, value string) { + p.interceptDNSTargetService = service + p.interceptDNSTargetSetValue = value + p.publishInterceptTarget(value) +} diff --git a/cmd/cli/dns_target_darwin_test.go b/cmd/cli/dns_target_darwin_test.go index 6f37a80..a04f785 100644 --- a/cmd/cli/dns_target_darwin_test.go +++ b/cmd/cli/dns_target_darwin_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" "github.com/Control-D-Inc/ctrld" @@ -125,8 +126,7 @@ func persistInterceptTargetForTest(t *testing.T, p *prog, service, value string) p.interceptDNSTargetMu.Lock() defer p.interceptDNSTargetMu.Unlock() p.interceptDNSTargetLoaded = true - p.interceptDNSTargetService = service - p.interceptDNSTargetSetValue = value + p.setInterceptDNSTargetLocked(service, value) p.persistInterceptDNSTargetStateLocked() } @@ -246,3 +246,104 @@ func TestRemoveInterceptDNSTargetRetainsStateOnFailure(t *testing.T) { }) } } + +// interceptTargetEvent returns the single event whose message starts with the +// given text. The two target events name the service and the value in their +// message, so no test can match a whole message. +func interceptTargetEvent(t *testing.T, logs *syncBuffer, prefix string) map[string]any { + t.Helper() + var found []map[string]any + for _, event := range jsonLogEvents(t, logs, "") { + message, _ := event["message"].(string) + if strings.HasPrefix(message, prefix) { + found = append(found, event) + } + } + if len(found) != 1 { + t.Fatalf("events with prefix %q: got %d, want 1", prefix, len(found)) + } + return found[0] +} + +func TestEnsureInterceptDNSTargetJournalsTheSetTarget(t *testing.T) { + logs := captureDebugMainLog(t) + newInterceptTargetHarness(t) + p := newInterceptTargetProg() + + p.ensureInterceptDNSTarget([]string{}) + + event := interceptTargetEvent(t, logs, "intercept DNS target: service ") + wantField(t, event, "journal", true) + wantField(t, event, "service", "Wi-Fi") + wantField(t, event, "target", "127.0.0.53") + wantField(t, event, "reason", "dns_less_network") +} + +func TestRemoveInterceptDNSTargetJournalsTheRemovedTarget(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"127.0.0.53"} + + p.removeInterceptDNSTarget("network has usable static IPv4 DNS") + + event := interceptTargetEvent(t, logs, "intercept DNS target: removed ") + wantField(t, event, "journal", true) + wantField(t, event, "service", "Wi-Fi") + wantField(t, event, "target", "127.0.0.53") + wantField(t, event, "reason", "network has usable static IPv4 DNS") +} + +// TestInterceptDNSTargetSnapshotFollowsTheWritesWithoutTheLock covers the read +// that a network report uses. The target mutex is held across networksetup +// calls that take seconds, so a report that takes it waits for them. +func TestInterceptDNSTargetSnapshotFollowsTheWritesWithoutTheLock(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + h.dns["Wi-Fi"] = nil + + p.ensureInterceptDNSTarget([]string{}) + + p.interceptDNSTargetMu.Lock() + written := p.interceptDNSTargetSetValue + got := p.interceptTargetSnapshot() + p.interceptDNSTargetMu.Unlock() + if written == "" { + t.Fatal("the pass wrote no target") + } + if got != written { + t.Fatalf("target snapshot = %q, want %q", got, written) + } + + p.removeInterceptDNSTarget("test") + + p.interceptDNSTargetMu.Lock() + got = p.interceptTargetSnapshot() + p.interceptDNSTargetMu.Unlock() + if got != "" { + t.Fatalf("target snapshot after the removal = %q, want an empty value", got) + } +} + +// TestRemoveInterceptDNSTargetJournalsAnExternalChange covers the branch that +// drops the ownership record. The user or another program owns the DNS of the +// service from that moment, and the journal held nothing about it. +func TestRemoveInterceptDNSTargetJournalsAnExternalChange(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"9.9.9.9"} + + p.removeInterceptDNSTarget("intercept shutdown") + + event := oneRecoveryEvent(t, logs, `intercept DNS target: "Wi-Fi" DNS changed externally; not removing (intercept shutdown)`) + wantField(t, event, "journal", true) + wantField(t, event, "service", "Wi-Fi") + wantField(t, event, "target", "127.0.0.53") + wantField(t, event, "reason", "external_change") + if p.interceptTargetSnapshot() != "" { + t.Fatal("the ownership record survived an external change") + } +} diff --git a/cmd/cli/intercept_target.go b/cmd/cli/intercept_target.go new file mode 100644 index 0000000..7fd3498 --- /dev/null +++ b/cmd/cli/intercept_target.go @@ -0,0 +1,15 @@ +package cli + +// publishInterceptTarget mirrors the loopback DNS target that ctrld wrote into +// a network service. Callers hold interceptDNSTargetMu. +func (p *prog) publishInterceptTarget(target string) { + p.interceptTargetMirror.Store(target) +} + +// interceptTargetSnapshot reads the mirrored loopback DNS target. It takes no +// lock, so a report of the network never waits on the networksetup calls that +// the target mutex covers. The value is empty while ctrld holds no service. +func (p *prog) interceptTargetSnapshot() string { + target, _ := p.interceptTargetMirror.Load().(string) + return target +} diff --git a/cmd/cli/internal_domains.go b/cmd/cli/internal_domains.go index 3a54cea..ae968fd 100644 --- a/cmd/cli/internal_domains.go +++ b/cmd/cli/internal_domains.go @@ -433,22 +433,25 @@ func isGeneratedInternalDomainUpstreamRef(upstream string, uc *ctrld.UpstreamCon return isGeneratedInternalDomainUpstream(strings.TrimPrefix(upstream, upstreamPrefix), uc) } -// logUpstreamProbeFailure reports a background probe failure against an -// upstream. +// logUpstreamProbeFailure reports a background probe failure against the +// upstream that the reference upstream names. // // Loop checking and recovery checking run on timers, so for a generated // Internal Domain resolver they would publish the organization's private -// address without any user ever querying the domain. They get the same -// treatment as a query failure: endpoint and the address-bearing error at -// debug, a classification above it, so the outcome stays visible while the -// address does not. -func (p *prog) logUpstreamProbeFailure(generated bool, uc *ctrld.UpstreamConfig, err error, level func() *ctrld.LogEvent, format string, args ...any) { +// address without any user ever querying the domain. The name and the +// endpoint of any upstream are operator text that can hold a token, so both +// stay on the debug line. The leveled line names the upstream by its bounded +// name, so the outcome stays visible while the operator text does not. A +// generated Internal Domain resolver also trades the address-bearing error +// for a classification. +func (p *prog) logUpstreamProbeFailure(upstream string, uc *ctrld.UpstreamConfig, err error, level func() *ctrld.LogEvent, format string, args ...any) { detail := append([]any{}, args...) - if !generated { - level().Err(err).Msgf(format+" for upstream: %q, endpoint: %q", append(detail, uc.Name, uc.Endpoint)...) + p.Debug().Err(err).Msgf(format+" for upstream: %q, endpoint: %q", append(detail, uc.Name, uc.Endpoint)...) + event := level().Str("upstream", journalUpstreamName(upstream)) + if !isGeneratedInternalDomainUpstreamRef(upstream, uc) { + event.Err(err).Msgf(format, detail...) return } - p.Debug().Err(err).Msgf(format+" for upstream: %q, endpoint: %q", append(detail, uc.Name, uc.Endpoint)...) - level().Str("failure", internalDomainFailureReason(err)). + event.Str("failure", internalDomainFailureReason(err)). Msgf(format+" for an Internal Domain resolver", detail...) } diff --git a/cmd/cli/internal_domains_review_test.go b/cmd/cli/internal_domains_review_test.go index 6ddda61..694b046 100644 --- a/cmd/cli/internal_domains_review_test.go +++ b/cmd/cli/internal_domains_review_test.go @@ -615,7 +615,7 @@ func TestInternalDomainsRecoveryCheckHidesResolverAddress(t *testing.T) { } mark := len(buf.String()) - if err := p.checkUpstreamOnce("upstream.internal_0", uc); err == nil { + if err := p.checkUpstreamOnce("upstream.internal_0", uc, &upstreamFailureLog{}); err == nil { t.Fatal("expected the probe to fail") } above := logLinesAboveDebug(buf, mark) diff --git a/cmd/cli/log_budget.go b/cmd/cli/log_budget.go new file mode 100644 index 0000000..551dfd2 --- /dev/null +++ b/cmd/cli/log_budget.go @@ -0,0 +1,41 @@ +package cli + +import ( + "github.com/Control-D-Inc/ctrld" +) + +// journalLogFileName is the journal file next to the debug log in the ctrld +// home directory. +const journalLogFileName = "ctrld-journal.log" + +// logBudgets is the disk space of the two streams. The debug stream takes the +// larger share, so a log covers more days. The journal holds the retained +// lines only, so it stays small. +func logBudgets() (debug, journal logBudget) { + return logBudget{maxSize: 10 << 20, backups: 4}, logBudget{maxSize: 2 << 20, backups: 2} +} + +const ( + // logMaxSizeMBLimit and logMaxBackupsLimit bound the two keys. The log + // file opens before ctrld validates its configuration, so a key outside + // these bounds reaches this code: a size near the largest integer never + // rotates, and a count of one million backups renames one million files. + // The validation rules are the first line of defense, and these bounds + // are the second. + logMaxSizeMBLimit = 1024 + logMaxBackupsLimit = 64 +) + +// debugLogBudget starts from the default. The two keys size the debug stream +// only. The journal budget stays fixed. A key outside its bounds keeps the +// default. +func debugLogBudget(svc *ctrld.ServiceConfig) logBudget { + budget, _ := logBudgets() + if mb := svc.LogMaxSizeMB; mb >= 1 && mb <= logMaxSizeMBLimit { + budget.maxSize = int64(mb) << 20 + } + if count := svc.LogMaxBackups; count != nil && *count >= 0 && *count <= logMaxBackupsLimit { + budget.backups = *count + } + return budget +} diff --git a/cmd/cli/log_budget_test.go b/cmd/cli/log_budget_test.go new file mode 100644 index 0000000..fc8f577 --- /dev/null +++ b/cmd/cli/log_budget_test.go @@ -0,0 +1,45 @@ +package cli + +import ( + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func Test_logBudgets(t *testing.T) { + debug, journalStream := logBudgets() + if want := (logBudget{maxSize: 10 << 20, backups: 4}); debug != want { + t.Errorf("debug budget = %+v, want %+v", debug, want) + } + if want := (logBudget{maxSize: 2 << 20, backups: 2}); journalStream != want { + t.Errorf("journal budget = %+v, want %+v", journalStream, want) + } +} + +func Test_debugLogBudget(t *testing.T) { + noBackup, oneBackup := 0, 1 + negativeBackups, tooManyBackups, mostBackups := -1, logMaxBackupsLimit+1, logMaxBackupsLimit + for _, tc := range []struct { + name string + svc ctrld.ServiceConfig + want logBudget + }{ + {"both keys set", ctrld.ServiceConfig{LogMaxSizeMB: 20, LogMaxBackups: &oneBackup}, logBudget{maxSize: 20 << 20, backups: 1}}, + {"size zero keeps the default size", ctrld.ServiceConfig{LogMaxBackups: &oneBackup}, logBudget{maxSize: 10 << 20, backups: 1}}, + {"absent backups keep the default count", ctrld.ServiceConfig{LogMaxSizeMB: 20}, logBudget{maxSize: 20 << 20, backups: 4}}, + {"zero backups keep no backup file", ctrld.ServiceConfig{LogMaxBackups: &noBackup}, logBudget{maxSize: 10 << 20, backups: 0}}, + {"empty config", ctrld.ServiceConfig{}, logBudget{maxSize: 10 << 20, backups: 4}}, + {"negative size keeps the default size", ctrld.ServiceConfig{LogMaxSizeMB: -1}, logBudget{maxSize: 10 << 20, backups: 4}}, + {"size above the limit keeps the default size", ctrld.ServiceConfig{LogMaxSizeMB: logMaxSizeMBLimit + 1}, logBudget{maxSize: 10 << 20, backups: 4}}, + {"the largest size the key takes", ctrld.ServiceConfig{LogMaxSizeMB: logMaxSizeMBLimit}, logBudget{maxSize: logMaxSizeMBLimit << 20, backups: 4}}, + {"negative backups keep the default count", ctrld.ServiceConfig{LogMaxBackups: &negativeBackups}, logBudget{maxSize: 10 << 20, backups: 4}}, + {"backups above the limit keep the default count", ctrld.ServiceConfig{LogMaxBackups: &tooManyBackups}, logBudget{maxSize: 10 << 20, backups: 4}}, + {"the largest backup count the key takes", ctrld.ServiceConfig{LogMaxBackups: &mostBackups}, logBudget{maxSize: 10 << 20, backups: logMaxBackupsLimit}}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := debugLogBudget(&tc.svc); got != tc.want { + t.Errorf("debug budget = %+v, want %+v", got, tc.want) + } + }) + } +} diff --git a/cmd/cli/log_file.go b/cmd/cli/log_file.go new file mode 100644 index 0000000..dee26e7 --- /dev/null +++ b/cmd/cli/log_file.go @@ -0,0 +1,416 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// errLogFileNotOpen tells a caller that the file is closed, so a write that +// returns no bytes never looks like a success. +var errLogFileNotOpen = errors.New("log file is not open") + +// retryOpenInterval is the wait between two attempts to open a log file that +// could not open. A file error is often short-lived, and one attempt for each +// line would cost a system call for every line. +var retryOpenInterval = time.Minute + +// logBudget is the disk space of one log stream: the size of the current +// file and the number of numbered backups that follow it. +type logBudget struct { + maxSize int64 + backups int +} + +// logRotation describes the file that a rotation moved away. A set err means +// that the file could not move, so the stream continues in the same file. +type logRotation struct { + file string + bytesWritten int64 + firstWrite time.Time + lastWrite time.Time + backups int + err error +} + +// rotatingFile writes one log file and rotates it by size with numbered +// backups. It holds the header bytes, so every file it opens starts with +// the header, and it records the write times that the rotation event needs. +type rotatingFile struct { + mu sync.Mutex + path string + budget logBudget + open func(string, int) (*os.File, error) + rename func(oldpath, newpath string) error + f *os.File + size int64 + firstWrite time.Time + lastWrite time.Time + header []byte + openErr error + onRotate func(logRotation) + // headerWritten tells whether the file that is open now already holds the + // header. A start wires its logging more than once, and each file takes + // one header. + headerWritten bool + // rotateFailedAt is the size at which a rotation could not move the file. + // The next try waits for another budget of data, so a directory that + // refuses the move does not cost one attempt per line. + rotateFailedAt int64 + // retryOpenAt is the time from which a closed file may open again. + retryOpenAt time.Time + // rotatedInFlight keeps onRotate from reporting a rotation that the + // rotation event itself caused. + rotatedInFlight atomic.Bool +} + +// newRotatingFile opens path in append mode and counts what it already holds, +// so a restart continues the current file instead of starting a new one. +func newRotatingFile(path string, budget logBudget, open func(string, int) (*os.File, error)) (*rotatingFile, error) { + if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil { + return nil, fmt.Errorf("creating log directory: %w", err) + } + rf := &rotatingFile{path: path, budget: budget, open: open} + f, err := rf.openFile(os.O_CREATE | os.O_RDWR | os.O_APPEND) + if err != nil { + return nil, fmt.Errorf("opening log file: %w", err) + } + st, err := f.Stat() + if err != nil { + f.Close() + return nil, fmt.Errorf("stat log file: %w", err) + } + rf.f = f + rf.size = st.Size() + return rf, nil +} + +// openFile opens the file through the platform helper, which holds the flags +// that each platform needs: a share mode on Windows, and a refusal to follow +// a symlink elsewhere. +func (rf *rotatingFile) openFile(flags int) (*os.File, error) { + if rf.open == nil { + return openLogFile(rf.path, flags) + } + return rf.open(rf.path, flags) +} + +func (rf *rotatingFile) renameFile(oldpath, newpath string) error { + if rf.rename == nil { + return os.Rename(oldpath, newpath) + } + return rf.rename(oldpath, newpath) +} + +// writeReporting appends p and reports a rotation that happened first. It +// never calls onRotate, because the caller must log the rotation outside of +// the lock. +func (rf *rotatingFile) writeReporting(p []byte) (int, *logRotation, error) { + rf.mu.Lock() + defer rf.mu.Unlock() + + var rotated *logRotation + if rf.needsRotation(len(p)) { + var err error + if rotated, err = rf.rotateLocked(); err != nil { + return 0, rotated, err + } + } + n, err := rf.writeLocked(p) + if err != nil { + return n, rotated, err + } + rf.noteEventWrite() + return n, rotated, nil +} + +// Write makes the file an io.Writer for the logger. It reports the rotation +// after the lock is free, so the rotation event can go back into this file. +func (rf *rotatingFile) Write(p []byte) (int, error) { + n, rotated, err := rf.writeReporting(p) + if rotated == nil || !rf.rotatedInFlight.CompareAndSwap(false, true) { + return n, err + } + defer rf.rotatedInFlight.Store(false) + rf.mu.Lock() + onRotate := rf.onRotate + rf.mu.Unlock() + if onRotate != nil { + onRotate(*rotated) + } + return n, err +} + +// setHeader stores a copy of the header bytes for the next file. +func (rf *rotatingFile) setHeader(b []byte) { + header := make([]byte, len(b)) + copy(header, b) + rf.mu.Lock() + defer rf.mu.Unlock() + rf.header = header +} + +// writeHeader appends the stored header to the current file, once for each +// file that the writer opens. A restart uses it to mark the point where the +// new run continues an old file. +func (rf *rotatingFile) writeHeader() error { + rf.mu.Lock() + defer rf.mu.Unlock() + if rf.headerWritten { + return nil + } + return rf.writeHeaderLocked() +} + +// setBudget takes the disk space of a later config. The file stays open, so a +// changed key costs no second header. +func (rf *rotatingFile) setBudget(budget logBudget) { + rf.mu.Lock() + defer rf.mu.Unlock() + rf.budget = budget +} + +// rotateNow rotates the file although the budget is not spent. +func (rf *rotatingFile) rotateNow() (*logRotation, error) { + rf.mu.Lock() + defer rf.mu.Unlock() + return rf.rotateLocked() +} + +// paths returns the files that exist, oldest first, for the log readers. +func (rf *rotatingFile) paths() []string { + rf.mu.Lock() + defer rf.mu.Unlock() + files := make([]string, 0, rf.budget.backups+1) + for i := rf.budget.backups; i >= 1; i-- { + backup := rf.backupPath(i) + if _, err := os.Stat(backup); err == nil { + files = append(files, backup) + } + } + if _, err := os.Stat(rf.path); err == nil { + files = append(files, rf.path) + } + return files +} + +func (rf *rotatingFile) currentPath() string { + return rf.path +} + +func (rf *rotatingFile) close() error { + rf.mu.Lock() + defer rf.mu.Unlock() + if rf.f == nil { + return nil + } + err := rf.f.Close() + rf.f = nil + return err +} + +func (rf *rotatingFile) backupPath(index int) string { + return fmt.Sprintf("%s.%d", rf.path, index) +} + +// rotatingSuffix marks the file that a rotation with no backup moves away +// before it opens the new file. The old bytes stay behind the readers that +// hold them open, and the file goes as soon as the new file is open. +const rotatingSuffix = ".rotating" + +func (rf *rotatingFile) rotatingPath() string { + return rf.path + rotatingSuffix +} + +// rotatedPath names the file that the next rotation leaves behind. +func (rf *rotatingFile) rotatedPath() string { + if rf.budget.backups == 0 { + return rf.path + } + return rf.backupPath(1) +} + +// needsRotation reports whether p makes the file cross its budget. A closed +// file never rotates, because a rotation would shift the backups away for a +// write that cannot land. +func (rf *rotatingFile) needsRotation(n int) bool { + if rf.f == nil || rf.budget.maxSize <= 0 { + return false + } + return rf.size+int64(n) > rf.rotateFailedAt+rf.budget.maxSize +} + +// rotateLocked moves the current file away and opens an empty one in its +// place. It must run with rf.mu held. +func (rf *rotatingFile) rotateLocked() (*logRotation, error) { + record := &logRotation{ + file: rf.rotatedPath(), + bytesWritten: rf.size, + firstWrite: rf.firstWrite, + lastWrite: rf.lastWrite, + backups: rf.budget.backups, + } + if rf.f != nil { + rf.f.Close() + rf.f = nil + } + if err := rf.shiftBackups(); err != nil { + record.err = err + return record, rf.keepFileAfterFailedShift() + } + + f, err := rf.openFile(os.O_CREATE | os.O_RDWR | os.O_TRUNC) + if err != nil { + rf.size = 0 + return record, rf.noteOpenFailure(err) + } + if rf.budget.backups == 0 { + // The readers that hold the old file keep their bytes; the name goes. + _ = os.Remove(rf.rotatingPath()) + } + rf.f = f + rf.openErr = nil + rf.size = 0 + rf.rotateFailedAt = 0 + rf.firstWrite = time.Time{} + rf.lastWrite = time.Time{} + rf.headerWritten = false + return record, rf.writeHeaderLocked() +} + +// keepFileAfterFailedShift opens the file that could not move. The append flag +// keeps the lines that are already in it, which a truncating open would drop +// although no backup holds them. It must run with rf.mu held. +func (rf *rotatingFile) keepFileAfterFailedShift() error { + f, err := rf.openFile(os.O_CREATE | os.O_RDWR | os.O_APPEND) + if err != nil { + rf.size = 0 + return rf.noteOpenFailure(err) + } + rf.f = f + rf.openErr = nil + rf.rotateFailedAt = rf.size + return nil +} + +// noteOpenFailure keeps the error of a failed open and holds the next attempt +// back. It must run with rf.mu held. +func (rf *rotatingFile) noteOpenFailure(err error) error { + rf.openErr = fmt.Errorf("reopening log file: %w", err) + rf.retryOpenAt = time.Now().Add(retryOpenInterval) + return rf.openErr +} + +// shiftBackups moves each backup one number up and drops the oldest one. A +// missing backup is normal here, so those moves are best effort: a rotation +// must continue even when one backup cannot move. The move of the current +// file decides the rotation, so its error goes back to the caller. With no +// backup allowed, the current file still moves away, because a truncating +// reopen would change the bytes under an open upload reader. +func (rf *rotatingFile) shiftBackups() error { + if rf.budget.backups == 0 { + return rf.moveCurrentFile(rf.rotatingPath()) + } + for i := rf.budget.backups - 1; i >= 1; i-- { + _ = rf.renameFile(rf.backupPath(i), rf.backupPath(i+1)) + } + return rf.moveCurrentFile(rf.backupPath(1)) +} + +// moveCurrentFile renames the current file. A file that is not there is not +// an error, because a fresh path has nothing to move. +func (rf *rotatingFile) moveCurrentFile(to string) error { + if err := rf.renameFile(rf.path, to); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// pruneNumberedBackups removes the backups of path above keep. A lowered +// log_max_backups would otherwise leave old files that no reader lists. It +// reads the directory, because the numbers can have gaps. Only a name with an +// exact number after the path goes, because the directory can hold the files +// of another writer. +func pruneNumberedBackups(path string, keep int) { + dir := filepath.Dir(path) + entries, err := os.ReadDir(dir) + if err != nil { + return + } + prefix := filepath.Base(path) + "." + for _, entry := range entries { + suffix, found := strings.CutPrefix(entry.Name(), prefix) + if !found { + continue + } + index, err := strconv.Atoi(suffix) + if err != nil || index <= keep { + continue + } + os.Remove(filepath.Join(dir, entry.Name())) + } +} + +// writeHeaderLocked must run with rf.mu held. +func (rf *rotatingFile) writeHeaderLocked() error { + if len(rf.header) == 0 { + return nil + } + rf.headerWritten = true + _, err := rf.writeLocked(rf.header) + return err +} + +// writeLocked appends to the open file and counts the bytes. It must run +// with rf.mu held. +func (rf *rotatingFile) writeLocked(p []byte) (int, error) { + if rf.f == nil { + if rf.openErr == nil { + return 0, errLogFileNotOpen + } + if err := rf.retryOpenLocked(); err != nil { + return 0, err + } + } + n, err := rf.f.Write(p) + rf.size += int64(n) + return n, err +} + +// retryOpenLocked opens a file again that could not open. A file whose open +// failed once must not stay closed for the life of the process, because the +// cause is often a full disk or a directory that comes back. It must run with +// rf.mu held. +func (rf *rotatingFile) retryOpenLocked() error { + if time.Now().Before(rf.retryOpenAt) { + return rf.openErr + } + f, err := rf.openFile(os.O_CREATE | os.O_RDWR | os.O_APPEND) + if err != nil { + return rf.noteOpenFailure(err) + } + rf.f = f + rf.openErr = nil + rf.size = 0 + if st, err := f.Stat(); err == nil { + rf.size = st.Size() + } + rf.headerWritten = false + return rf.writeHeaderLocked() +} + +// noteEventWrite keeps the times that the rotation event reports. A header +// is not an event, so it does not move these times. +func (rf *rotatingFile) noteEventWrite() { + now := time.Now() + if rf.firstWrite.IsZero() { + rf.firstWrite = now + } + rf.lastWrite = now +} diff --git a/cmd/cli/log_file_test.go b/cmd/cli/log_file_test.go new file mode 100644 index 0000000..428be98 --- /dev/null +++ b/cmd/cli/log_file_test.go @@ -0,0 +1,553 @@ +package cli + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "sync/atomic" + "testing" + "time" +) + +const rotatingFileTestLineSize = 1024 + +// rotatingFileTestLine returns a line of a fixed size that names its +// generation, so a test can tell the content of one file from another. +func rotatingFileTestLine(generation, seq int) []byte { + head := fmt.Sprintf("gen=%d seq=%d ", generation, seq) + return []byte(head + strings.Repeat("x", rotatingFileTestLineSize-len(head)-1) + "\n") +} + +func newTestRotatingFile(t *testing.T, budget logBudget) *rotatingFile { + t.Helper() + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + return rf +} + +func rotatingFileTestContent(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(b) +} + +func writeRotatingFileTestLine(t *testing.T, rf *rotatingFile, generation, seq int) *logRotation { + t.Helper() + line := rotatingFileTestLine(generation, seq) + n, rotated, err := rf.writeReporting(line) + if err != nil { + t.Fatalf("write gen=%d seq=%d: %v", generation, seq, err) + } + if n != len(line) { + t.Fatalf("write gen=%d seq=%d wrote %d bytes, want %d", generation, seq, n, len(line)) + } + return rotated +} + +func Test_rotatingFileKeepsNumberedBackups(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 4}) + rotations := 0 + for generation := 1; generation <= 7; generation++ { + for seq := 1; seq <= 4; seq++ { + if writeRotatingFileTestLine(t, rf, generation, seq) != nil { + rotations++ + } + } + } + if rotations != 6 { + t.Fatalf("rotations = %d, want 6", rotations) + } + + generationOf := map[string]int{"": 7, ".1": 6, ".2": 5, ".3": 4, ".4": 3} + for suffix, generation := range generationOf { + content := rotatingFileTestContent(t, rf.path+suffix) + if want := fmt.Sprintf("gen=%d seq=1 ", generation); !strings.HasPrefix(content, want) { + t.Fatalf("file %q starts with %.16q, want %q", rf.path+suffix, content, want) + } + if newer := fmt.Sprintf("gen=%d ", generation+1); strings.Contains(content, newer) { + t.Fatalf("file %q holds %q", rf.path+suffix, newer) + } + } + if _, err := os.Stat(rf.path + ".5"); !os.IsNotExist(err) { + t.Fatalf("stat %s.5 = %v, want a not exist error", rf.path, err) + } +} + +func Test_rotatingFileHeaderStartsEveryFile(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 1}) + header := []byte(`{"level":"info","journal":true,"message":"Log header"}` + "\n") + rf.setHeader(header) + if err := rf.writeHeader(); err != nil { + t.Fatalf("writeHeader: %v", err) + } + if rf.size != int64(len(header)) { + t.Fatalf("size = %d, want %d", rf.size, len(header)) + } + if got := strings.Count(rotatingFileTestContent(t, rf.path), string(header)); got != 1 { + t.Fatalf("header count in the first file = %d, want 1", got) + } + + rotations := 0 + for seq := 1; seq <= 5; seq++ { + if writeRotatingFileTestLine(t, rf, 1, seq) != nil { + rotations++ + } + } + if rotations != 1 { + t.Fatalf("rotations = %d, want 1", rotations) + } + if !strings.HasPrefix(rotatingFileTestContent(t, rf.path+".1"), string(header)) { + t.Fatal("the backup file does not start with the header") + } + content := rotatingFileTestContent(t, rf.path) + if !strings.HasPrefix(content, string(header)) { + t.Fatal("the new file does not start with the header") + } + if got := strings.Count(content, string(header)); got != 1 { + t.Fatalf("header count in the new file = %d, want 1", got) + } + if want := int64(len(header) + 2*rotatingFileTestLineSize); rf.size != want { + t.Fatalf("size = %d, want %d", rf.size, want) + } +} + +func Test_rotatingFileRotationRecord(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 2}) + if rf.currentPath() != rf.path { + t.Fatalf("currentPath = %q, want %q", rf.currentPath(), rf.path) + } + start := time.Now() + for seq := 1; seq <= 3; seq++ { + writeRotatingFileTestLine(t, rf, 1, seq) + } + sizeBeforeRotation := rf.size + + record, err := rf.rotateNow() + if err != nil { + t.Fatalf("rotateNow: %v", err) + } + if record == nil { + t.Fatal("rotateNow returned no record") + } + if want := rf.path + ".1"; record.file != want { + t.Fatalf("record.file = %q, want %q", record.file, want) + } + if record.backups != 2 { + t.Fatalf("record.backups = %d, want 2", record.backups) + } + if record.bytesWritten != sizeBeforeRotation { + t.Fatalf("record.bytesWritten = %d, want %d", record.bytesWritten, sizeBeforeRotation) + } + st, err := os.Stat(record.file) + if err != nil { + t.Fatalf("stat %s: %v", record.file, err) + } + if st.Size() != record.bytesWritten { + t.Fatalf("%s holds %d bytes, record.bytesWritten = %d", record.file, st.Size(), record.bytesWritten) + } + if record.firstWrite.Before(start) { + t.Fatalf("record.firstWrite = %v, want at or after %v", record.firstWrite, start) + } + if record.lastWrite.Before(record.firstWrite) { + t.Fatalf("record.lastWrite = %v is before record.firstWrite = %v", record.lastWrite, record.firstWrite) + } + + want := []string{rf.path + ".1", rf.path} + if got := rf.paths(); !slices.Equal(got, want) { + t.Fatalf("paths = %v, want %v", got, want) + } + writeRotatingFileTestLine(t, rf, 2, 1) + if _, err := rf.rotateNow(); err != nil { + t.Fatalf("second rotateNow: %v", err) + } + want = []string{rf.path + ".2", rf.path + ".1", rf.path} + if got := rf.paths(); !slices.Equal(got, want) { + t.Fatalf("paths after two rotations = %v, want %v", got, want) + } +} + +func Test_rotatingFileZeroBackupsTruncatesInPlace(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 2048, backups: 0}) + var record *logRotation + for seq := 1; seq <= 3; seq++ { + if rotated := writeRotatingFileTestLine(t, rf, 1, seq); rotated != nil { + record = rotated + } + } + if record == nil { + t.Fatal("no rotation happened") + } + if record.file != rf.path { + t.Fatalf("record.file = %q, want %q", record.file, rf.path) + } + if record.backups != 0 { + t.Fatalf("record.backups = %d, want 0", record.backups) + } + if want := int64(2 * rotatingFileTestLineSize); record.bytesWritten != want { + t.Fatalf("record.bytesWritten = %d, want %d", record.bytesWritten, want) + } + if _, err := os.Stat(rf.path + ".1"); !os.IsNotExist(err) { + t.Fatalf("stat %s.1 = %v, want a not exist error", rf.path, err) + } + if content := rotatingFileTestContent(t, rf.path); strings.Contains(content, "seq=1 ") { + t.Fatal("the truncated file still holds the old content") + } + if got, want := rf.paths(), []string{rf.path}; !slices.Equal(got, want) { + t.Fatalf("paths = %v, want %v", got, want) + } +} + +func Test_rotatingFileWriteCallsOnRotate(t *testing.T) { + t.Run("one call per rotation", func(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 4}) + var rotations atomic.Int64 + rf.onRotate = func(logRotation) { rotations.Add(1) } + for generation := 1; generation <= 7; generation++ { + for seq := 1; seq <= 4; seq++ { + if _, err := rf.Write(rotatingFileTestLine(generation, seq)); err != nil { + t.Fatalf("Write: %v", err) + } + } + } + if got := rotations.Load(); got != 6 { + t.Fatalf("onRotate calls = %d, want 6", got) + } + }) + + t.Run("reentrant callback does not deadlock", func(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 1}) + const reentrantLine = "written from onRotate\n" + var rotations atomic.Int64 + rf.onRotate = func(logRotation) { + rotations.Add(1) + if _, err := rf.Write([]byte(reentrantLine)); err != nil { + t.Errorf("reentrant Write: %v", err) + } + } + done := make(chan struct{}) + go func() { + defer close(done) + for seq := 1; seq <= 5; seq++ { + if _, err := rf.Write(rotatingFileTestLine(1, seq)); err != nil { + t.Errorf("Write: %v", err) + return + } + } + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Write did not return, a reentrant onRotate deadlocks it") + } + if got := rotations.Load(); got != 1 { + t.Fatalf("onRotate calls = %d, want 1", got) + } + if !strings.Contains(rotatingFileTestContent(t, rf.path), reentrantLine) { + t.Fatal("the new file does not hold the reentrant line") + } + }) +} + +func Test_rotatingFileCountsExistingSize(t *testing.T) { + path := filepath.Join(t.TempDir(), "logs", "ctrld.log") + budget := logBudget{maxSize: 8192, backups: 1} + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + for seq := 1; seq <= 3; seq++ { + writeRotatingFileTestLine(t, rf, 1, seq) + } + if err := rf.close(); err != nil { + t.Fatalf("close: %v", err) + } + + reopened, err := newRotatingFile(path, budget, nil) + if err != nil { + t.Fatalf("newRotatingFile on an existing file: %v", err) + } + t.Cleanup(func() { _ = reopened.close() }) + if want := int64(3 * rotatingFileTestLineSize); reopened.size != want { + t.Fatalf("size = %d, want %d", reopened.size, want) + } + record, err := reopened.rotateNow() + if err != nil { + t.Fatalf("rotateNow: %v", err) + } + if want := int64(3 * rotatingFileTestLineSize); record.bytesWritten != want { + t.Fatalf("record.bytesWritten = %d, want %d", record.bytesWritten, want) + } + st, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + if st.Size() != 0 { + t.Fatalf("the new file holds %d bytes, want 0", st.Size()) + } +} + +func Test_rotatingFileKeepsReopenError(t *testing.T) { + reopenErr := errors.New("reopen refused") + var opens atomic.Int64 + open := func(path string, flags int) (*os.File, error) { + if opens.Add(1) > 1 { + return nil, reopenErr + } + return os.OpenFile(path, flags, 0600) + } + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, logBudget{maxSize: 1024, backups: 1}, open) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + + writeRotatingFileTestLine(t, rf, 1, 1) + if _, _, err := rf.writeReporting(rotatingFileTestLine(1, 2)); !errors.Is(err, reopenErr) { + t.Fatalf("write after a failed reopen = %v, want %v", err, reopenErr) + } + if rf.f != nil { + t.Fatal("the file is still open after a failed reopen") + } + if _, _, err := rf.writeReporting(rotatingFileTestLine(1, 3)); !errors.Is(err, reopenErr) { + t.Fatalf("later write = %v, want %v", err, reopenErr) + } + + oversized := make([]byte, 2*rotatingFileTestLineSize) + _, rotated, err := rf.writeReporting(oversized) + if !errors.Is(err, reopenErr) { + t.Fatalf("oversized write = %v, want %v", err, reopenErr) + } + if rotated != nil { + t.Fatalf("a closed file reported a rotation of %q", rotated.file) + } + if _, err := os.Stat(path + ".2"); !os.IsNotExist(err) { + t.Fatalf("stat %s.2 = %v, want a not exist error: a closed file shifted the backups", path, err) + } +} + +func Test_rotatingFileRetriesAFailedOpen(t *testing.T) { + origInterval := retryOpenInterval + retryOpenInterval = 10 * time.Millisecond + t.Cleanup(func() { retryOpenInterval = origInterval }) + + openErr := errors.New("open refused") + opens := 0 + open := func(path string, flags int) (*os.File, error) { + opens++ + if opens == 2 { + return nil, openErr + } + return os.OpenFile(path, flags, 0600) + } + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, logBudget{maxSize: rotatingFileTestLineSize, backups: 1}, open) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + header := []byte(`{"level":"info","journal":true,"message":"Log header"}` + "\n") + rf.setHeader(header) + + writeRotatingFileTestLine(t, rf, 1, 1) + if _, _, err := rf.writeReporting(rotatingFileTestLine(1, 2)); !errors.Is(err, openErr) { + t.Fatalf("write after a failed reopen = %v, want %v", err, openErr) + } + if _, _, err := rf.writeReporting(rotatingFileTestLine(1, 3)); !errors.Is(err, openErr) { + t.Fatalf("write inside the retry interval = %v, want %v", err, openErr) + } + if opens != 2 { + t.Fatalf("opens = %d, want 2: a write inside the retry interval opened the file again", opens) + } + + time.Sleep(2 * retryOpenInterval) + writeRotatingFileTestLine(t, rf, 1, 4) + + content := rotatingFileTestContent(t, path) + if !strings.HasPrefix(content, string(header)) { + t.Fatalf("the reopened file starts with %.40q, want the header", content) + } + if !strings.Contains(content, "gen=1 seq=4 ") { + t.Fatalf("the reopened file misses the line that followed the retry: %.80q", content) + } +} + +func Test_pruneNumberedBackups(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, logFileName) + numbered := []string{path + ".1", path + ".2", path + ".3", path + ".4"} + // A prune must keep every file that another writer owns, because log_path + // can name a file in a directory full of them. + foreign := []string{path + ".bak", filepath.Join(dir, journalLogFileName+".3"), filepath.Join(dir, "syslog.5")} + for _, name := range append(slices.Clone(numbered), foreign...) { + if err := os.WriteFile(name, []byte("old\n"), 0600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + + pruneNumberedBackups(path, 1) + + if _, err := os.Stat(numbered[0]); err != nil { + t.Fatalf("the backup inside the count went away: %v", err) + } + for _, name := range numbered[1:] { + if _, err := os.Stat(name); !os.IsNotExist(err) { + t.Fatalf("%s still exists after a prune that keeps one backup (err %v)", name, err) + } + } + for _, name := range foreign { + if _, err := os.Stat(name); err != nil { + t.Fatalf("the prune removed %s, which it does not own: %v", name, err) + } + } +} + +func Test_rotatingFileKeepsFilesAboveTheCount(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 2 * rotatingFileTestLineSize, backups: 1}) + late := rf.backupPath(3) + if err := os.WriteFile(late, []byte("late\n"), 0600); err != nil { + t.Fatalf("write the late backup: %v", err) + } + + for seq := 1; seq <= 3; seq++ { + if _, err := rf.Write(rotatingFileTestLine(1, seq)); err != nil { + t.Fatalf("write %d: %v", seq, err) + } + } + + if _, err := os.Stat(late); err != nil { + t.Fatalf("a rotation removed %s, which sits above the backup count: %v", late, err) + } + if _, err := os.Stat(rf.backupPath(1)); err != nil { + t.Fatalf("the rotation left no .1 file: %v", err) + } +} + +func Test_rotatingFileDoesNotFollowASymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the Windows opener takes no symlink flag") + } + dir := t.TempDir() + planted := filepath.Join(dir, "planted.log") + if err := os.WriteFile(planted, []byte("planted\n"), 0600); err != nil { + t.Fatalf("write the planted file: %v", err) + } + path := filepath.Join(dir, "ctrld.log") + if err := os.Symlink(planted, path); err != nil { + t.Fatalf("symlink: %v", err) + } + + rf, err := newRotatingFile(path, logBudget{maxSize: 4096, backups: 1}, nil) + + if err == nil { + _ = rf.close() + t.Fatal("newRotatingFile followed the symlink") + } + if got := rotatingFileTestContent(t, planted); got != "planted\n" { + t.Fatalf("the planted file = %q, want it untouched", got) + } +} + +func Test_rotatingFileKeepsContentWhenRenameFails(t *testing.T) { + renameErr := errors.New("rename refused") + renameFails := true + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, logBudget{maxSize: 2 * rotatingFileTestLineSize, backups: 1}, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + rf.rename = func(oldpath, newpath string) error { + if renameFails { + return renameErr + } + return os.Rename(oldpath, newpath) + } + + for seq := 1; seq <= 2; seq++ { + writeRotatingFileTestLine(t, rf, 1, seq) + } + record := writeRotatingFileTestLine(t, rf, 1, 3) + if record == nil { + t.Fatal("the write past the budget reported no rotation") + } + if !errors.Is(record.err, renameErr) { + t.Fatalf("record.err = %v, want %v", record.err, renameErr) + } + content := rotatingFileTestContent(t, path) + for seq := 1; seq <= 3; seq++ { + if want := fmt.Sprintf("gen=1 seq=%d ", seq); !strings.Contains(content, want) { + t.Fatalf("the file lost %q after a failed rename", want) + } + } + if _, err := os.Stat(path + ".1"); !os.IsNotExist(err) { + t.Fatalf("stat %s.1 = %v, want a not exist error", path, err) + } + if writeRotatingFileTestLine(t, rf, 1, 4) != nil { + t.Fatal("the next write rotated again instead of waiting for another budget") + } + + renameFails = false + var second *logRotation + for seq := 5; seq <= 6; seq++ { + if rotated := writeRotatingFileTestLine(t, rf, 1, seq); rotated != nil { + second = rotated + } + } + if second == nil { + t.Fatal("no rotation happened after the rename worked again") + } + if second.err != nil { + t.Fatalf("record.err = %v, want no error", second.err) + } + if _, err := os.Stat(path + ".1"); err != nil { + t.Fatalf("stat %s.1 after a working rename: %v", path, err) + } +} + +// Test_zeroBackupRotationKeepsTheOpenReader opens an upload reader over the +// current file, then rotates with no backup. The reader must return the bytes +// and the size that it advertised, and the current file holds the new +// generation only. +func Test_zeroBackupRotationKeepsTheOpenReader(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 1 << 20, backups: 0}) + if _, err := rf.Write([]byte("old generation!\n")); err != nil { + t.Fatal(err) + } + var upload logParts + upload.addBoundedFiles(rf.paths(), 0) + t.Cleanup(upload.close) + + if _, err := rf.rotateNow(); err != nil { + t.Fatalf("rotate: %v", err) + } + if _, err := rf.Write([]byte("new\n")); err != nil { + t.Fatal(err) + } + + got, err := io.ReadAll(io.MultiReader(upload.readers...)) + if err != nil { + t.Fatal(err) + } + if upload.size != 16 || string(got) != "old generation!\n" { + t.Fatalf("reader gave %q (%d bytes advertised), want the old generation of 16 bytes", got, upload.size) + } + if content := rotatingFileTestContent(t, rf.currentPath()); content != "new\n" { + t.Fatalf("current file = %q, want the new generation only", content) + } + if _, err := os.Stat(rf.currentPath() + rotatingSuffix); !os.IsNotExist(err) { + t.Fatalf("the moved file is still on disk: %v", err) + } +} diff --git a/cmd/cli/log_header.go b/cmd/cli/log_header.go new file mode 100644 index 0000000..80ad906 --- /dev/null +++ b/cmd/cli/log_header.go @@ -0,0 +1,218 @@ +package cli + +import ( + "bytes" + "net" + "os" + "runtime" + "slices" + "strconv" + "sync" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/Control-D-Inc/ctrld" +) + +// processStartTime is the time this process started. A reader compares it with +// the time of the header line to tell a restart from a rotation. +var processStartTime = time.Now() + +const ( + // logHeaderMessage is the message of the header line. Log tools select the + // header by this message. + logHeaderMessage = "Log header" + + // interceptModeOff is the mode word for a ctrld that does not intercept. + interceptModeOff = "off" + + // logHeaderTriggerSend marks the header that a log request renders. It + // tells a reader that the line came from an upload, not from a file open. + logHeaderTriggerSend = "send" + + // upstreamTypeUnspecified names an upstream that has no type in the config. + // ctrld resolves the type at start, so the header must not claim one. + upstreamTypeUnspecified = "unspecified" +) + +// headerOSVersion reads the OS version once. The read starts a subprocess on +// macOS, and a header refresh follows every network change. +var headerOSVersion = sync.OnceValue(osVersion) + +// logHeaderInput holds every value of a header line. The caller collects the +// values, so the renderer stays free of locks and of system calls. +type logHeaderInput struct { + Version string + Commit string + OS string + Arch string + PID int + StartTime time.Time + InterceptMode string + Listeners []string + UpstreamCount int + UpstreamTypes []string + ResolverUID string + LogFiles []string + Network networkSnapshot + Trigger string +} + +// renderLogHeader renders the header as one log line. A file writer stores the +// bytes and writes them itself, so the header reaches a file that the logger +// never fills. +func renderLogHeader(in logHeaderInput) []byte { + var buf bytes.Buffer + // The console sits above info on a default start, and a log file must hold + // its header at every level, so the header takes a core of its own. The + // line is JSON like the journal, because log tools read its fields. + logger := &ctrld.Logger{Logger: zap.New(zapcore.NewCore(newJournalEncoder(), zapcore.AddSync(&buf), zapcore.DebugLevel))} + event := journal(logger.Info()). + Str("version", in.Version). + Str("commit", in.Commit). + Str("os", in.OS). + Str("arch", in.Arch). + Int("pid", in.PID). + Str("start_time", in.StartTime.Format(time.RFC3339)). + Str("intercept_mode", in.InterceptMode). + Strs("listeners", in.Listeners). + Int("upstream_count", in.UpstreamCount). + Strs("upstream_types", in.UpstreamTypes). + Str("resolver_uid", in.ResolverUID). + Strs("log_files", in.LogFiles). + Dict("network", snapshotDict(in.Network)) + if in.Trigger != "" { + event = event.Str("trigger", in.Trigger) + } + event.Msg(logHeaderMessage) + + // The header is the first line of every file and of every upload, and it + // bypasses the writers, so it passes the journal redaction here. + return redactRetainedLine(buf.Bytes()) +} + +// logHeaderInput collects every header value of the running program. +func (p *prog) logHeaderInput() logHeaderInput { + return p.logHeaderInputWith(p.networkSnapshot()) +} + +// logHeaderInputWith collects every header value around one network snapshot. +func (p *prog) logHeaderInputWith(network networkSnapshot) logHeaderInput { + in := logHeaderInput{ + Version: curVersion(), + Commit: commit, + OS: headerOSVersion(), + Arch: runtime.GOARCH, + PID: os.Getpid(), + StartTime: processStartTime, + // The resolver UID is a secret that the provisioning code strips from + // every artifact, so the header keeps its first characters only. + ResolverUID: redactToken(cdUID), + LogFiles: p.logHeaderFiles(), + Network: network, + } + p.headerConfigInto(&in) + return in +} + +// headerConfigInto reads the config under p.mu, because a reload replaces the +// config under that lock while log view and log send render a header. +func (p *prog) headerConfigInto(in *logHeaderInput) { + p.mu.Lock() + defer p.mu.Unlock() + logHeaderFromConfig(p.cfg, in) +} + +// sendLogHeader renders the header that leads one upload. The stored state can +// be older than the request, and support reads this line to learn the network +// of the moment the user asked. +func (p *prog) sendLogHeader() []byte { + in := p.logHeaderInputWith(buildNetworkSnapshot(p.freshSnapshotInputs())) + in.Trigger = logHeaderTriggerSend + return renderLogHeader(in) +} + +// logHeaderFiles names the files that this run writes, debug first. +func (p *prog) logHeaderFiles() []string { + files := make([]string, 0, 3) + for _, rf := range p.openLogFiles() { + files = append(files, rf.currentPath()) + } + return files +} + +// refreshLogHeader renders the header and hands the bytes to every open file. +// The caller must hold no writer lock, because the render reads the config +// and the network. +func (p *prog) refreshLogHeader() { + header := renderLogHeader(p.logHeaderInput()) + for _, rf := range p.openLogFiles() { + rf.setHeader(header) + } +} + +// writeLogHeaders puts the header at the append point of each open file. A +// header that cannot land leaves the file usable, so the error only earns a +// warning. +func (p *prog) writeLogHeaders() { + for _, rf := range p.openLogFiles() { + if err := rf.writeHeader(); err != nil { + mainLog.Load().Warn().Err(err).Msg("Could not write log header") + } + } +} + +// logHeaderFromConfig fills the header fields that the config holds. It keeps +// every upstream endpoint, name, and bootstrap IP out of them, because a token +// can hide in each of them. +func logHeaderFromConfig(cfg *ctrld.Config, in *logHeaderInput) { + if cfg == nil { + in.InterceptMode = interceptModeOff + return + } + in.InterceptMode = headerInterceptMode(cfg) + in.Listeners = listenerAddresses(cfg) + in.UpstreamCount = len(cfg.Upstream) + in.UpstreamTypes = upstreamTypeWords(cfg) +} + +// headerInterceptMode names the mode for the header. An unset mode means that +// ctrld does not intercept, which reads as off. +func headerInterceptMode(cfg *ctrld.Config) string { + if mode := listenerInterceptMode(cfg); mode != "" { + return mode + } + return interceptModeOff +} + +func listenerAddresses(cfg *ctrld.Config) []string { + addresses := make([]string, 0, len(cfg.Listener)) + for _, listener := range cfg.Listener { + if listener == nil { + continue + } + addresses = append(addresses, net.JoinHostPort(listener.IP, strconv.Itoa(listener.Port))) + } + slices.Sort(addresses) + return addresses +} + +func upstreamTypeWords(cfg *ctrld.Config) []string { + words := make([]string, 0, len(cfg.Upstream)) + for _, upstream := range cfg.Upstream { + if upstream == nil { + continue + } + word := upstream.Type + if word == "" { + word = upstreamTypeUnspecified + } + if !slices.Contains(words, word) { + words = append(words, word) + } + } + slices.Sort(words) + return words +} diff --git a/cmd/cli/log_header_test.go b/cmd/cli/log_header_test.go new file mode 100644 index 0000000..f0b3631 --- /dev/null +++ b/cmd/cli/log_header_test.go @@ -0,0 +1,638 @@ +package cli + +import ( + "encoding/json" + "net" + "net/netip" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// logHeaderTestSecret is the string that must never reach a header line. +const logHeaderTestSecret = "secret-token" + +// logHeaderTestLine is the parsed form of a header line. The json tags name +// the fields that a log tool reads, so a renamed field fails the test. +type logHeaderTestLine struct { + Level string `json:"level"` + Time string `json:"time"` + Journal bool `json:"journal"` + Message string `json:"message"` + Version string `json:"version"` + Commit string `json:"commit"` + OS string `json:"os"` + Arch string `json:"arch"` + PID int `json:"pid"` + StartTime string `json:"start_time"` + InterceptMode string `json:"intercept_mode"` + Listeners []string `json:"listeners"` + UpstreamCount int `json:"upstream_count"` + UpstreamTypes []string `json:"upstream_types"` + ResolverUID string `json:"resolver_uid"` + LogFiles []string `json:"log_files"` + Network struct { + DefaultRouteV4 string `json:"default_route_v4"` + DefaultRouteV6 string `json:"default_route_v6"` + GatewayV4 string `json:"gateway_v4"` + GatewayV6 string `json:"gateway_v6"` + HaveV4 bool `json:"have_v4"` + HaveV6 bool `json:"have_v6"` + Interfaces []struct { + Name string `json:"name"` + Class string `json:"class"` + Up bool `json:"up"` + IPs []string `json:"ips"` + HardwarePort string `json:"hardware_port"` + Service string `json:"service"` + } `json:"interfaces"` + Resolvers []string `json:"resolvers"` + SourceIPv4 string `json:"source_ipv4"` + SourceIPv6 string `json:"source_ipv6"` + InterceptTarget string `json:"intercept_target"` + LinkType string `json:"link_type"` + Tethered bool `json:"tethered"` + } `json:"network"` + Trigger string `json:"trigger"` +} + +func logHeaderTestInput() logHeaderInput { + return logHeaderInput{ + Version: "1.5.8", + Commit: "abc1234", + OS: "darwin 15.6", + Arch: "arm64", + PID: 4242, + StartTime: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + InterceptMode: "dns", + Listeners: []string{"127.0.0.1:53", "127.0.0.1:5354"}, + UpstreamCount: 2, + UpstreamTypes: []string{"doh", "os"}, + ResolverUID: redactToken("abcd1234"), + LogFiles: []string{"/tmp/ctrld.log", "/tmp/ctrld-journal.log"}, + Network: networkSnapshot{ + DefaultRouteV4: "en0", + GatewayV4: "192.0.2.1", + HaveV4: true, + Interfaces: []snapshotInterface{ + {Name: "en0", Class: "hardware", Up: true, IPs: []string{"192.0.2.10/24"}, HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + }, + Resolvers: []string{"192.0.2.1:53"}, + LinkType: "wifi", + }, + } +} + +// parseLogHeader checks that the render is one line and returns it twice: as +// values, and as raw fields for the tests that ask whether a field is there. +func parseLogHeader(t *testing.T, line []byte) (logHeaderTestLine, map[string]json.RawMessage) { + t.Helper() + if len(line) == 0 { + t.Fatal("renderLogHeader returned no bytes") + } + if line[len(line)-1] != '\n' { + t.Fatalf("header does not end with a newline: %q", line) + } + if count := strings.Count(string(line), "\n"); count != 1 { + t.Fatalf("header holds %d newlines, want 1: %q", count, line) + } + var parsed logHeaderTestLine + if err := json.Unmarshal(line, &parsed); err != nil { + t.Fatalf("parse header %q: %v", line, err) + } + fields := map[string]json.RawMessage{} + if err := json.Unmarshal(line, &fields); err != nil { + t.Fatalf("parse header fields %q: %v", line, err) + } + return parsed, fields +} + +func Test_renderLogHeaderHoldsEveryField(t *testing.T) { + in := logHeaderTestInput() + + parsed, fields := parseLogHeader(t, renderLogHeader(in)) + + checks := []struct { + field string + got any + want any + }{ + {"level", parsed.Level, "info"}, + {"journal", parsed.Journal, true}, + {"message", parsed.Message, "Log header"}, + {"version", parsed.Version, in.Version}, + {"commit", parsed.Commit, in.Commit}, + {"os", parsed.OS, in.OS}, + {"arch", parsed.Arch, in.Arch}, + {"pid", parsed.PID, in.PID}, + {"start_time", parsed.StartTime, in.StartTime.Format(time.RFC3339)}, + {"intercept_mode", parsed.InterceptMode, in.InterceptMode}, + {"listeners", parsed.Listeners, in.Listeners}, + {"upstream_count", parsed.UpstreamCount, in.UpstreamCount}, + {"upstream_types", parsed.UpstreamTypes, in.UpstreamTypes}, + // The resolver UID is a secret, so the header keeps the first four + // characters only. + {"resolver_uid", parsed.ResolverUID, "abcd***"}, + {"log_files", parsed.LogFiles, in.LogFiles}, + {"network.default_route_v4", parsed.Network.DefaultRouteV4, in.Network.DefaultRouteV4}, + {"network.default_route_v6", parsed.Network.DefaultRouteV6, in.Network.DefaultRouteV6}, + {"network.gateway_v4", parsed.Network.GatewayV4, in.Network.GatewayV4}, + {"network.have_v4", parsed.Network.HaveV4, in.Network.HaveV4}, + {"network.have_v6", parsed.Network.HaveV6, in.Network.HaveV6}, + {"network.resolvers", parsed.Network.Resolvers, in.Network.Resolvers}, + {"network.link_type", parsed.Network.LinkType, in.Network.LinkType}, + {"network.tethered", parsed.Network.Tethered, in.Network.Tethered}, + } + for _, c := range checks { + if !reflect.DeepEqual(c.got, c.want) { + t.Errorf("%s = %v, want %v", c.field, c.got, c.want) + } + } + + if len(parsed.Network.Interfaces) != 1 { + t.Fatalf("network.interfaces holds %d entries, want 1", len(parsed.Network.Interfaces)) + } + iface := parsed.Network.Interfaces[0] + if iface.Name != "en0" || iface.Class != "hardware" || !iface.Up || iface.HardwarePort != "Wi-Fi" { + t.Errorf("network.interfaces[0] = %+v, want the hardware port en0 up on Wi-Fi", iface) + } + if !reflect.DeepEqual(iface.IPs, []string{"192.0.2.10/24"}) { + t.Errorf("network.interfaces[0].ips = %v, want 192.0.2.10/24", iface.IPs) + } + if parsed.Time == "" { + t.Error("time is empty") + } + if _, ok := fields["trigger"]; ok { + t.Errorf("trigger is present for an empty Trigger: %s", fields["trigger"]) + } +} + +func Test_renderLogHeaderAddsTriggerWhenSet(t *testing.T) { + in := logHeaderTestInput() + in.Trigger = "send" + + parsed, _ := parseLogHeader(t, renderLogHeader(in)) + + if parsed.Trigger != "send" { + t.Errorf("trigger = %q, want %q", parsed.Trigger, "send") + } +} + +// Test_renderLogHeaderIgnoresConsoleLevel guards the promise that every log +// file starts with a header. The console core sits above info on a default +// start, and the header takes a core of its own. +func Test_renderLogHeaderIgnoresConsoleLevel(t *testing.T) { + parsed, _ := parseLogHeader(t, renderLogHeader(logHeaderTestInput())) + + if parsed.Level != "info" { + t.Errorf("level = %q, want %q", parsed.Level, "info") + } + if parsed.Message != "Log header" { + t.Errorf("message = %q, want %q", parsed.Message, "Log header") + } +} + +func Test_logHeaderFromConfigKeepsUpstreamSecretsOut(t *testing.T) { + old := interceptMode + t.Cleanup(func() { interceptMode = old }) + interceptMode = "" + + cfg := &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{ + "0": {IP: "127.0.0.1", Port: 53}, + logHeaderTestSecret + "-l": {IP: "10.0.0.1", Port: 5354}, + }, + Upstream: map[string]*ctrld.UpstreamConfig{ + "0": { + Name: logHeaderTestSecret + "-name", + Type: "doh", + Endpoint: "https://dns.example/" + logHeaderTestSecret, + BootstrapIP: "192.0.2.53", + }, + "1": {Endpoint: "https://dns.example/" + logHeaderTestSecret}, + }, + } + cfg.Service.InterceptMode = "dns" + + in := logHeaderTestInput() + logHeaderFromConfig(cfg, &in) + + if in.InterceptMode != "dns" { + t.Errorf("intercept mode = %q, want %q", in.InterceptMode, "dns") + } + if want := []string{"10.0.0.1:5354", "127.0.0.1:53"}; !reflect.DeepEqual(in.Listeners, want) { + t.Errorf("listeners = %v, want %v", in.Listeners, want) + } + if in.UpstreamCount != 2 { + t.Errorf("upstream count = %d, want 2", in.UpstreamCount) + } + if want := []string{"doh", "unspecified"}; !reflect.DeepEqual(in.UpstreamTypes, want) { + t.Errorf("upstream types = %v, want %v", in.UpstreamTypes, want) + } + + line := renderLogHeader(in) + if strings.Contains(string(line), logHeaderTestSecret) { + t.Errorf("header holds %q: %s", logHeaderTestSecret, line) + } +} + +func Test_logHeaderFromConfigReportsOffWhenUnset(t *testing.T) { + old := interceptMode + t.Cleanup(func() { interceptMode = old }) + interceptMode = "" + + var in logHeaderInput + logHeaderFromConfig(&ctrld.Config{}, &in) + + if in.InterceptMode != "off" { + t.Errorf("intercept mode = %q, want %q", in.InterceptMode, "off") + } +} + +// Test_renderLogHeaderShowsTheSnapshotOfAState renders the header of one +// network state, so a reader of the file sees every interface of the host in +// one order. +func Test_renderLogHeaderShowsTheSnapshotOfAState(t *testing.T) { + stubHeaderSnapshotSources(t) + state := &netmon.State{ + DefaultRouteInterface: "en0", + HaveV4: true, + Interface: map[string]netmon.Interface{ + "en0": {Interface: &net.Interface{Name: "en0", Flags: net.FlagUp}}, + "awdl0": {Interface: &net.Interface{Name: "awdl0"}}, + }, + InterfaceIPs: map[string][]netip.Prefix{ + "en0": {netip.MustParsePrefix("192.0.2.10/24"), netip.MustParsePrefix("2001:db8::1/64")}, + "awdl0": {netip.MustParsePrefix("fe80::2/64")}, + }, + } + + in := logHeaderTestInput() + in.Network = buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "192.0.2.1", Interface: "en0"}, + RouteV6: defaultRoute{Gateway: "fe80::1%en0", Interface: "en0"}, + Meta: interfaceMetaFor, + }) + parsed, _ := parseLogHeader(t, renderLogHeader(in)) + + if parsed.Network.DefaultRouteV4 != "en0" || parsed.Network.DefaultRouteV6 != "en0" { + t.Errorf("default route = %q, %q, want en0 for both families", parsed.Network.DefaultRouteV4, parsed.Network.DefaultRouteV6) + } + if len(parsed.Network.Interfaces) != 2 { + t.Fatalf("network.interfaces holds %d entries, want 2", len(parsed.Network.Interfaces)) + } + if parsed.Network.Interfaces[0].Name != "awdl0" || parsed.Network.Interfaces[0].Class != "airdrop" || parsed.Network.Interfaces[0].Up { + t.Errorf("network.interfaces[0] = %+v, want awdl0 down in the airdrop class", parsed.Network.Interfaces[0]) + } + if got := parsed.Network.Interfaces[1].IPs; !reflect.DeepEqual(got, []string{"192.0.2.10/24", "2001:db8::1/64"}) { + t.Errorf("network.interfaces[1].ips = %v", got) + } +} + +// setupLogHeaderWiringTest puts ctrld in cd mode with a temporary home +// directory, so initInternalLogging writes its files there. Every global that +// the logging setup touches goes back when the test ends. +func setupLogHeaderWiringTest(t *testing.T) string { + t.Helper() + stubHeaderSnapshotSources(t) + origSilent, origCdUID, origHomedir, origVerbose := silent, cdUID, homedir, verbose + origMainLog := mainLog.Load() + origLogPathFile := logPathFile.Load() + t.Cleanup(func() { + silent, cdUID, homedir, verbose = origSilent, origCdUID, origHomedir, origVerbose + mainLog.Store(origMainLog) + logPathFile.Store(origLogPathFile) + }) + + homedir = t.TempDir() + cdUID = "header-wiring-uid" + silent, verbose = false, 0 + mainLog.Store(ctrld.NopLogger) + logPathFile.Store(nil) + return homedir +} + +// startLogHeaderWiringProg runs the internal logging setup and closes both +// files when the test ends. +func startLogHeaderWiringProg(t *testing.T) *prog { + t.Helper() + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.initInternalLogging(nil) + t.Cleanup(func() { + p.internalLogWriter.closeLogFile() + p.internalJournalWriter.closeLogFile() + }) + return p +} + +// logHeaderFileLines parses every JSON line of a log file. It returns line 1 +// and the number of header lines, so a test can tell a missing header from a +// repeated one. The debug stream writes console lines, which hold no header, +// so they do not count. +func logHeaderFileLines(t *testing.T, path string) (logHeaderTestLine, int) { + t.Helper() + content := strings.TrimRight(rotatingFileTestContent(t, path), "\n") + if content == "" { + t.Fatalf("%s is empty", path) + } + var firstLine logHeaderTestLine + headers := 0 + for i, line := range strings.Split(content, "\n") { + if i > 0 && !strings.HasPrefix(line, "{") { + continue + } + var parsed logHeaderTestLine + if err := json.Unmarshal([]byte(line), &parsed); err != nil { + t.Fatalf("parse line %d of %s (%q): %v", i+1, path, line, err) + } + if i == 0 { + firstLine = parsed + } + if parsed.Message == logHeaderMessage { + headers++ + } + } + return firstLine, headers +} + +func Test_logHeaderWiring_writesHeaderOnStart(t *testing.T) { + dir := setupLogHeaderWiringTest(t) + + startLogHeaderWiringProg(t) + + for _, name := range []string{logFileName, journalLogFileName} { + path := filepath.Join(dir, name) + firstLine, headers := logHeaderFileLines(t, path) + if firstLine.Message != logHeaderMessage { + t.Errorf("line 1 of %s = %q, want %q", path, firstLine.Message, logHeaderMessage) + } + if !firstLine.Journal { + t.Errorf("line 1 of %s carries no journal marker", path) + } + if want := redactToken(cdUID); firstLine.ResolverUID != want { + t.Errorf("resolver_uid in %s = %q, want %q", path, firstLine.ResolverUID, want) + } + if headers != 1 { + t.Errorf("%s holds %d header lines, want 1", path, headers) + } + } +} + +func Test_logHeaderInput_keepsSecretsOut(t *testing.T) { + dir := setupLogHeaderWiringTest(t) + origInterceptMode := interceptMode + t.Cleanup(func() { interceptMode = origInterceptMode }) + interceptMode = "" + p := &prog{cfg: &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{ + logHeaderTestSecret + "-listener": {IP: "127.0.0.1", Port: 53}, + }, + Upstream: map[string]*ctrld.UpstreamConfig{ + "0": { + Name: logHeaderTestSecret + "-name", + Type: "doh", + Endpoint: "https://dns.example/" + logHeaderTestSecret, + BootstrapIP: "192.0.2.53", + Domain: logHeaderTestSecret + ".example", + }, + }, + }} + p.initInternalLogging(nil) + t.Cleanup(func() { + p.internalLogWriter.closeLogFile() + p.internalJournalWriter.closeLogFile() + }) + + if line := renderLogHeader(p.logHeaderInput()); strings.Contains(string(line), logHeaderTestSecret) { + t.Errorf("the header holds %q: %s", logHeaderTestSecret, line) + } + + for _, name := range []string{logFileName, journalLogFileName} { + path := filepath.Join(dir, name) + if content := rotatingFileTestContent(t, path); strings.Contains(content, logHeaderTestSecret) { + t.Errorf("%s holds %q: %s", path, logHeaderTestSecret, content) + } + } +} + +func Test_logHeaderWiring_secondInitAddsNoHeader(t *testing.T) { + dir := setupLogHeaderWiringTest(t) + p := startLogHeaderWiringProg(t) + + p.initInternalLogging(nil) + + for _, name := range []string{logFileName, journalLogFileName} { + path := filepath.Join(dir, name) + if _, headers := logHeaderFileLines(t, path); headers != 1 { + t.Errorf("%s holds %d header lines after a second start, want 1", path, headers) + } + } +} + +func Test_logHeaderWiring_headerLeadsRotatedFile(t *testing.T) { + captureMainLog(t) + lw, path := newTestFileLogWriter(t, testLogWriterBudget) + header := string(renderLogHeader(logHeaderInput{Version: "v1"})) + + rf := lw.rotating() + rf.setHeader([]byte(header)) + if err := rf.writeHeader(); err != nil { + t.Fatalf("writeHeader: %v", err) + } + linesPerFile := int(testLogWriterBudget.maxSize) / rotatingFileTestLineSize + for seq := 1; seq <= linesPerFile+1; seq++ { + if _, err := lw.Write(rotatingFileTestLine(1, seq)); err != nil { + t.Fatalf("write line %d: %v", seq, err) + } + } + + for _, name := range []string{path + ".1", path} { + if content := rotatingFileTestContent(t, name); !strings.HasPrefix(content, header) { + t.Errorf("%s starts with %q, want the header", name, strings.SplitN(content, "\n", 2)[0]) + } + } +} + +func Test_logHeaderWiring_restartAppendsHeader(t *testing.T) { + dir := setupLogHeaderWiringTest(t) + path := filepath.Join(dir, logFileName) + const seed = `{"level":"info","message":"line of the run before"}` + if err := os.WriteFile(path, []byte(seed+"\n"), 0o600); err != nil { + t.Fatalf("seed %s: %v", path, err) + } + + startLogHeaderWiringProg(t) + + lines := strings.Split(strings.TrimRight(rotatingFileTestContent(t, path), "\n"), "\n") + if len(lines) < 2 { + t.Fatalf("%s holds %d lines, want the seed line and the header", path, len(lines)) + } + if lines[0] != seed { + t.Errorf("line 1 of %s = %q, want the seed line", path, lines[0]) + } + var header logHeaderTestLine + if err := json.Unmarshal([]byte(lines[1]), &header); err != nil { + t.Fatalf("parse line 2 of %s (%q): %v", path, lines[1], err) + } + if header.Message != logHeaderMessage { + t.Errorf("line 2 of %s = %q, want %q", path, header.Message, logHeaderMessage) + } + if _, headers := logHeaderFileLines(t, path); headers != 1 { + t.Errorf("%s holds %d header lines, want 1", path, headers) + } +} + +func Test_logHeaderWiring_usesLastNetworkState(t *testing.T) { + stubHeaderSnapshotSources(t) + origRead := readNetworkSourceStateFn + t.Cleanup(func() { readNetworkSourceStateFn = origRead }) + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + + p.lastNetworkState.Store(&netmon.State{ + DefaultRouteInterface: "en7", + Interface: map[string]netmon.Interface{"en7": {Interface: &net.Interface{Name: "en7", Flags: net.FlagUp}}}, + InterfaceIPs: map[string][]netip.Prefix{"en7": {netip.MustParsePrefix("192.0.2.7/24")}}, + }) + + if got := p.logHeaderInput().Network.DefaultRouteV4; got != "en7" { + t.Errorf("default_route_v4 = %q, want %q", got, "en7") + } + if got := p.logHeaderInput().Network.GatewayV4; got != "192.0.2.1" { + t.Errorf("gateway_v4 = %q, want %q", got, "192.0.2.1") + } + + p.lastNetworkState.Store(nil) + readNetworkSourceStateFn = func() (*netmon.State, error) { + return &netmon.State{ + Interface: map[string]netmon.Interface{"en9": {Interface: &net.Interface{Name: "en9", Flags: net.FlagUp}}}, + InterfaceIPs: map[string][]netip.Prefix{"en9": {netip.MustParsePrefix("192.0.2.9/24")}}, + }, nil + } + + network := p.logHeaderInput().Network + if len(network.Interfaces) != 1 || network.Interfaces[0].Name != "en9" || !network.Interfaces[0].Up { + t.Fatalf("interfaces = %+v, want one up en9", network.Interfaces) + } + if !network.HaveV4 { + t.Error("have_v4 = false, want true for a global unicast v4 address") + } + if network.HaveV6 { + t.Error("have_v6 = true, want false without a v6 address") + } +} + +// stubHeaderSnapshotSources keeps the header snapshot away from the host. One +// render reads the hardware ports, the route table, and the NAT64 class, and no +// test may run the commands behind them. +func stubHeaderSnapshotSources(t *testing.T) { + t.Helper() + stubSnapshotDNS64(t) + stubSnapshotGateways(t, "192.0.2.1", "") + stubSnapshotPlatformMeta(t, nil) + stubSnapshotVirtualSet(t) +} + +// stubLogHeaderNetworkRead makes the fresh network read return one up +// interface, so a header test never depends on the host network. +func stubLogHeaderNetworkRead(t *testing.T, name string) { + t.Helper() + stubHeaderSnapshotSources(t) + origRead := readNetworkSourceStateFn + t.Cleanup(func() { readNetworkSourceStateFn = origRead }) + readNetworkSourceStateFn = func() (*netmon.State, error) { + return &netmon.State{ + Interface: map[string]netmon.Interface{name: {Interface: &net.Interface{Name: name, Flags: net.FlagUp}}}, + InterfaceIPs: map[string][]netip.Prefix{name: {netip.MustParsePrefix("192.0.2.9/24")}}, + }, nil + } +} + +func Test_sendLogHeader_usesFreshNetworkAndTrigger(t *testing.T) { + stubHeaderSnapshotSources(t) + stubLogHeaderNetworkRead(t, "en9") + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.lastNetworkState.Store(&netmon.State{ + DefaultRouteInterface: "en7", + Interface: map[string]netmon.Interface{"en7": {Interface: &net.Interface{Name: "en7", Flags: net.FlagUp}}}, + InterfaceIPs: map[string][]netip.Prefix{"en7": {netip.MustParsePrefix("192.0.2.7/24")}}, + }) + + parsed, _ := parseLogHeader(t, p.sendLogHeader()) + + if parsed.Trigger != "send" { + t.Errorf("trigger = %q, want %q", parsed.Trigger, "send") + } + if parsed.Message != logHeaderMessage { + t.Errorf("message = %q, want %q", parsed.Message, logHeaderMessage) + } + names := make([]string, 0, len(parsed.Network.Interfaces)) + for _, iface := range parsed.Network.Interfaces { + names = append(names, iface.Name) + } + if want := []string{"en9"}; !reflect.DeepEqual(names, want) { + t.Fatalf("network.interfaces = %v, want %v from the fresh read", names, want) + } +} + +// Test_logHeaderInputReadsTheConfigUnderTheLock renders headers beside the +// config swap of a reload. The race detector fails the test when the render +// reads the config outside the lock. +func Test_logHeaderInputReadsTheConfigUnderTheLock(t *testing.T) { + stubLogHeaderNetworkRead(t, "en0") + p := &prog{cfg: &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}, + Upstream: map[string]*ctrld.UpstreamConfig{"0": {Type: "doh"}}, + }} + reloaded := &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}}, + Upstream: map[string]*ctrld.UpstreamConfig{"0": {Type: "doq"}, "1": {Type: "legacy"}}, + } + + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + } + p.mu.Lock() + *p.cfg = *reloaded + p.mu.Unlock() + } + }() + for i := 0; i < 300; i++ { + if in := p.logHeaderInput(); len(in.Listeners) != 1 { + t.Fatalf("listeners = %v, want one", in.Listeners) + } + } + close(stop) + <-done +} + +// Test_renderLogHeaderPassesTheSink puts a provisioning secret where the +// header does not expect one. The header is the first line of every file and +// of every upload, so it must pass the same redaction as every other line. +func Test_renderLogHeaderPassesTheSink(t *testing.T) { + origUID := cdUID + cdUID = "abcd1234SECRETUID" + t.Cleanup(func() { cdUID = origUID }) + + line := string(renderLogHeader(logHeaderInput{Version: "1.0", LogFiles: []string{"/var/log/" + cdUID + "/ctrld.log"}})) + + if strings.Contains(line, cdUID) { + t.Fatalf("the header holds the secret: %s", line) + } +} diff --git a/cmd/cli/log_journal.go b/cmd/cli/log_journal.go new file mode 100644 index 0000000..7f998ad --- /dev/null +++ b/cmd/cli/log_journal.go @@ -0,0 +1,146 @@ +package cli + +import ( + "io" + "regexp" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// journalField marks an event for the retained journal stream. Both packages +// share the name, so one core keeps the lines of both. +const journalField = ctrld.JournalField + +// retainedURLPath matches the part of a URL that follows the host, for every +// scheme: a DoH, DoH3, or DoQ endpoint carries its token in the path. The +// class stops the match at the characters that end a value inside a log line. +var retainedURLPath = regexp.MustCompile(`([a-z][a-z0-9+.-]*://[^/?"\\\s]+)[/?][^"\\\s]*`) + +// retainedControlDHost matches a Control D resolver host. The first label of +// the host is the resolver ID, which a DoT or DoQ endpoint carries in place of +// a path. +var retainedControlDHost = regexp.MustCompile(`([A-Za-z0-9-]+)(\.dns\.controld\.(?:com|dev))`) + +// redactRetainedLine strips the secrets of one line that the journal keeps. +// Support downloads the journal whole, and any error text can carry a DoH URL +// with the resolver path and the packed query of the user. +func redactRetainedLine(p []byte) []byte { + stripped := retainedURLPath.ReplaceAll(p, []byte(`${1}/[redacted]`)) + stripped = retainedControlDHost.ReplaceAll(stripped, []byte(`[redacted]${2}`)) + return []byte(redactSecrets(string(stripped), journalSecrets()...)) +} + +// journalSecrets names the values that no retained line may hold. The client +// ID stays out: it is a device label, not a secret. redactSecrets drops the +// values that are too short to redact. +func journalSecrets() []string { + uid, _ := controld.ParseRawUID(cdUID) + return []string{cdUID, cdOrg, uid} +} + +// journal marks an event for the retained journal stream. The event keeps +// its own level, so an info event stays info on the console. +func journal(e *ctrld.LogEvent) *ctrld.LogEvent { + return ctrld.Journal(e) +} + +// journalTimeLayout is the time format of a journal line: RFC 3339 with +// milliseconds. +const journalTimeLayout = "2006-01-02T15:04:05.000Z07:00" + +// newJournalEncoder is the encoder of the journal and of the header line: one +// JSON object for each line, with the field names that the log tools read. +// The debug stream keeps its console layout for the operator who reads it on +// the host. +func newJournalEncoder() zapcore.Encoder { + encoderConfig := zap.NewProductionEncoderConfig() + encoderConfig.TimeKey = "time" + encoderConfig.EncodeTime = zapcore.TimeEncoderOfLayout(journalTimeLayout) + encoderConfig.LevelKey = "level" + encoderConfig.EncodeLevel = zapcore.LowercaseLevelEncoder + encoderConfig.MessageKey = "message" + return zapcore.NewJSONEncoder(encoderConfig) +} + +// journalCore writes the lines that the journal keeps and drops the rest, so +// the journal holds a small history that survives a restart. It keeps every +// line at warn level or above, and every line that carries the journal field. +// Notice shares its level value with warn, so a notice line stays too. +type journalCore struct { + zapcore.LevelEnabler + encoder zapcore.Encoder + out zapcore.WriteSyncer + // marked is set when With bound the journal field, so every line of that + // logger belongs to the journal. + marked bool +} + +var _ zapcore.Core = (*journalCore)(nil) + +// newJournalCore makes the core of the journal stream. It accepts every level, +// because a debug line with the journal field belongs to the journal too. +func newJournalCore(w io.Writer) *journalCore { + return &journalCore{ + LevelEnabler: zapcore.DebugLevel, + encoder: newJournalEncoder(), + out: zapcore.AddSync(w), + } +} + +func (c *journalCore) With(fields []zapcore.Field) zapcore.Core { + clone := &journalCore{ + LevelEnabler: c.LevelEnabler, + encoder: c.encoder.Clone(), + out: c.out, + marked: c.marked || hasJournalField(fields), + } + for _, field := range fields { + field.AddTo(clone.encoder) + } + return clone +} + +func (c *journalCore) Check(entry zapcore.Entry, checked *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if c.Enabled(entry.Level) { + return checked.AddCore(entry, c) + } + return checked +} + +// Write keeps or drops one line. A dropped line is not an error. +func (c *journalCore) Write(entry zapcore.Entry, fields []zapcore.Field) error { + if !c.retains(entry, fields) { + return nil + } + buf, err := c.encoder.EncodeEntry(entry, fields) + if err != nil { + return err + } + defer buf.Free() + _, err = c.out.Write(redactRetainedLine(buf.Bytes())) + return err +} + +func (c *journalCore) Sync() error { + return c.out.Sync() +} + +// retains reads the level first, because a compare costs less than a scan of +// the fields. +func (c *journalCore) retains(entry zapcore.Entry, fields []zapcore.Field) bool { + return c.marked || entry.Level >= zapcore.WarnLevel || hasJournalField(fields) +} + +// hasJournalField reports whether fields carry the journal marker. +func hasJournalField(fields []zapcore.Field) bool { + for _, field := range fields { + if field.Key == journalField && field.Type == zapcore.BoolType && field.Integer == 1 { + return true + } + } + return false +} diff --git a/cmd/cli/log_journal_test.go b/cmd/cli/log_journal_test.go new file mode 100644 index 0000000..1cdb5ad --- /dev/null +++ b/cmd/cli/log_journal_test.go @@ -0,0 +1,292 @@ +package cli + +import ( + "bytes" + "path/filepath" + "strings" + "testing" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/Control-D-Inc/ctrld" +) + +func Test_journalSetsMarker(t *testing.T) { + var buf bytes.Buffer + logger := newTestJSONLogger(&buf) + journal(logger.Info()).Msg("marked") + if !bytes.Contains(buf.Bytes(), journalMarker) { + t.Fatalf("journal event lacks the marker: %s", buf.String()) + } + if !bytes.Contains(buf.Bytes(), []byte(`"level":"info"`)) { + t.Fatalf("journal event changed its level: %s", buf.String()) + } +} + +// newJournalTestLogger returns a logger whose only core is the journal core. +func newJournalTestLogger(buf *bytes.Buffer) *ctrld.Logger { + return &ctrld.Logger{Logger: zap.New(newJournalCore(buf))} +} + +func Test_journalCoreKeepsWarningsAndMarkedLines(t *testing.T) { + for _, tc := range []struct { + name string + log func(logger *ctrld.Logger) + kept bool + }{ + {"info with the marker", func(logger *ctrld.Logger) { journal(logger.Info()).Msg("state") }, true}, + {"info without the marker", func(logger *ctrld.Logger) { logger.Info().Msg("state") }, false}, + {"warn without the marker", func(logger *ctrld.Logger) { logger.Warn().Msg("slow") }, true}, + {"debug with the marker", func(logger *ctrld.Logger) { journal(logger.Debug()).Msg("state") }, true}, + // Notice shares its level value with warn, so the journal keeps it. + {"notice without the marker", func(logger *ctrld.Logger) { logger.Notice().Msg("note") }, true}, + {"error without the marker", func(logger *ctrld.Logger) { logger.Error().Msg("failed") }, true}, + } { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + tc.log(newJournalTestLogger(&buf)) + if kept := buf.Len() > 0; kept != tc.kept { + t.Fatalf("kept = %v, want %v, line: %q", kept, tc.kept, buf.String()) + } + }) + } +} + +// Test_journalCoreWriteKeepsMarkedEntryOnly drives the core directly, which +// is the path that zap takes for every line. +func Test_journalCoreWriteKeepsMarkedEntryOnly(t *testing.T) { + var buf bytes.Buffer + core := newJournalCore(&buf) + entry := zapcore.Entry{Level: zapcore.InfoLevel, Message: "state"} + + if err := core.Write(entry, []zapcore.Field{zap.Bool(journalField, true)}); err != nil { + t.Fatalf("write of a marked entry: %v", err) + } + if !bytes.Contains(buf.Bytes(), journalMarker) { + t.Fatalf("journal holds %q, want the marked line", buf.String()) + } + marked := buf.Len() + + if err := core.Write(entry, nil); err != nil { + t.Fatalf("write of a plain entry: %v", err) + } + if buf.Len() != marked { + t.Fatalf("journal kept an unmarked line: %q", buf.String()) + } +} + +// Test_journalCoreWithBindsTheMarker proves that a logger with the field +// bound keeps every line, because With moves the field out of Write. +func Test_journalCoreWithBindsTheMarker(t *testing.T) { + var buf bytes.Buffer + logger := newJournalTestLogger(&buf).Bool(journalField, true) + + logger.Info().Msg("state") + + if !bytes.Contains(buf.Bytes(), journalMarker) { + t.Fatalf("journal holds %q, want the bound line", buf.String()) + } +} + +// journalDoHError has the shape of a resolver error text: the DoH URL carries +// the resolver path and the packed query. +const journalDoHError = `Get "https://dns.example/org-v1-SECRET0123?dns=pOgBAAAB": dial tcp` + +// journalRedactedURL is what the sink keeps of journalDoHError. +const journalRedactedURL = "https://dns.example/[redacted]" + +// journalTestLogger returns a logger that writes through the journal sink. +func journalTestLogger(t *testing.T) (*bytes.Buffer, *ctrld.Logger) { + t.Helper() + var buf bytes.Buffer + return &buf, newJournalTestLogger(&buf) +} + +func Test_journalCoreRedactsAKeptLine(t *testing.T) { + buf, logger := journalTestLogger(t) + + logger.Warn().Str("error", journalDoHError).Msg("upstream failed") + + line := buf.String() + if !strings.Contains(line, journalRedactedURL) { + t.Fatalf("the sink kept the URL path: %s", line) + } + for _, secret := range []string{"SECRET0123", "pOgBAAAB"} { + if strings.Contains(line, secret) { + t.Fatalf("the sink kept %q: %s", secret, line) + } + } +} + +func Test_journalCoreRedactsTheProvisionSecrets(t *testing.T) { + origCdUID := cdUID + t.Cleanup(func() { cdUID = origCdUID }) + cdUID = "org-v1-uidofthisdevice" + buf, logger := journalTestLogger(t) + + logger.Warn().Str("detail", "device "+cdUID+" is gone").Msg("provision failed") + + line := buf.String() + if strings.Contains(line, cdUID) { + t.Fatalf("the sink kept the resolver uid: %s", line) + } + if !strings.Contains(line, "[redacted]") { + t.Fatalf("the sink wrote no redaction mark: %s", line) + } +} + +func Test_journalCoreDropsADebugLineWithAURL(t *testing.T) { + buf, logger := journalTestLogger(t) + + logger.Debug().Str("error", journalDoHError).Msg("upstream failed") + + if buf.Len() != 0 { + t.Fatalf("the sink kept a debug line: %s", buf.String()) + } +} + +func Test_journalCoreRedactsAMarkedInfoLine(t *testing.T) { + buf, logger := journalTestLogger(t) + + journal(logger.Info()).Str("error", journalDoHError).Msg("upstream failed") + + line := buf.String() + if !strings.Contains(line, journalRedactedURL) { + t.Fatalf("the sink kept the URL path: %s", line) + } + if !strings.Contains(line, `"level":"info"`) { + t.Fatalf("the sink changed the level: %s", line) + } +} + +// startInternalLogging drives the real cd mode setup and returns the program +// and the directory that holds the debug file and the journal file. +func startInternalLogging(t *testing.T) (*prog, string) { + t.Helper() + origSilent, origCdUID, origHomedir, origVerbose := silent, cdUID, homedir, verbose + origMainLog := mainLog.Load() + t.Cleanup(func() { + silent, cdUID, homedir, verbose = origSilent, origCdUID, origHomedir, origVerbose + mainLog.Store(origMainLog) + }) + if origMainLog == nil { + mainLog.Store(ctrld.NopLogger) + } + + dir := t.TempDir() + homedir = dir + cdUID = "test-uid" + silent = false + verbose = 0 + stubHeaderSnapshotSources(t) + + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.initInternalLogging(nil) + // Stop logs through the prog logger, as initLogging wires it. + p.logger.Store(mainLog.Load()) + t.Cleanup(p.closeInternalLogs) + return p, dir +} + +// Test_initInternalLogging_redactsTheJournalFile drives a cd mode run: support +// downloads the journal file, so it holds the redacted URL, while the debug +// file stays raw for the developer who reads it on the host. +func Test_initInternalLogging_redactsTheJournalFile(t *testing.T) { + _, dir := startInternalLogging(t) + + mainLog.Load().Warn().Str("error", journalDoHError).Msg("upstream failed") + + journalText := rotatingFileTestContent(t, filepath.Join(dir, journalLogFileName)) + if !strings.Contains(journalText, journalRedactedURL) { + t.Fatalf("journal file misses the redacted URL: %s", journalText) + } + if strings.Contains(journalText, "SECRET0123") { + t.Fatalf("journal file kept the resolver path: %s", journalText) + } + + debugText := rotatingFileTestContent(t, filepath.Join(dir, logFileName)) + if !strings.Contains(debugText, "org-v1-SECRET0123?dns=pOgBAAAB") { + t.Fatalf("debug file lost the raw URL: %s", debugText) + } +} + +// Test_stopClosesTheInternalLogFiles proves that a stopped service leaves no +// open handle on the debug file and the journal file. +func Test_stopClosesTheInternalLogFiles(t *testing.T) { + origPin := cdDeactivationPin.Load() + t.Cleanup(func() { cdDeactivationPin.Store(origPin) }) + cdDeactivationPin.Store(defaultDeactivationPin) + + p, _ := startInternalLogging(t) + p.stopCh = make(chan struct{}) + p.dnsWatcherStopCh = make(chan struct{}) + for _, writer := range []*logWriter{p.internalLogWriter, p.internalJournalWriter} { + if writer.rotating() == nil { + t.Fatal("the setup opened no file") + } + } + + if err := p.Stop(nil); err != nil { + t.Fatalf("Stop: %v", err) + } + + for name, writer := range map[string]*logWriter{ + logFileName: p.internalLogWriter, + journalLogFileName: p.internalJournalWriter, + } { + if writer.rotating() != nil { + t.Fatalf("%s stayed open after the stop", name) + } + } +} + +// Test_redactRetainedLineKeepsAShortClientID puts a two-letter client ID in +// the resolver UID. The keys of a JSON line hold the same letters, and a +// redaction of the client ID would break the line. +func Test_redactRetainedLineKeepsAShortClientID(t *testing.T) { + origCdUID := cdUID + t.Cleanup(func() { cdUID = origCdUID }) + cdUID = "uid12345678/os" + + line := string(redactRetainedLine([]byte(`{"level":"info","os":"darwin","resolver":"uid12345678/os"}`))) + + if !strings.Contains(line, `"os":"darwin"`) { + t.Fatalf("the redaction changed the os key: %s", line) + } + if strings.Contains(line, "uid12345678") { + t.Fatalf("the redaction kept the resolver uid: %s", line) + } +} + +// Test_redactRetainedLineStripsEverySchemePath covers the endpoints that carry +// a token in their path with a scheme other than https. +func Test_redactRetainedLineStripsEverySchemePath(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {`Get "quic://dns.example/org-v1-SECRET0123?dns=pOgBAAAB": timeout`, `Get "quic://dns.example/[redacted]": timeout`}, + {`h3://dns.example/SECRET0123 failed`, `h3://dns.example/[redacted] failed`}, + {`tls://dns.example:853 failed`, `tls://dns.example:853 failed`}, + {`tls://abcdef12.dns.controld.com:853 failed`, `tls://[redacted].dns.controld.com:853 failed`}, + {`quic://abcdef12.dns.controld.dev failed`, `quic://[redacted].dns.controld.dev failed`}, + {`https://freedns.controld.com/p1 failed`, `https://freedns.controld.com/[redacted] failed`}, + } { + if got := string(redactRetainedLine([]byte(tc.in))); got != tc.want { + t.Errorf("redactRetainedLine(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// Test_redactRetainedLineKeepsAShortResolverUID puts a two-letter resolver UID +// in place. The keys of a JSON line hold the same letters, and a redaction of +// the UID would break every retained line and the header. +func Test_redactRetainedLineKeepsAShortResolverUID(t *testing.T) { + origCdUID := cdUID + t.Cleanup(func() { cdUID = origCdUID }) + cdUID = "os" + + const in = `{"level":"info","os":"darwin","message":"loss of service"}` + if got := string(redactRetainedLine([]byte(in))); got != in { + t.Fatalf("redactRetainedLine(%q) = %q, want the line unchanged", in, got) + } +} diff --git a/cmd/cli/log_path.go b/cmd/cli/log_path.go new file mode 100644 index 0000000..b715645 --- /dev/null +++ b/cmd/cli/log_path.go @@ -0,0 +1,71 @@ +package cli + +import ( + "os" + "sync/atomic" +) + +// logPathFile is the open log_path file. The header renderer and the log +// readers need it, and both run outside this file. +var logPathFile atomic.Pointer[rotatingFile] + +// pendingLogPathRotation holds the rotation of a start-time backup. The +// logger reaches no file at that point, so initLogging reports it later. +var pendingLogPathRotation atomic.Pointer[logRotation] + +// openLogPathWriter opens the log_path file of this run and stores it for the +// header renderer and the log readers. It returns nil when no log_path is set. +// A start calls it more than once, so a call for the file that is already open +// keeps that file: a second open would put a second header in it. +func openLogPathWriter(doBackup bool) *rotatingFile { + logFilePath := normalizeLogFilePath(cfg.Service.LogPath) + if logFilePath == "" { + return nil + } + budget := debugLogBudget(&cfg.Service) + rf := logPathFile.Load() + if rf == nil || rf.currentPath() != logFilePath { + rf = newLogPathFile(logFilePath, budget) + } else { + rf.setBudget(budget) + } + if doBackup { + backupLogPathFile(rf) + } + return rf +} + +// newLogPathFile opens path and puts it in place of the file of an earlier +// call. A ctrld that cannot write the log file its operator asked for leaves +// no record of the run, so the open failure ends the process. +func newLogPathFile(path string, budget logBudget) *rotatingFile { + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + mainLog.Load().Error().Msgf("failed to create log file: %v", err) + os.Exit(1) + } + rf.onRotate = emitLogRotated + if old := logPathFile.Swap(rf); old != nil { + _ = old.close() + } + return rf +} + +// backupLogPathFile moves the file of an earlier run away, so no run +// overwrites another. +func backupLogPathFile(rf *rotatingFile) { + if !logFileHasContent(rf.currentPath()) { + return + } + record, err := rf.rotateNow() + if err != nil { + mainLog.Load().Error().Msgf("could not backup old log file: %v", err) + return + } + pendingLogPathRotation.Store(record) +} + +func logFileHasContent(path string) bool { + st, err := os.Stat(path) + return err == nil && st.Size() > 0 +} diff --git a/cmd/cli/log_path_rotation_test.go b/cmd/cli/log_path_rotation_test.go new file mode 100644 index 0000000..cb87632 --- /dev/null +++ b/cmd/cli/log_path_rotation_test.go @@ -0,0 +1,178 @@ +package cli + +import ( + "encoding/json" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +const ( + logPathTestBackups = 2 + logPathTestBudgetMB = 1 + logPathTestLineSize = 4 * 1024 + logPathTestLineCount = 300 +) + +// setupLogPathTest points log_path at a temporary file and puts every global +// that initLoggingWithBackup touches back when the test ends. The globals are +// the reason why no test of this package calls t.Parallel. +// +// The temporary directory comes first: its own cleanup removes the directory, +// and a cleanup runs before every cleanup that was added earlier, so the file +// must close before that. +func setupLogPathTest(t *testing.T) string { + t.Helper() + dir := t.TempDir() + origLogPath, origLogLevel := cfg.Service.LogPath, cfg.Service.LogLevel + origMaxSizeMB, origMaxBackups := cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups + origSilent, origVerbose := silent, verbose + origConsoleWriter := consoleWriter + origMainLog := mainLog.Load() + origLogPathFile, origPendingRotation := logPathFile.Load(), pendingLogPathRotation.Load() + t.Cleanup(func() { + if rf := logPathFile.Load(); rf != nil && rf != origLogPathFile { + _ = rf.close() + } + logPathFile.Store(origLogPathFile) + pendingLogPathRotation.Store(origPendingRotation) + cfg.Service.LogPath, cfg.Service.LogLevel = origLogPath, origLogLevel + cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups = origMaxSizeMB, origMaxBackups + silent, verbose = origSilent, origVerbose + consoleWriter = origConsoleWriter + mainLog.Store(origMainLog) + }) + + backups := logPathTestBackups + cfg.Service.LogPath = filepath.Join(dir, "ctrld.log") + cfg.Service.LogLevel = "info" + cfg.Service.LogMaxSizeMB = logPathTestBudgetMB + cfg.Service.LogMaxBackups = &backups + silent, verbose = false, 0 + consoleWriter = newHumanReadableZapCore(io.Discard, ctrld.NoticeLevel) + mainLog.Store(ctrld.NopLogger) + logPathFile.Store(nil) + pendingLogPathRotation.Store(nil) + return cfg.Service.LogPath +} + +func Test_logPathBackupChainOnStart(t *testing.T) { + path := setupLogPathTest(t) + if err := os.WriteFile(path, []byte("run1\n"), 0o600); err != nil { + t.Fatalf("seed log file: %v", err) + } + + initLoggingWithBackup(true) + if _, err := logPathFile.Load().Write([]byte("run2\n")); err != nil { + t.Fatalf("write run2: %v", err) + } + initLoggingWithBackup(true) + + newest := rotatingFileTestContent(t, path+".1") + if !strings.Contains(newest, "run2") { + t.Fatalf("%s.1 = %q, want the run2 line", path, newest) + } + if strings.Contains(newest, "run1") { + t.Fatalf("%s.1 = %q, want no run1 line", path, newest) + } + oldest := rotatingFileTestContent(t, path+".2") + if !strings.Contains(oldest, "run1") { + t.Fatalf("%s.2 = %q, want the run1 line", path, oldest) + } + current := rotatingFileTestContent(t, path) + if strings.Contains(current, "run1") || strings.Contains(current, "run2") { + t.Fatalf("%s = %q, want neither run line", path, current) + } +} + +func Test_logPathRotatesBySize(t *testing.T) { + path := setupLogPathTest(t) + initLoggingWithBackup(false) + + line := strings.Repeat("x", logPathTestLineSize) + for i := 0; i < logPathTestLineCount; i++ { + mainLog.Load().Info().Msg(line) + } + + if _, err := os.Stat(path + ".2"); !os.IsNotExist(err) { + t.Fatalf("%s.2 exists although the lines fill one backup (stat err = %v)", path, err) + } + want := []string{path + ".1", path} + if got := logPathFile.Load().paths(); !slices.Equal(got, want) { + t.Fatalf("paths() = %v, want %v", got, want) + } + + rotatedLines := 0 + for _, logLine := range strings.Split(rotatingFileTestContent(t, path), "\n") { + if !strings.Contains(logLine, "Log rotated") { + continue + } + rotatedLines++ + if !hasJournalMarker(logLine) { + t.Fatalf("Log rotated line %q carries no journal marker", logLine) + } + } + if rotatedLines != 1 { + t.Fatalf("Log rotated lines in %s = %d, want 1", path, rotatedLines) + } +} + +func Test_logPathHeaderLeadsEveryFile(t *testing.T) { + path := setupLogPathTest(t) + stubLogHeaderNetworkRead(t, "en0") + p := &prog{cfg: &cfg} + + // A start wires its logging twice: once for the config it reads from + // disk, and once more for the config that the API sends. + p.initLogging(true) + p.initLogging(false) + + line := strings.Repeat("x", logPathTestLineSize) + for i := 0; i < logPathTestLineCount; i++ { + mainLog.Load().Info().Msg(line) + } + + for _, name := range []string{path + ".1", path} { + firstLine, headers := logHeaderFileLines(t, name) + if firstLine.Message != logHeaderMessage { + t.Errorf("line 1 of %s = %q, want %q", name, firstLine.Message, logHeaderMessage) + } + if headers != 1 { + t.Errorf("%s holds %d header lines, want 1", name, headers) + } + } +} + +func Test_logPathStartBackupLogsRotation(t *testing.T) { + path := setupLogPathTest(t) + stubLogHeaderNetworkRead(t, "en0") + if err := os.WriteFile(path, []byte(`{"level":"info","message":"run1"}`+"\n"), 0o600); err != nil { + t.Fatalf("seed log file: %v", err) + } + p := &prog{cfg: &cfg} + + p.initLogging(true) + + rotatedLines := 0 + for _, logLine := range strings.Split(rotatingFileTestContent(t, path), "\n") { + if !strings.Contains(logLine, "Log rotated") { + continue + } + rotatedLines++ + var file string + if err := json.Unmarshal(parseLogLineFields(t, logLine)["file"], &file); err != nil { + t.Fatalf("parse the file field of %q: %v", logLine, err) + } + if file != path+".1" { + t.Errorf("file = %q, want %q", file, path+".1") + } + } + if rotatedLines != 1 { + t.Fatalf("Log rotated lines in %s = %d, want 1", path, rotatedLines) + } +} diff --git a/cmd/cli/log_reader.go b/cmd/cli/log_reader.go new file mode 100644 index 0000000..912b65d --- /dev/null +++ b/cmd/cli/log_reader.go @@ -0,0 +1,366 @@ +package cli + +import ( + "bytes" + "errors" + "io" + "os" + "strings" +) + +// logSendDebugBudget bounds the debug bytes of one upload. The journal is +// always whole, so an upload stays small and keeps the retained stream. A test +// lowers it, because a few small files must cross it. +var logSendDebugBudget int64 = 10 << 20 + +// lineScanBufferSize bounds the memory that the search for a line boundary +// needs, whatever the length of the line. +const lineScanBufferSize = 32 * 1024 + +var ( + errInternalLogEmpty = errors.New("internal log is empty") + errLogFileEmpty = errors.New("log file is empty") +) + +type logReader struct { + r io.ReadCloser + size int64 +} + +// logFilePart is the part of one open log file that an upload carries. An +// offset of 0 means the whole file. size is the size at the open, so a write +// after the selection stays out of the upload and the budget holds. +type logFilePart struct { + path string + f *os.File + offset int64 + size int64 +} + +// logReader composes the upload of the current logging mode and leads it with +// the header of this request. full lifts the bound on the debug bytes. +// stripColor removes the color codes of the internal debug stream, which an +// upload must not carry. The size counts the bytes before that removal. +func (p *prog) logReader(full, stripColor bool) (*logReader, error) { + lr, err := p.modeLogReader(full) + if err != nil { + return nil, err + } + // An empty upload must stay empty, because the handlers answer 301 on a + // size of 0. + if lr.size == 0 { + return lr, nil + } + header := p.sendLogHeader() + var body io.Reader = io.MultiReader(bytes.NewReader(header), lr.r) + if stripColor { + body = newANSIStripReader(body) + } + return &logReader{ + r: &multiCloser{Reader: body, closers: []io.Closer{lr.r}}, + size: lr.size + int64(len(header)), + }, nil +} + +// ansiStripChunkSize is the size of one read from the source of an +// ansiStripReader. +const ansiStripChunkSize = 32 * 1024 + +// ansiStripReader removes the ANSI color codes from what r yields. The level +// encoder of the internal debug stream colors each level, and an upload must +// hold plain text. +type ansiStripReader struct { + r io.Reader + // pending holds the bytes that are read and not yet stripped. It ends in + // an escape sequence that the next chunk completes, or it is empty. + pending []byte + // out holds the stripped bytes that no Read took yet. + out []byte + err error +} + +func newANSIStripReader(r io.Reader) io.Reader { + return &ansiStripReader{r: r} +} + +func (s *ansiStripReader) Read(p []byte) (int, error) { + for len(s.out) == 0 { + if s.err != nil { + return 0, s.err + } + s.fill() + } + n := copy(p, s.out) + s.out = s.out[n:] + return n, nil +} + +// fill reads one chunk and strips the bytes that hold no unfinished escape +// sequence. An unfinished sequence waits for the next chunk. The end of the +// source flushes everything, because nothing can complete the sequence. +func (s *ansiStripReader) fill() { + chunk := make([]byte, ansiStripChunkSize) + n, err := s.r.Read(chunk) + s.pending = append(s.pending, chunk[:n]...) + keep := 0 + if err == nil { + keep = partialEscapeLen(s.pending) + } + ready := len(s.pending) - keep + s.out = ansiRegex.ReplaceAll(s.pending[:ready], nil) + s.pending = append([]byte(nil), s.pending[ready:]...) + s.err = err +} + +// partialEscapeLen returns the length of the escape sequence that starts in b +// and does not end in it, or 0 when b ends outside a sequence. +func partialEscapeLen(b []byte) int { + start := bytes.LastIndexByte(b, 0x1b) + if start < 0 { + return 0 + } + tail := b[start:] + if len(tail) == 1 { + return 1 + } + if tail[1] != '[' { + return 0 + } + for _, c := range tail[2:] { + if (c < '0' || c > '9') && c != ';' { + return 0 + } + } + return len(tail) +} + +// modeLogReader composes the body of the current logging mode. +func (p *prog) modeLogReader(full bool) (*logReader, error) { + if p.needInternalLogging() { + return p.internalLogReader(full) + } + return p.logPathReader(full) +} + +// internalLogReader puts the newest debug bytes first, then the marker, then +// the journal, so a support reader finds the retained stream after the marker. +func (p *prog) internalLogReader(full bool) (*logReader, error) { + lw, jlw := p.internalWriters() + if lw == nil { + return nil, errors.New("nil internal log writer") + } + if jlw == nil { + return nil, errors.New("nil internal journal writer") + } + + var upload logParts + debugBytes := upload.addStream(lw, logSendBudget(full)) + upload.addBytes([]byte(logWriterLogEndMarker)) + journalBytes := upload.addStream(jlw, 0) + // The marker alone is not a log, and the handlers answer 301 on an empty + // upload. + if debugBytes+journalBytes == 0 { + upload.close() + return nil, errInternalLogEmpty + } + return upload.reader(errInternalLogEmpty) +} + +// logPathReader composes the upload of the log_path stream. The journal needs +// internal logging, so this upload holds the debug files alone. A run that +// opened no log_path file has nothing to send. +func (p *prog) logPathReader(full bool) (*logReader, error) { + rf := logPathFile.Load() + if p.cfg.Service.LogPath == "" || rf == nil { + return emptyLogReader(), nil + } + var upload logParts + upload.addBoundedFiles(rf.paths(), logSendBudget(full)) + return upload.reader(errLogFileEmpty) +} + +// emptyLogReader yields no bytes, so the handlers answer 301 instead of +// sending an upload without a line. +func emptyLogReader() *logReader { + return &logReader{r: io.NopCloser(strings.NewReader(""))} +} + +// logSendBudget reports the debug bytes that one upload may carry. Zero means +// every file. +func logSendBudget(full bool) int64 { + if full { + return 0 + } + return logSendDebugBudget +} + +// boundedLogFiles opens the files, reads each size once, and picks the newest +// bytes that fit in budget. files come oldest first, and so do the parts. The +// oldest part starts at a line boundary, so the upload holds whole lines only. +// A budget of 0 or less takes every file. The files that fall out of the +// budget close here; the parts own the files they hold. +func boundedLogFiles(files []string, budget int64) []logFilePart { + parts := openLogParts(files) + remaining := budget + first := len(parts) + for first > 0 { + part := &parts[first-1] + if budget <= 0 || part.size <= remaining { + remaining -= part.size + first-- + continue + } + if offset, ok := lineStartAt(part.f, part.size-remaining); ok { + part.offset = offset + first-- + } + // Without a line boundary in the tail, the upload would start inside a + // line, so the file stays out. + break + } + for _, part := range parts[:first] { + part.f.Close() + } + return parts[first:] +} + +// openLogParts opens each file through the platform opener, whose share flags +// let a rotation move the file away while Windows reads it. A rotation can +// also move a file away between the listing and the open, so a file that does +// not open must not stop the rest. +func openLogParts(files []string) []logFilePart { + parts := make([]logFilePart, 0, len(files)) + for _, path := range files { + f, err := openLogFile(path, os.O_RDONLY) + if err != nil { + continue + } + st, err := f.Stat() + if err != nil { + f.Close() + continue + } + parts = append(parts, logFilePart{path: path, f: f, size: st.Size()}) + } + return parts +} + +// lineStartAt returns the offset where a whole line starts at or after from. +// A cut that already lands on a line start keeps the line that follows it. It +// reports false when no newline follows. +func lineStartAt(f *os.File, from int64) (int64, bool) { + if startsLine(f, from) { + return from, true + } + if _, err := f.Seek(from, io.SeekStart); err != nil { + return 0, false + } + buf := make([]byte, lineScanBufferSize) + offset := from + for { + n, readErr := f.Read(buf) + if index := bytes.IndexByte(buf[:n], '\n'); index >= 0 { + return offset + int64(index) + 1, true + } + offset += int64(n) + if readErr != nil { + return 0, false + } + } +} + +// startsLine reports whether from is the first byte of a line. +func startsLine(f *os.File, from int64) bool { + if from == 0 { + return true + } + var previous [1]byte + if _, err := f.ReadAt(previous[:], from-1); err != nil { + return false + } + return previous[0] == '\n' +} + +// logParts collects the readers of one upload and the files it must close. +type logParts struct { + readers []io.Reader + closers []io.Closer + size int64 +} + +func (lp *logParts) addBytes(b []byte) { + if len(b) == 0 { + return + } + lp.readers = append(lp.readers, bytes.NewReader(b)) + lp.size += int64(len(b)) +} + +// addFiles takes the open parts. Each reader ends at the size of the open, so +// the bytes that a writer adds later stay out of this upload. +func (lp *logParts) addFiles(parts []logFilePart) { + for _, part := range parts { + length := part.size - part.offset + if length <= 0 { + part.f.Close() + continue + } + lp.readers = append(lp.readers, io.NewSectionReader(part.f, part.offset, length)) + lp.closers = append(lp.closers, part.f) + lp.size += length + } +} + +// addBoundedFiles adds the newest bytes of files that fit in budget. A budget +// of 0 or less takes every file. +func (lp *logParts) addBoundedFiles(files []string, budget int64) { + lp.addFiles(boundedLogFiles(files, budget)) +} + +// addStream adds the bytes of one stream and reports how many it added. A +// stream that has a file takes its files, and a stream that keeps its lines in +// memory takes its buffer, so one failed file never hides the other stream. +func (lp *logParts) addStream(lw *logWriter, budget int64) int64 { + before := lp.size + if rf := lw.rotating(); rf != nil { + lp.addBoundedFiles(rf.paths(), budget) + return lp.size - before + } + lp.addBytes(lw.bufferedBytes()) + return lp.size - before +} + +// reader joins the parts. The size is the number of bytes that the reader +// yields, so the caller reports what it uploads. +func (lp *logParts) reader(empty error) (*logReader, error) { + if lp.size == 0 { + lp.close() + return nil, empty + } + return &logReader{ + r: &multiCloser{Reader: io.MultiReader(lp.readers...), closers: lp.closers}, + size: lp.size, + }, nil +} + +func (lp *logParts) close() { + for _, c := range lp.closers { + c.Close() + } +} + +// multiCloser wraps an io.Reader and closes multiple underlying closers. +type multiCloser struct { + io.Reader + closers []io.Closer +} + +func (mc *multiCloser) Close() error { + var firstErr error + for _, c := range mc.closers { + if err := c.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} diff --git a/cmd/cli/log_reader_ansi_test.go b/cmd/cli/log_reader_ansi_test.go new file mode 100644 index 0000000..5ed0edd --- /dev/null +++ b/cmd/cli/log_reader_ansi_test.go @@ -0,0 +1,69 @@ +package cli + +import ( + "io" + "strings" + "testing" +) + +// splitReader yields its text in the pieces that the test names, one piece +// for each Read, so a test puts a chunk boundary where it wants one. +type splitReader struct { + pieces []string +} + +func (r *splitReader) Read(p []byte) (int, error) { + if len(r.pieces) == 0 { + return 0, io.EOF + } + n := copy(p, r.pieces[0]) + if n < len(r.pieces[0]) { + r.pieces[0] = r.pieces[0][n:] + } else { + r.pieces = r.pieces[1:] + } + return n, nil +} + +func Test_ansiStripReader_stripsSequencesSplitAcrossReads(t *testing.T) { + const red, reset = "\x1b[31m", "\x1b[0m" + for _, tc := range []struct { + name string + pieces []string + want string + }{ + {"one read", []string{"a " + red + "b" + reset + " c"}, "a b c"}, + {"split after the escape byte", []string{"a \x1b", "[31mb" + reset}, "a b"}, + {"split after the bracket", []string{"a \x1b[", "31mb"}, "a b"}, + {"split inside the parameters", []string{"a \x1b[3", "1mb"}, "a b"}, + {"split before the final byte", []string{"a \x1b[31", "mb"}, "a b"}, + {"two sequences in a row across a read", []string{"a " + red + "\x1b[", "1mb"}, "a b"}, + {"sequence open at the end of the source", []string{"a \x1b[31"}, "a \x1b[31"}, + {"lone escape at the end of the source", []string{"a \x1b"}, "a \x1b"}, + {"escape that starts no color sequence", []string{"a \x1bXb"}, "a \x1bXb"}, + {"letters inside the parameters stay", []string{"a \x1b[inva", "lidm b"}, "a \x1b[invalidm b"}, + {"empty source", nil, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := io.ReadAll(newANSIStripReader(&splitReader{pieces: tc.pieces})) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != tc.want { + t.Fatalf("got %q, want %q", got, tc.want) + } + }) + } +} + +func Test_ansiStripReader_holdsALongTailAcrossTheChunkSize(t *testing.T) { + // The escape byte lands as the last byte of one chunk. + head := strings.Repeat("x", ansiStripChunkSize-1) + got, err := io.ReadAll(newANSIStripReader(&splitReader{pieces: []string{head + "\x1b", "[32my"}})) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != head+"y" { + t.Fatalf("got %d bytes ending in %q, want the text without the color", len(got), got[len(got)-4:]) + } +} diff --git a/cmd/cli/log_reader_test.go b/cmd/cli/log_reader_test.go new file mode 100644 index 0000000..cec45a3 --- /dev/null +++ b/cmd/cli/log_reader_test.go @@ -0,0 +1,571 @@ +package cli + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +const ( + boundedLogTestLineSize = 100 + boundedLogTestLineCount = 30 + boundedLogTestFileCount = 5 + boundedLogTestBudget = 10000 +) + +// boundedLogTestLine returns a line of a fixed size that names its file, so a +// test can tell one part from another. +func boundedLogTestLine(file, line int) []byte { + head := fmt.Sprintf("file=%d line=%d ", file, line) + return []byte(head + strings.Repeat("x", boundedLogTestLineSize-len(head)-1) + "\n") +} + +// writeBoundedLogTestFiles creates the log files of one stream, oldest first. +func writeBoundedLogTestFiles(t *testing.T) []string { + t.Helper() + dir := t.TempDir() + files := make([]string, 0, boundedLogTestFileCount) + for file := 0; file < boundedLogTestFileCount; file++ { + var content bytes.Buffer + for line := 0; line < boundedLogTestLineCount; line++ { + content.Write(boundedLogTestLine(file, line)) + } + path := filepath.Join(dir, fmt.Sprintf("ctrld.log.%d", boundedLogTestFileCount-file)) + if err := os.WriteFile(path, content.Bytes(), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } + files = append(files, path) + } + return files +} + +func logFilesContent(t *testing.T, files []string) []byte { + t.Helper() + var joined []byte + for _, path := range files { + joined = append(joined, []byte(rotatingFileTestContent(t, path))...) + } + return joined +} + +// boundedLogTestFileSize is the size of each file that the bounded reader +// tests select from. +const boundedLogTestFileSize = int64(boundedLogTestLineCount * boundedLogTestLineSize) + +// wantTailLine reports the first line that fits in budget, for a budget that +// ends inside the fourth newest file. A cut that lands on a line start keeps +// that line. +func wantTailLine(budget int64) int { + cut := boundedLogTestFileSize - (budget - 3*boundedLogTestFileSize) + line := cut / boundedLogTestLineSize + if cut%boundedLogTestLineSize != 0 { + line++ + } + return int(line) +} + +// assertTailPart checks that a part starts at a line boundary and that it +// starts with the expected whole line. +func assertTailPart(t *testing.T, part logFilePart, file, line int) { + t.Helper() + content := []byte(rotatingFileTestContent(t, part.path)) + if part.offset <= 0 || part.offset >= int64(len(content)) { + t.Fatalf("offset = %d, want a position inside %s", part.offset, part.path) + } + if content[part.offset-1] != '\n' { + t.Fatalf("offset %d in %s does not follow a newline", part.offset, part.path) + } + if want := boundedLogTestLine(file, line); !bytes.HasPrefix(content[part.offset:], want) { + t.Fatalf("tail of %s starts with %.24q, want %.24q", part.path, content[part.offset:], want) + } +} + +// closeLogParts closes the files of parts that no upload took. +func closeLogParts(parts []logFilePart) { + for _, part := range parts { + part.f.Close() + } +} + +// partsSize reports the bytes that the parts hold on disk. +func partsSize(t *testing.T, parts []logFilePart) int64 { + t.Helper() + var size int64 + for _, part := range parts { + content := rotatingFileTestContent(t, part.path) + size += int64(len(content)) - part.offset + } + return size +} + +// assertBoundedParts checks the shape that every bounded selection shares: +// the newest three files whole, oldest first, after the tail of the fourth. +func assertBoundedParts(t *testing.T, files []string, parts []logFilePart, tailLine int) { + t.Helper() + wantPaths := files[1:] + if len(parts) != len(wantPaths) { + t.Fatalf("parts = %d, want %d", len(parts), len(wantPaths)) + } + for i, part := range parts { + if part.path != wantPaths[i] { + t.Fatalf("parts[%d].path = %s, want %s", i, part.path, wantPaths[i]) + } + } + assertTailPart(t, parts[0], 1, tailLine) + for i, part := range parts[1:] { + if part.offset != 0 { + t.Fatalf("parts[%d].offset = %d, want the whole file", i+1, part.offset) + } + } +} + +func Test_boundedLogFiles(t *testing.T) { + files := writeBoundedLogTestFiles(t) + + t.Run("budget ends at a line boundary", func(t *testing.T) { + parts := boundedLogFiles(files, boundedLogTestBudget) + t.Cleanup(func() { closeLogParts(parts) }) + + assertBoundedParts(t, files, parts, wantTailLine(boundedLogTestBudget)) + // The cut lands on a line start, so the tail begins at the cut and the + // parts fill the budget exactly. + if size := partsSize(t, parts); size != boundedLogTestBudget { + t.Fatalf("size = %d, want the whole budget of %d", size, boundedLogTestBudget) + } + }) + + t.Run("budget ends inside a line", func(t *testing.T) { + const budget = boundedLogTestBudget + boundedLogTestLineSize/2 + parts := boundedLogFiles(files, budget) + t.Cleanup(func() { closeLogParts(parts) }) + + assertBoundedParts(t, files, parts, wantTailLine(budget)) + if size := partsSize(t, parts); size > budget { + t.Fatalf("size = %d, want at most %d", size, budget) + } + }) + + t.Run("no budget keeps every file", func(t *testing.T) { + parts := boundedLogFiles(files, 0) + t.Cleanup(func() { closeLogParts(parts) }) + + if len(parts) != len(files) { + t.Fatalf("parts = %d, want %d", len(parts), len(files)) + } + for i, part := range parts { + if part.path != files[i] || part.offset != 0 { + t.Fatalf("parts[%d] = %+v, want %s whole", i, part, files[i]) + } + } + want := int64(boundedLogTestFileCount) * boundedLogTestFileSize + if size := partsSize(t, parts); size != want { + t.Fatalf("size = %d, want %d", size, want) + } + }) +} + +// Test_logReaderDoesNotFollowASymlink covers the one open of the reader. A +// symlink in place of a log file points at a file of another owner, and the +// writer refuses it in the same way. +func Test_logReaderDoesNotFollowASymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the Windows opener takes no symlink flag") + } + dir := t.TempDir() + planted := filepath.Join(dir, "planted.log") + if err := os.WriteFile(planted, []byte("planted line\n"), 0600); err != nil { + t.Fatalf("write the planted file: %v", err) + } + path := filepath.Join(dir, "ctrld.log") + if err := os.Symlink(planted, path); err != nil { + t.Fatalf("symlink: %v", err) + } + + parts := boundedLogFiles([]string{path}, 4) + if len(parts) != 0 { + t.Fatalf("the selection opened a symlink: %+v", parts) + } + var upload logParts + upload.addFiles(parts) + if upload.size != 0 { + t.Fatalf("the upload holds %d bytes of a symlink, want none", upload.size) + } +} + +// internalLogReaderTestBudget keeps the files small: four lines fill one file, +// so a few writes build the whole backup chain. +var internalLogReaderTestBudget = logBudget{maxSize: 4 * rotatingFileTestLineSize, backups: 2} + +const internalLogReaderTestLines = 12 + +// newTestStreamWriter opens one persisted stream in the temporary home +// directory. +func newTestStreamWriter(t *testing.T, name string) *logWriter { + t.Helper() + lw := newLogWriterWithSize(logWriterSize) + if err := lw.setLogFile(absHomeDir(name), internalLogReaderTestBudget); err != nil { + t.Fatalf("setLogFile %s: %v", name, err) + } + t.Cleanup(lw.closeLogFile) + return lw +} + +// internalLogReaderTestSendBudget holds one debug file and a half, so the +// default upload cuts inside the oldest of the three files. +var internalLogReaderTestSendBudget = internalLogReaderTestBudget.maxSize * 3 / 2 + +// useTestSendBudget lowers the bound on the debug bytes of one upload, so a +// few small files cross it. +func useTestSendBudget(t *testing.T, budget int64) { + t.Helper() + origBudget := logSendDebugBudget + t.Cleanup(func() { logSendDebugBudget = origBudget }) + logSendDebugBudget = budget +} + +// setupInternalLogReaderTest returns a cd mode prog whose debug stream and +// journal stream both hold a full backup chain on disk. +func setupInternalLogReaderTest(t *testing.T) *prog { + t.Helper() + origSilent, origCdUID, origHomedir := silent, cdUID, homedir + t.Cleanup(func() { silent, cdUID, homedir = origSilent, origCdUID, origHomedir }) + captureMainLog(t) + stubLogHeaderNetworkRead(t, "en9") + silent, cdUID, homedir = false, "test-uid", t.TempDir() + + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.internalLogWriter = newTestStreamWriter(t, logFileName) + p.internalJournalWriter = newTestStreamWriter(t, journalLogFileName) + for seq := 1; seq <= internalLogReaderTestLines; seq++ { + if _, err := p.internalLogWriter.Write(rotatingFileTestLine(1, seq)); err != nil { + t.Fatalf("write debug line %d: %v", seq, err) + } + if _, err := p.internalJournalWriter.Write(rotatingFileTestLine(2, seq)); err != nil { + t.Fatalf("write journal line %d: %v", seq, err) + } + } + return p +} + +// readLogReader reads one upload and checks that the reported size is the +// number of bytes it yields. +func readLogReader(t *testing.T, p *prog, full bool) []byte { + t.Helper() + lr, err := p.logReader(full, false) + if err != nil { + t.Fatalf("logReader(%v): %v", full, err) + } + defer lr.r.Close() + data, err := io.ReadAll(lr.r) + if err != nil { + t.Fatalf("read upload: %v", err) + } + if lr.size != int64(len(data)) { + t.Fatalf("size = %d, want %d", lr.size, len(data)) + } + return data +} + +// splitSendHeader takes the header line off the front of an upload and checks +// that it names this request. It returns the rest of the upload. +func splitSendHeader(t *testing.T, upload []byte, since time.Time) []byte { + t.Helper() + end := bytes.IndexByte(upload, '\n') + if end < 0 { + t.Fatalf("upload holds no header line: %.64q", upload) + } + parsed, _ := parseLogHeader(t, upload[:end+1]) + if parsed.Message != logHeaderMessage { + t.Fatalf("line 1 of the upload = %q, want %q", parsed.Message, logHeaderMessage) + } + if parsed.Trigger != "send" { + t.Fatalf("trigger = %q, want %q", parsed.Trigger, "send") + } + rendered, err := time.Parse(time.RFC3339, parsed.Time) + if err != nil { + t.Fatalf("parse header time %q: %v", parsed.Time, err) + } + // The header format drops the fraction of a second, so the window starts + // at the second that holds since. + if rendered.Before(since.Truncate(time.Second)) || rendered.After(time.Now().Add(time.Second)) { + t.Fatalf("header time = %s, want a time between %s and now", rendered, since) + } + return upload[end+1:] +} + +// splitUpload takes the header off an upload and cuts the rest at the log end +// marker, which parts the debug bytes from the journal bytes. +func splitUpload(t *testing.T, upload []byte, since time.Time) (debugPart, journalPart []byte) { + t.Helper() + body := splitSendHeader(t, upload, since) + debugPart, journalPart, found := bytes.Cut(body, []byte(logWriterLogEndMarker)) + if !found { + t.Fatalf("upload holds no log end marker: %.64q", body) + } + return debugPart, journalPart +} + +func Test_logRequestPath(t *testing.T) { + if got := logRequestPath(sendLogsPath, false); got != sendLogsPath { + t.Fatalf("logRequestPath(%q, false) = %q, want the path itself", sendLogsPath, got) + } + want := viewLogsPath + "?full=1" + if got := logRequestPath(viewLogsPath, true); got != want { + t.Fatalf("logRequestPath(%q, true) = %q, want %q", viewLogsPath, got, want) + } +} + +func Test_logReader_internalComposition(t *testing.T) { + p := setupInternalLogReaderTest(t) + useTestSendBudget(t, internalLogReaderTestSendBudget) + + debugFiles := p.internalLogWriter.rotating().paths() + journalFiles := p.internalJournalWriter.rotating().paths() + wantFiles := internalLogReaderTestBudget.backups + 1 + if len(debugFiles) != wantFiles || len(journalFiles) != wantFiles { + t.Fatalf("files: debug %v, journal %v, want %d of each", debugFiles, journalFiles, wantFiles) + } + wholeDebug := logFilesContent(t, debugFiles) + wholeJournal := logFilesContent(t, journalFiles) + + start := time.Now() + debugPart, journalPart := splitUpload(t, readLogReader(t, p, false), start) + + if int64(len(debugPart)) > logSendDebugBudget { + t.Fatalf("debug part = %d bytes, want at most the budget of %d", len(debugPart), logSendDebugBudget) + } + if len(debugPart) >= len(wholeDebug) { + t.Fatalf("debug part = %d bytes, want fewer than the %d bytes of %v", len(debugPart), len(wholeDebug), debugFiles) + } + if !bytes.HasSuffix(wholeDebug, debugPart) { + t.Fatalf("debug part = %.24q, want the newest %d bytes of %v", debugPart, len(debugPart), debugFiles) + } + cut := len(wholeDebug) - len(debugPart) + if wholeDebug[cut-1] != '\n' { + t.Fatalf("the debug part starts at %d, which does not follow a newline", cut) + } + if !bytes.Equal(journalPart, wholeJournal) { + t.Fatalf("journal part = %d bytes, want the %d bytes of %v", len(journalPart), len(wholeJournal), journalFiles) + } + + // The full upload lifts the bound, so it holds every debug file. + fullDebug, fullJournal := splitUpload(t, readLogReader(t, p, true), start) + if !bytes.Equal(fullDebug, wholeDebug) { + t.Fatalf("full debug part = %d bytes, want the %d bytes of %v", len(fullDebug), len(wholeDebug), debugFiles) + } + if !bytes.Equal(fullJournal, wholeJournal) { + t.Fatalf("full journal part = %d bytes, want the %d bytes of %v", len(fullJournal), len(wholeJournal), journalFiles) + } +} + +func Test_logReader_logPathComposition(t *testing.T) { + path := setupLogPathTest(t) + stubLogHeaderNetworkRead(t, "en9") + initLoggingWithBackup(false) + + line := strings.Repeat("x", logPathTestLineSize) + for i := 0; i < logPathTestLineCount; i++ { + mainLog.Load().Info().Msg(line) + } + + files := logPathFile.Load().paths() + if want := []string{path + ".1", path}; !slices.Equal(files, want) { + t.Fatalf("paths() = %v, want %v", files, want) + } + + p := &prog{cfg: &cfg} + p.logger.Store(mainLog.Load()) + start := time.Now() + got := splitSendHeader(t, readLogReader(t, p, false), start) + if want := logFilesContent(t, files); !bytes.Equal(got, want) { + t.Fatalf("upload = %d bytes, want the %d bytes of %v", len(got), len(want), files) + } + if bytes.Contains(got, []byte(logWriterLogEndMarker)) { + t.Fatalf("log_path upload holds the log end marker") + } +} + +// setupMemoryLogReaderTest returns a cd mode prog whose streams keep their +// lines in memory, because no file is open. +func setupMemoryLogReaderTest(t *testing.T) *prog { + t.Helper() + origSilent, origCdUID := silent, cdUID + t.Cleanup(func() { silent, cdUID = origSilent, origCdUID }) + captureMainLog(t) + stubLogHeaderNetworkRead(t, "en9") + silent, cdUID = false, "test-uid" + + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.internalLogWriter = newLogWriterWithSize(logWriterSize) + p.internalJournalWriter = newLogWriterWithSize(logWriterSize) + return p +} + +func Test_logReader_memoryFallbackStartsWithHeader(t *testing.T) { + p := setupMemoryLogReaderTest(t) + debugLine := []byte("debug line\n") + journalLine := []byte("journal line\n") + if _, err := p.internalLogWriter.Write(debugLine); err != nil { + t.Fatalf("write the debug buffer: %v", err) + } + if _, err := p.internalJournalWriter.Write(journalLine); err != nil { + t.Fatalf("write the journal buffer: %v", err) + } + + start := time.Now() + got := splitSendHeader(t, readLogReader(t, p, false), start) + + var want []byte + want = append(want, debugLine...) + want = append(want, []byte(logWriterLogEndMarker)...) + want = append(want, journalLine...) + if !bytes.Equal(got, want) { + t.Fatalf("upload after the header = %q, want %q", got, want) + } +} + +// useTestHomeDir points the ctrld home directory at a temporary directory, so +// the internal log files of a test never reach the real one. +func useTestHomeDir(t *testing.T) { + t.Helper() + origHomedir := homedir + t.Cleanup(func() { homedir = origHomedir }) + homedir = t.TempDir() +} + +// newSeededJournalWriter writes the journal files of an earlier run and +// returns a new writer over them. Its memory buffer is empty, which is what a +// restart finds, so a rule that reads the buffer loses these lines. +func newSeededJournalWriter(t *testing.T, lines int) *logWriter { + t.Helper() + previous := newTestStreamWriter(t, journalLogFileName) + for seq := 1; seq <= lines; seq++ { + if _, err := previous.Write(rotatingFileTestLine(2, seq)); err != nil { + t.Fatalf("write journal line %d: %v", seq, err) + } + } + previous.closeLogFile() + return newTestStreamWriter(t, journalLogFileName) +} + +// mixedStreamJournalLines fill two journal files, because four lines fill one. +const mixedStreamJournalLines = 6 + +func Test_logReader_mixedStreams(t *testing.T) { + p := setupMemoryLogReaderTest(t) + useTestHomeDir(t) + p.internalJournalWriter = newSeededJournalWriter(t, mixedStreamJournalLines) + debugLine := []byte("debug line\n") + if _, err := p.internalLogWriter.Write(debugLine); err != nil { + t.Fatalf("write the debug buffer: %v", err) + } + journalFiles := p.internalJournalWriter.rotating().paths() + if len(journalFiles) != 2 { + t.Fatalf("journal files = %v, want two", journalFiles) + } + + start := time.Now() + got := splitSendHeader(t, readLogReader(t, p, false), start) + + var want []byte + want = append(want, debugLine...) + want = append(want, []byte(logWriterLogEndMarker)...) + want = append(want, logFilesContent(t, journalFiles)...) + if !bytes.Equal(got, want) { + t.Fatalf("upload after the header = %d bytes, want the %d bytes of the debug buffer, the marker and %v", len(got), len(want), journalFiles) + } +} + +func Test_logReader_emptyStaysEmpty(t *testing.T) { + t.Run("internal mode without a line", func(t *testing.T) { + p := setupMemoryLogReaderTest(t) + + lr, err := p.logReader(false, false) + + if !errors.Is(err, errInternalLogEmpty) { + t.Fatalf("logReader error = %v, want %v", err, errInternalLogEmpty) + } + if lr != nil { + t.Fatalf("logReader = %+v, want no reader", lr) + } + }) + + t.Run("internal mode with empty files", func(t *testing.T) { + p := setupMemoryLogReaderTest(t) + useTestHomeDir(t) + p.internalLogWriter = newTestStreamWriter(t, logFileName) + p.internalJournalWriter = newTestStreamWriter(t, journalLogFileName) + + lr, err := p.logReader(false, false) + + if !errors.Is(err, errInternalLogEmpty) { + t.Fatalf("logReader error = %v, want %v", err, errInternalLogEmpty) + } + if lr != nil { + t.Fatalf("logReader = %+v, want no reader", lr) + } + }) + + t.Run("no log_path to read", func(t *testing.T) { + origSilent, origCdUID := silent, cdUID + t.Cleanup(func() { silent, cdUID = origSilent, origCdUID }) + stubLogHeaderNetworkRead(t, "en9") + silent, cdUID = false, "" + + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + lr, err := p.logReader(false, false) + if err != nil { + t.Fatalf("logReader: %v", err) + } + defer lr.r.Close() + + if lr.size != 0 { + t.Fatalf("size = %d, want 0 so the handler answers 301", lr.size) + } + data, err := io.ReadAll(lr.r) + if err != nil { + t.Fatalf("read the upload: %v", err) + } + if len(data) != 0 { + t.Fatalf("upload = %q, want no bytes", data) + } + }) +} + +// Test_boundedLogFilesFixTheSizeAtSelection appends through the real writer +// between the selection and the reader creation. The upload must hold the +// bytes that the selection counted, so the budget holds under a live writer. +func Test_boundedLogFilesFixTheSizeAtSelection(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 1 << 20, backups: 1}) + if _, err := rf.Write([]byte("one\n")); err != nil { + t.Fatal(err) + } + parts := boundedLogFiles(rf.paths(), 4) + if _, err := rf.Write([]byte("two\n")); err != nil { + t.Fatal(err) + } + + var upload logParts + upload.addFiles(parts) + t.Cleanup(upload.close) + got, err := io.ReadAll(io.MultiReader(upload.readers...)) + if err != nil { + t.Fatal(err) + } + if upload.size != 4 || string(got) != "one\n" { + t.Fatalf("upload = %q (%d bytes), want %q within the budget of 4", got, upload.size, "one\n") + } +} diff --git a/cmd/cli/log_setup_test.go b/cmd/cli/log_setup_test.go new file mode 100644 index 0000000..a372a09 --- /dev/null +++ b/cmd/cli/log_setup_test.go @@ -0,0 +1,173 @@ +package cli + +import ( + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +// setupLogSetupTest puts ctrld in cd mode with no log_path and a temporary +// home directory, and puts back every global that the logging setup touches. +// The globals are the reason why no test of this package calls t.Parallel. +func setupLogSetupTest(t *testing.T) string { + t.Helper() + dir := t.TempDir() + origLogPath, origLogLevel := cfg.Service.LogPath, cfg.Service.LogLevel + origSilent, origVerbose, origCdUID, origHomedir := silent, verbose, cdUID, homedir + origConsoleWriter := consoleWriter + origMainLog := mainLog.Load() + origLogPathFile, origPendingRotation := logPathFile.Load(), pendingLogPathRotation.Load() + t.Cleanup(func() { + if rf := logPathFile.Load(); rf != nil && rf != origLogPathFile { + _ = rf.close() + } + logPathFile.Store(origLogPathFile) + pendingLogPathRotation.Store(origPendingRotation) + cfg.Service.LogPath, cfg.Service.LogLevel = origLogPath, origLogLevel + silent, verbose, cdUID, homedir = origSilent, origVerbose, origCdUID, origHomedir + consoleWriter = origConsoleWriter + mainLog.Store(origMainLog) + }) + cfg.Service.LogPath, cfg.Service.LogLevel = "", "info" + silent, verbose, cdUID, homedir = false, 0, "", dir + consoleWriter = newHumanReadableZapCore(io.Discard, ctrld.NoticeLevel) + mainLog.Store(ctrld.NopLogger) + logPathFile.Store(nil) + pendingLogPathRotation.Store(nil) + stubHeaderSnapshotSources(t) + stubLogHeaderNetworkRead(t, "en0") + return dir +} + +// Test_initLoggingAfterProvisioning_opensTheJournal drives a --cd-org start: +// the first logging setup runs before the resolver UID is known, and the +// provisioning step then opens the internal streams. +func Test_initLoggingAfterProvisioning_opensTheJournal(t *testing.T) { + dir := setupLogSetupTest(t) + p := &prog{cfg: &cfg} + p.logger.Store(mainLog.Load()) + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + + journalPath := filepath.Join(dir, journalLogFileName) + if _, err := os.Stat(journalPath); !os.IsNotExist(err) { + t.Fatalf("journal exists before the UID is known (stat err = %v)", err) + } + + cdUID = "provisioned-uid" + p.initLoggingAfterProvisioning() + + firstLine, headers := logHeaderFileLines(t, journalPath) + if firstLine.Message != logHeaderMessage || headers != 1 { + t.Fatalf("journal starts with %q and holds %d headers, want the header once", firstLine.Message, headers) + } + mainLog.Load().Warn().Msg("retained after provisioning") + if content := rotatingFileTestContent(t, journalPath); !strings.Contains(content, "retained after provisioning") { + t.Fatalf("journal misses the line after provisioning: %s", content) + } + + p.initLoggingAfterProvisioning() + if _, headers := logHeaderFileLines(t, journalPath); headers != 1 { + t.Fatalf("a second call added a header: %d headers", headers) + } +} + +// Test_switchLogPath_leadsTheNewFileAndClosesTheInternalFiles drives the +// log_path that the API config sets after the first logging setup. +func Test_switchLogPath_leadsTheNewFileAndClosesTheInternalFiles(t *testing.T) { + dir := setupLogSetupTest(t) + cdUID = "test-uid" + p := &prog{cfg: &cfg} + p.logger.Store(mainLog.Load()) + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + mainLog.Load().Info().Msg("line before the switch") + + newPath := filepath.Join(dir, "logs", "ctrld.log") + cfg.Service.LogPath = newPath + p.switchLogPath("", newPath) + + firstLine, headers := logHeaderFileLines(t, newPath) + if firstLine.Message != logHeaderMessage || headers != 1 { + t.Fatalf("log_path file starts with %q and holds %d headers, want the header once", firstLine.Message, headers) + } + lw, jlw := p.internalWriters() + if lw.rotating() != nil || jlw.rotating() != nil { + t.Fatal("the internal files stayed open after the switch") + } + if files := p.logHeaderFiles(); !slices.Equal(files, []string{newPath}) { + t.Fatalf("log_files = %v, want the log_path only", files) + } + // cd mode has no local log_path, so the lines of the run stay in the + // internal debug stream that this call closes. + if content := rotatingFileTestContent(t, newPath); strings.Contains(content, "line before the switch") { + t.Fatalf("a line moved although the run had no local log_path: %s", content) + } +} + +// Test_switchLogPath_carriesTheOldFileBelowOneHeader drives a run that already +// wrote a local log_path file. The new file must hold one header, and the +// header must lead the lines that the old file holds. +func Test_switchLogPath_carriesTheOldFileBelowOneHeader(t *testing.T) { + dir := setupLogSetupTest(t) + cdUID = "test-uid" + oldPath := filepath.Join(dir, "old", "ctrld.log") + cfg.Service.LogPath = oldPath + p := &prog{cfg: &cfg} + p.logger.Store(mainLog.Load()) + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + mainLog.Load().Info().Msg("line before the switch") + + newPath := filepath.Join(dir, "logs", "ctrld.log") + cfg.Service.LogPath = newPath + p.switchLogPath(oldPath, newPath) + + firstLine, headers := logHeaderFileLines(t, newPath) + if firstLine.Message != logHeaderMessage || headers != 1 { + t.Fatalf("log_path file starts with %q and holds %d headers, want the header once", firstLine.Message, headers) + } + if !slices.Equal(firstLine.LogFiles, []string{newPath}) { + t.Fatalf("header log_files = %v, want the new log_path", firstLine.LogFiles) + } + if content := rotatingFileTestContent(t, newPath); !strings.Contains(content, "line before the switch") { + t.Fatalf("the new file misses the lines of the old file: %s", content) + } +} + +// Test_initLoggingAfterProvisioning_skipsTheGuardedRuns covers the two runs +// that keep no internal files: a run with a local log_path, and a silent run. +// Neither may gain a journal when the provisioning step learns the UID. +func Test_initLoggingAfterProvisioning_skipsTheGuardedRuns(t *testing.T) { + for _, tc := range []struct { + name string + guard func(t *testing.T, dir string) + }{ + {"log_path set", func(t *testing.T, dir string) { + cfg.Service.LogPath = filepath.Join(dir, "operator.log") + }}, + {"silent set", func(t *testing.T, _ string) { silent = true }}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := setupLogSetupTest(t) + tc.guard(t, dir) + p := &prog{cfg: &cfg} + p.logger.Store(mainLog.Load()) + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + + cdUID = "provisioned-uid" + p.initLoggingAfterProvisioning() + + journalPath := filepath.Join(dir, journalLogFileName) + if _, err := os.Stat(journalPath); !os.IsNotExist(err) { + t.Fatalf("the guarded run opened a journal (stat err = %v)", err) + } + }) + } +} diff --git a/cmd/cli/log_tail_follow.go b/cmd/cli/log_tail_follow.go new file mode 100644 index 0000000..045b4e1 --- /dev/null +++ b/cmd/cli/log_tail_follow.go @@ -0,0 +1,80 @@ +package cli + +import ( + "context" + "io" + "net/http" + "os" + "time" +) + +// logTailPollInterval is the time between two reads of a tailed log file. +const logTailPollInterval = 200 * time.Millisecond + +// followLogFile streams the lines of path until ctx ends. It first sends the +// last numLines lines, or starts at the end when numLines is 0. A rotation +// moves the file away or truncates it, so the poll reopens the path when the +// file behind it changes or shrinks. +func followLogFile(ctx context.Context, w io.Writer, flusher http.Flusher, path string, numLines int) { + f, err := os.Open(path) + if err != nil { + return + } + defer func() { f.Close() }() + + if numLines > 0 { + if tail := tailFileLastLines(f, numLines); len(tail) > 0 { + w.Write(tail) + flusher.Flush() + } + } else { + f.Seek(0, io.SeekEnd) + } + + buf := make([]byte, 4096) + ticker := time.NewTicker(logTailPollInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + n, err := f.Read(buf) + if n > 0 { + if _, werr := w.Write(buf[:n]); werr != nil { + return + } + flusher.Flush() + } + if err != nil && err != io.EOF { + return + } + if n == 0 && logFileReplaced(f, path) { + reopened, err := os.Open(path) + if err != nil { + continue + } + f.Close() + f = reopened + } + case <-ctx.Done(): + return + } + } +} + +// logFileReplaced reports whether path names another file than f, or a file +// shorter than the read position. Both mean that a rotation happened. +func logFileReplaced(f *os.File, path string) bool { + current, err := os.Stat(path) + if err != nil { + return false + } + open, err := f.Stat() + if err != nil { + return true + } + if !os.SameFile(current, open) { + return true + } + position, err := f.Seek(0, io.SeekCurrent) + return err == nil && current.Size() < position +} diff --git a/cmd/cli/log_test_helpers_test.go b/cmd/cli/log_test_helpers_test.go new file mode 100644 index 0000000..68c7ffb --- /dev/null +++ b/cmd/cli/log_test_helpers_test.go @@ -0,0 +1,67 @@ +package cli + +import ( + "encoding/json" + "io" + "strings" + "testing" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/Control-D-Inc/ctrld" +) + +// journalMarker is the serialized form of the journal field in a JSON line. +var journalMarker = []byte(`"` + journalField + `":true`) + +// hasJournalMarker reports whether a log line carries the journal field. A +// JSON line writes the field without a space, a console line with one. +func hasJournalMarker(line string) bool { + return strings.Contains(line, string(journalMarker)) || strings.Contains(line, `"`+journalField+`": true`) +} + +// newTestJSONLogger writes one JSON object for each line to every writer, at +// debug level, with the field names of the journal. A test parses the lines +// back, so the encoder must be the JSON one. +func newTestJSONLogger(writers ...io.Writer) *ctrld.Logger { + cores := make([]zapcore.Core, 0, len(writers)) + for _, w := range writers { + cores = append(cores, zapcore.NewCore(newJournalEncoder(), zapcore.AddSync(w), zapcore.DebugLevel)) + } + return &ctrld.Logger{Logger: zap.New(zapcore.NewTee(cores...))} +} + +// captureJSONMainLog swaps mainLog for a JSON logger that writes into a +// buffer at debug level and puts the old logger back when the test ends. The +// JSON form lets a test parse the lines it captured. It swaps a global, so +// no test of this package runs in parallel. +func captureJSONMainLog(t *testing.T) *syncBuffer { + t.Helper() + buf := &syncBuffer{} + old := mainLog.Load() + mainLog.Store(newTestJSONLogger(buf)) + t.Cleanup(func() { mainLog.Store(old) }) + return buf +} + +// parseLogLineFields returns the fields of one log line. A JSON line is one +// object. A console line of the debug stream ends with its fields as one +// object after the last tab. +func parseLogLineFields(t *testing.T, line string) map[string]json.RawMessage { + t.Helper() + line = strings.TrimSpace(line) + object := line + if !strings.HasPrefix(line, "{") { + index := strings.LastIndex(line, "\t{") + if index < 0 { + t.Fatalf("log line holds no fields: %q", line) + } + object = line[index+1:] + } + fields := map[string]json.RawMessage{} + if err := json.Unmarshal([]byte(object), &fields); err != nil { + t.Fatalf("parse fields of %q: %v", line, err) + } + return fields +} diff --git a/cmd/cli/log_writer.go b/cmd/cli/log_writer.go index 82cbdbb..819b3c2 100644 --- a/cmd/cli/log_writer.go +++ b/cmd/cli/log_writer.go @@ -2,11 +2,9 @@ package cli import ( "bytes" - "errors" - "fmt" + "encoding/json" "io" "os" - "path/filepath" "regexp" "strings" "sync" @@ -44,8 +42,7 @@ const ( // This provides clear boundaries for log parsing and analysis logWriterLogEndMarker = "\n\n=== LOG_END ===\n\n" - logFileName = "ctrld.log" - logFileMaxSize = 1024 * 1024 * 5 // 5 MB + logFileName = "ctrld.log" ) // Custom level encoders that handle NOTICE level @@ -89,42 +86,21 @@ type logSentResponse struct { Error string `json:"error"` } -// logReader provides read access to log data with size information. -// -// This struct encapsulates log reading functionality for external consumers, -// providing both the log content and metadata about the log size. It supports -// reading from both internal log buffers (when no external logging is configured) -// and external log files (when logging to file is enabled). -// -// Fields: -// - r: An io.ReadCloser that provides access to the log content -// - size: The total size of the log data in bytes -// -// The logReader is used by the control server to serve log content to clients -// and by various CLI commands that need to display or process log data. -type logReader struct { - r io.ReadCloser - size int64 -} - // logSubscriber represents a subscriber to live log output. type logSubscriber struct { ch chan []byte } // logWriter is an internal buffer to keep track of runtime log when no logging is enabled. -// When a file path is configured via setLogFile, writes are also persisted to -// a rotated file on disk (max logFileMaxSize, 1 backup) so logs survive restarts. +// When a file is configured via setLogFile, writes also go to that file, which +// rotates within its budget, so logs survive restarts. type logWriter struct { mu sync.Mutex buf bytes.Buffer size int subscribers []*logSubscriber - // File persistence fields. - logFile *os.File - logFilePath string - logFileSize int64 + file *rotatingFile } // newLogWriter creates an internal log writer. @@ -142,82 +118,50 @@ func newSmallLogWriter() *logWriter { // newLogWriterWithSize creates an internal log writer with a given buffer size. // This allows customization of log buffer size based on specific requirements func newLogWriterWithSize(size int) *logWriter { - lw := &logWriter{size: size} - return lw + return &logWriter{size: size} } -// setLogFile configures file-backed persistence for the log writer. -// The directory is created if it does not exist. An existing file is -// opened in append mode and its current size is tracked for rotation. -func (lw *logWriter) setLogFile(path string) error { - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0750); err != nil { - return fmt.Errorf("creating log directory: %w", err) - } - f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0600) +// setLogFile persists the stream to a file that rotates within the budget. +func (lw *logWriter) setLogFile(path string, budget logBudget) error { + rf, err := newRotatingFile(path, budget, nil) if err != nil { - return fmt.Errorf("opening log file: %w", err) - } - st, err := f.Stat() - if err != nil { - f.Close() - return fmt.Errorf("stat log file: %w", err) + return err } + rf.onRotate = emitLogRotated lw.mu.Lock() defer lw.mu.Unlock() - lw.logFile = f - lw.logFilePath = path - lw.logFileSize = st.Size() + lw.file = rf return nil } -// rotateLogFile rotates the current log file to a .1 backup. -// It returns true if lw.logFile is usable after the call, false otherwise. -// Must be called with lw.mu held. -func (lw *logWriter) rotateLogFile() bool { - if lw.logFile == nil { - return false - } - lw.logFile.Close() - backupPath := lw.logFilePath + ".1" - // Best effort: rename current to backup (overwrites old backup). - os.Rename(lw.logFilePath, backupPath) - f, err := os.OpenFile(lw.logFilePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) - if err != nil { - // If we can't reopen, disable file logging. - lw.logFile = nil - lw.logFileSize = 0 - return false - } - lw.logFile = f - lw.logFileSize = 0 - return true -} - // closeLogFile closes the backing file if open. func (lw *logWriter) closeLogFile() { lw.mu.Lock() defer lw.mu.Unlock() - if lw.logFile != nil { - lw.logFile.Close() - lw.logFile = nil + if lw.file == nil { + return } + lw.file.close() + lw.file = nil } -// logFilePaths returns the paths to the current log file and its backup -// (if they exist) for inclusion in log send payloads. -func (lw *logWriter) logFilePaths() (current, backup string) { +// rotating returns the backing file, so a caller works with the file itself +// instead of one pass-through method for each of its calls. A stream that +// keeps its lines in memory only has no file. +func (lw *logWriter) rotating() *rotatingFile { lw.mu.Lock() defer lw.mu.Unlock() - if lw.logFilePath == "" { - return "", "" - } - current = lw.logFilePath - bp := lw.logFilePath + ".1" - if _, err := os.Stat(bp); err == nil { - backup = bp - } - return current, backup + return lw.file +} + +// bufferedBytes copies the memory buffer, so a reader of the copy holds no +// writer lock while it works. +func (lw *logWriter) bufferedBytes() []byte { + lw.mu.Lock() + defer lw.mu.Unlock() + buffered := make([]byte, lw.buf.Len()) + copy(buffered, lw.buf.Bytes()) + return buffered } // Subscribe returns a channel that receives new log data as it's written, @@ -268,8 +212,20 @@ func (lw *logWriter) tailLastLines(n int) []byte { } // Write implements io.Writer interface for logWriter -// This manages buffer overflow by discarding old data while preserving important markers func (lw *logWriter) Write(p []byte) (int, error) { + // The file takes the line outside lw.mu: it holds its own lock, and it + // reports its rotation with an event that comes back through this writer. + // A file error must not stop the memory buffer, which is the fallback for + // the log readers. + if rf := lw.rotating(); rf != nil { + _, _ = rf.Write(p) + } + return lw.writeStreams(p) +} + +// writeStreams feeds the subscribers and the memory buffer. +// This manages buffer overflow by discarding old data while preserving important markers +func (lw *logWriter) writeStreams(p []byte) (int, error) { lw.mu.Lock() defer lw.mu.Unlock() @@ -286,16 +242,6 @@ func (lw *logWriter) Write(p []byte) (int, error) { } } - // Write to backing file if configured. - if lw.logFile != nil { - needsRotation := lw.logFileSize+int64(len(p)) > logFileMaxSize - if !needsRotation || lw.rotateLogFile() { - if n, err := lw.logFile.Write(p); err == nil { - lw.logFileSize += int64(n) - } - } - } - // If writing p causes overflows, discard old data. // This prevents unbounded memory growth while maintaining recent logs if lw.buf.Len()+len(p) > lw.size { @@ -326,6 +272,29 @@ func (lw *logWriter) Write(p []byte) (int, error) { return lw.buf.Write(p) } +// emitLogRotated logs a rotation. The file that rotated guards this call, +// because the event is a line that can rotate that file again. A rotation +// that could not move the file keeps the lines it holds, so it earns a +// warning instead of the normal report. +func emitLogRotated(r logRotation) { + event := journal(mainLog.Load().Info()) + message := "Log rotated" + if r.err != nil { + event = journal(mainLog.Load().Warn()).Err(r.err) + message = "Log rotation failed" + } + event = event.Str("file", r.file).Int64("bytes_written", r.bytesWritten) + // A file that holds a header only carries no event, and the year 1 of the + // zero time reads as a wrong date. + if !r.firstWrite.IsZero() { + event = event.Time("first_event_at", r.firstWrite) + } + if !r.lastWrite.IsZero() { + event = event.Time("last_event_at", r.lastWrite) + } + event.Int("backups", r.backups).Msg(message) +} + // initLogging initializes global logging setup. func (p *prog) initLogging(backup bool) { logCores := initLoggingWithBackup(backup) @@ -333,57 +302,202 @@ func (p *prog) initLogging(backup bool) { // Initializing internal logging after global logging. p.initInternalLogging(logCores) p.logger.Store(mainLog.Load()) + + if rf := logPathFile.Load(); rf != nil { + p.refreshLogHeader() + if err := rf.writeHeader(); err != nil { + p.Warn().Err(err).Msg("Could not write log header") + } + } + // A start-time backup happens before the logger reaches the new file, so + // its event waits here for a logger that can report it. + if record := pendingLogPathRotation.Swap(nil); record != nil { + emitLogRotated(*record) + } } -// internalLogFilePath returns the path for persisted internal logs. -// The file lives in the ctrld home directory alongside other runtime state. -func internalLogFilePath() string { - return ctrld.AbsHomeDir(logFileName) +// initLoggingAfterProvisioning opens the internal streams of a run whose +// resolver UID came from the provisioning token. The first logging setup ran +// before that UID was known, so it opened no debug file and no journal. +func (p *prog) initLoggingAfterProvisioning() { + if !p.needInternalLogging() { + return + } + if lw, _ := p.internalWriters(); lw != nil { + return + } + p.initLogging(false) +} + +// switchLogPath moves the logging of this run to the log_path that the API +// config set. The header of the new file leads it, the lines of an earlier +// local log_path file follow that header, and the internal streams close, +// because a run with a log_path keeps no internal files. A run without a local +// log_path carries no lines: its history stays in the internal debug stream, +// which this call closes. +func (p *prog) switchLogPath(oldLogPath, newLogPath string) { + // After processCDFlags, log config may change, so reset mainLog and re-init logging. + mainLog.Store(&ctrld.Logger{Logger: zap.NewNop()}) + + carried := carriedLogLines(oldLogPath, newLogPath) + p.initLogging(false) + p.carryLogLines(carried) +} + +// carriedLogLines reads the lines of the old log_path file and empties the new +// file, so the header of the new file leads it. It drops the header lines of +// the old file, because each of them names the old file. +func carriedLogLines(oldLogPath, newLogPath string) []byte { + buf, err := os.ReadFile(oldLogPath) + if err != nil { + return nil + } + if err := os.Remove(newLogPath); err != nil && !os.IsNotExist(err) { + mainLog.Load().Warn().Err(err).Msg("could not clear the new log file") + return nil + } + return withoutLogHeaderLines(buf) +} + +// carryLogLines appends the lines of the old log_path file below the header of +// the new one. +func (p *prog) carryLogLines(lines []byte) { + if len(lines) == 0 { + return + } + rf := logPathFile.Load() + if rf == nil { + return + } + if _, err := rf.Write(lines); err != nil { + p.Warn().Err(err).Msg("Could not copy old log file") + } +} + +// withoutLogHeaderLines removes every header line of a log file. +func withoutLogHeaderLines(buf []byte) []byte { + var kept bytes.Buffer + for _, line := range bytes.SplitAfter(buf, []byte("\n")) { + if isLogHeaderLine(line) { + continue + } + kept.Write(line) + } + return kept.Bytes() +} + +// isLogHeaderLine reports whether one line of a log file is a header line. +func isLogHeaderLine(line []byte) bool { + var parsed struct { + Message string `json:"message"` + } + if err := json.Unmarshal(bytes.TrimSpace(line), &parsed); err != nil { + return false + } + return parsed.Message == logHeaderMessage } // initInternalLogging performs internal logging if there's no log enabled. func (p *prog) initInternalLogging(externalCores []zapcore.Core) { if !p.needInternalLogging() { + // A log_path that the API set takes over from the internal files. + p.closeInternalLogs() return } + headersWritten := false p.initInternalLogWriterOnce.Do(func() { - p.Notice().Msg("Internal logging enabled") + // mainLog, not p.logger: a test drives this setup on a bare prog. + mainLog.Load().Notice().Msg("Internal logging enabled") p.internalLogWriter = newLogWriter() p.internalLogSent = time.Now().Add(-logWriterSentInterval) - p.internalWarnLogWriter = newSmallLogWriter() - // Persist internal logs to disk so they survive restarts. - if path := internalLogFilePath(); path != "" { - if err := p.internalLogWriter.setLogFile(path); err != nil { - mainLog.Load().Warn().Err(err).Msg("could not enable persistent internal logging") - } else { - mainLog.Load().Notice().Msgf("internal log file: %s", path) - } - } + p.internalJournalWriter = newSmallLogWriter() + p.openInternalLogFiles() + // A restart appends to the file it finds, so the header marks the + // point where this run starts. + p.refreshLogHeader() + p.writeLogHeaders() + headersWritten = true }) - p.mu.Lock() - lw := p.internalLogWriter - wlw := p.internalWarnLogWriter - p.mu.Unlock() + lw, jlw := p.internalWriters() // Create zap cores for different writers - var cores []zapcore.Core + cores := make([]zapcore.Core, 0, len(externalCores)+2) cores = append(cores, externalCores...) - // Add core for internal log writer. // Run the internal logging at debug level, so we could - // have enough information for troubleshooting. - internalCore := newHumanReadableZapCore(lw, zapcore.DebugLevel) - cores = append(cores, internalCore) - - // Add core for internal warn log writer - warnCore := newHumanReadableZapCore(wlw, zapcore.WarnLevel) - cores = append(cores, warnCore) + // have enough information for troubleshooting. The journal core keeps + // the lines that survive a restart. + cores = append(cores, newHumanReadableZapCore(lw, zapcore.DebugLevel), newJournalCore(jlw)) // Create a multi-core logger - multiCore := zapcore.NewTee(cores...) - logger := zap.New(multiCore) + mainLog.Store(&ctrld.Logger{Logger: zap.New(zapcore.NewTee(cores...))}) + if headersWritten { + return + } + // A later call reaches a changed config, so the stored bytes need the new + // values. The files keep the header they already hold. + p.refreshLogHeader() +} - mainLog.Store(&ctrld.Logger{Logger: logger}) +// openInternalLogFiles persists both internal streams, so they survive a +// restart. +func (p *prog) openInternalLogFiles() { + _, journalBudget := logBudgets() + openInternalLogFile(p.internalLogWriter, absHomeDir(logFileName), debugLogBudget(&p.cfg.Service), "internal") + openInternalLogFile(p.internalJournalWriter, absHomeDir(journalLogFileName), journalBudget, "journal") +} + +// openInternalLogFile persists one internal stream. A stream whose file cannot +// open keeps its lines in memory. Only these files get a prune: ctrld owns +// their names, while log_path can name a file in a directory that holds the +// files of other programs. +func openInternalLogFile(lw *logWriter, path string, budget logBudget, name string) { + pruneNumberedBackups(path, budget.backups) + if err := lw.setLogFile(path, budget); err != nil { + mainLog.Load().Warn().Err(err).Msgf("Could not enable persistent %s logging", name) + return + } + mainLog.Load().Notice().Msgf("%s log file: %s", name, path) +} + +// internalWriters returns the debug stream and the journal stream. Both stay +// nil until initInternalLogging opens them, and outside cd mode. +func (p *prog) internalWriters() (debugWriter, journalWriter *logWriter) { + p.mu.Lock() + defer p.mu.Unlock() + return p.internalLogWriter, p.internalJournalWriter +} + +// openLogFiles returns the files that this run writes, debug first. A stream +// that keeps its lines in memory only has no file, and log_path belongs to +// this list although no internal writer owns it. +func (p *prog) openLogFiles() []*rotatingFile { + debugWriter, journalWriter := p.internalWriters() + files := make([]*rotatingFile, 0, 3) + for _, writer := range []*logWriter{debugWriter, journalWriter} { + if writer == nil { + continue + } + if rf := writer.rotating(); rf != nil { + files = append(files, rf) + } + } + if rf := logPathFile.Load(); rf != nil { + files = append(files, rf) + } + return files +} + +// closeInternalLogs closes both internal files, so a stop leaves no open +// handle behind. +func (p *prog) closeInternalLogs() { + debugWriter, journalWriter := p.internalWriters() + for _, writer := range []*logWriter{debugWriter, journalWriter} { + if writer == nil { + continue + } + writer.closeLogFile() + } } // needInternalLogging reports whether prog needs to run internal logging. @@ -405,125 +519,6 @@ func (p *prog) needInternalLogging() bool { return true } -// logReaderNoColor returns a logReader with ANSI color codes stripped from the log content. -// -// This method is useful when log content needs to be processed by tools that don't -// handle ANSI escape sequences properly, or when storing logs in plain text format. -// It internally calls logReader(true) to strip color codes. -// -// Returns: -// - *logReader: A logReader instance with color codes removed, or nil if no logs available -// - error: Any error encountered during log reading (e.g., empty logs, file access issues) -// -// Use cases: -// - Log processing pipelines that require plain text -// - Storing logs in databases or text files -// - Displaying logs in environments that don't support color -func (p *prog) logReaderNoColor() (*logReader, error) { - return p.logReader(true) -} - -// logReaderRaw returns a logReader with ANSI color codes preserved in the log content. -// -// This method maintains the original formatting of log entries including color codes, -// which is useful for displaying logs in terminals that support ANSI colors or when -// the original visual formatting needs to be preserved. It internally calls logReader(false). -// -// Returns: -// - *logReader: A logReader instance with color codes preserved, or nil if no logs available -// - error: Any error encountered during log reading (e.g., empty logs, file access issues) -// -// Use cases: -// - Terminal-based log viewers that support color -// - Interactive debugging sessions -// - Preserving original log formatting for display -func (p *prog) logReaderRaw() (*logReader, error) { - return p.logReader(false) -} - -// logReader creates a logReader instance for accessing log content with optional color stripping. -// -// This is the core method that handles log reading from different sources based on the -// current logging configuration. It supports both internal logging (when no external -// logging is configured) and external file logging (when logging to file is enabled). -// -// Behavior: -// - Internal logging: Reads from internal log buffers (normal logs + warning logs) -// and combines them with appropriate markers for separation -// - External logging: Reads directly from the configured log file -// - Empty logs: Returns appropriate error messages when no log content is available -// -// Parameters: -// - stripColor: If true, removes ANSI color codes from log content; if false, preserves them -// -// Returns: -// - *logReader: A logReader instance providing access to log content and size metadata -// - error: Any error encountered during log reading, including: -// - "nil internal log writer" - Internal logging not properly initialized -// - "nil internal warn log writer" - Warning log writer not properly initialized -// - "internal log is empty" - No content in internal log buffers -// - "log file is empty" - External log file exists but contains no data -// - File system errors when accessing external log files -// -// The method handles thread-safe access to internal log buffers and provides -// comprehensive error handling for various edge cases. -func (p *prog) logReader(stripColor bool) (*logReader, error) { - if p.needInternalLogging() { - p.mu.Lock() - lw := p.internalLogWriter - wlw := p.internalWarnLogWriter - p.mu.Unlock() - if lw == nil { - return nil, errors.New("nil internal log writer") - } - if wlw == nil { - return nil, errors.New("nil internal warn log writer") - } - - // If we have a persisted log file, read from disk (includes data - // from previous runs that the in-memory buffer wouldn't have). - current, backup := lw.logFilePaths() - if current != "" { - return p.logReaderFromFiles(current, backup, wlw) - } - - // Fall back to in-memory buffer. - lw.mu.Lock() - lwReader := newLogReader(&lw.buf, stripColor) - lwSize := lw.buf.Len() - lw.mu.Unlock() - // Warn log content. - wlw.mu.Lock() - wlwReader := newLogReader(&wlw.buf, stripColor) - wlwSize := wlw.buf.Len() - wlw.mu.Unlock() - reader := io.MultiReader(lwReader, bytes.NewReader([]byte(logWriterLogEndMarker)), wlwReader) - lr := &logReader{r: io.NopCloser(reader)} - lr.size = int64(lwSize + wlwSize) - if lr.size == 0 { - return nil, errors.New("internal log is empty") - } - return lr, nil - } - if p.cfg.Service.LogPath == "" { - return &logReader{r: io.NopCloser(strings.NewReader(""))}, nil - } - f, err := os.Open(normalizeLogFilePath(p.cfg.Service.LogPath)) - if err != nil { - return nil, err - } - lr := &logReader{r: f} - if st, err := f.Stat(); err == nil { - lr.size = st.Size() - } else { - return nil, fmt.Errorf("f.Stat: %w", err) - } - if lr.size == 0 { - return nil, errors.New("log file is empty") - } - return lr, nil -} - // newHumanReadableZapCore creates a zap core optimized for human-readable log output. // // Features: @@ -588,87 +583,7 @@ var ansiRegex = regexp.MustCompile(`\x1b\[[0-9;]*m`) // Returns an io.Reader that provides access to the processed log content. func newLogReader(buf *bytes.Buffer, stripColor bool) io.Reader { if stripColor { - return strings.NewReader(ansiRegex.ReplaceAllString(buf.String(), "")) + return newANSIStripReader(strings.NewReader(buf.String())) } return strings.NewReader(buf.String()) } - -// logReaderFromFiles builds a logReader that concatenates the backup file -// (if it exists), the current log file, and the in-memory warn log buffer. -func (p *prog) logReaderFromFiles(current, backup string, wlw *logWriter) (*logReader, error) { - var rcs []io.ReadCloser - var totalSize int64 - - closeAll := func() { - for _, rc := range rcs { - rc.Close() - } - } - - // Read backup file first (older entries). - if backup != "" { - if bf, err := os.Open(backup); err == nil { - if st, err := bf.Stat(); err == nil { - totalSize += st.Size() - } - rcs = append(rcs, bf) - } - } - - // Read current file. - cf, err := os.Open(current) - if err != nil { - closeAll() - return nil, fmt.Errorf("opening current log file: %w", err) - } - if st, err := cf.Stat(); err == nil { - totalSize += st.Size() - } - rcs = append(rcs, cf) - - // Append warn log content from memory. - wlw.mu.Lock() - warnData := make([]byte, wlw.buf.Len()) - copy(warnData, wlw.buf.Bytes()) - wlw.mu.Unlock() - - if len(warnData) > 0 { - rcs = append(rcs, io.NopCloser(bytes.NewReader([]byte(logWriterLogEndMarker)))) - rcs = append(rcs, io.NopCloser(bytes.NewReader(warnData))) - totalSize += int64(len(logWriterLogEndMarker) + len(warnData)) - } - - if totalSize == 0 { - closeAll() - return nil, errors.New("internal log is empty") - } - - readers := make([]io.Reader, len(rcs)) - closers := make([]io.Closer, len(rcs)) - for i, rc := range rcs { - readers[i] = rc - closers[i] = rc - } - combined := io.MultiReader(readers...) - lr := &logReader{ - r: &multiCloser{Reader: combined, closers: closers}, - size: totalSize, - } - return lr, nil -} - -// multiCloser wraps an io.Reader and closes multiple underlying closers. -type multiCloser struct { - io.Reader - closers []io.Closer -} - -func (mc *multiCloser) Close() error { - var firstErr error - for _, c := range mc.closers { - if err := c.Close(); err != nil && firstErr == nil { - firstErr = err - } - } - return firstErr -} diff --git a/cmd/cli/log_writer_silent_test.go b/cmd/cli/log_writer_silent_test.go index e40f779..75b52f2 100644 --- a/cmd/cli/log_writer_silent_test.go +++ b/cmd/cli/log_writer_silent_test.go @@ -3,6 +3,7 @@ package cli import ( "os" "path/filepath" + "strings" "testing" "github.com/Control-D-Inc/ctrld" @@ -35,6 +36,7 @@ func Test_needInternalLogging_silent(t *testing.T) { silent = tt.silent cdUID = tt.cdUID p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) p.cfg.Service.LogPath = tt.logPath if got := p.needInternalLogging(); got != tt.want { t.Fatalf("needInternalLogging() = %v, want %v", got, tt.want) @@ -55,12 +57,81 @@ func Test_initInternalLogging_silentCreatesNoFile(t *testing.T) { homedir = dir cdUID = "test-uid" // cd mode, which would otherwise enable internal logging silent = true + stubHeaderSnapshotSources(t) p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) p.initInternalLogging(nil) - logPath := filepath.Join(dir, logFileName) - if _, err := os.Stat(logPath); !os.IsNotExist(err) { - t.Fatalf("silent mode must not create %s (stat err = %v)", logPath, err) + for _, name := range []string{logFileName, journalLogFileName} { + path := filepath.Join(dir, name) + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("silent mode must not create %s (stat err = %v)", path, err) + } + } +} + +// Test_initInternalLogging_createsJournalFile drives a cd mode run and asserts +// that the journal file holds the retained lines only, while the debug file +// holds every line. +func Test_initInternalLogging_createsJournalFile(t *testing.T) { + origSilent, origCdUID, origHomedir, origVerbose := silent, cdUID, homedir, verbose + origMainLog := mainLog.Load() + t.Cleanup(func() { + silent, cdUID, homedir, verbose = origSilent, origCdUID, origHomedir, origVerbose + mainLog.Store(origMainLog) + }) + if origMainLog == nil { + mainLog.Store(ctrld.NopLogger) + } + + dir := t.TempDir() + homedir = dir + cdUID = "test-uid" + silent = false + verbose = 0 + stubHeaderSnapshotSources(t) + + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.initInternalLogging(nil) + t.Cleanup(func() { + p.internalLogWriter.closeLogFile() + p.internalJournalWriter.closeLogFile() + }) + + debugPath := filepath.Join(dir, logFileName) + journalPath := filepath.Join(dir, journalLogFileName) + for _, path := range []string{debugPath, journalPath} { + if _, err := os.Stat(path); err != nil { + t.Fatalf("stat %s: %v", path, err) + } + } + + const ( + warnMsg = "retained warn line" + markedMsg = "retained marked info line" + plainMsg = "dropped plain info line" + ) + logger := mainLog.Load() + logger.Warn().Msg(warnMsg) + journal(logger.Info()).Msg(markedMsg) + logger.Info().Msg(plainMsg) + + journalText := rotatingFileTestContent(t, journalPath) + for _, msg := range []string{warnMsg, markedMsg} { + if !strings.Contains(journalText, msg) { + t.Fatalf("journal file misses %q: %s", msg, journalText) + } + } + if strings.Contains(journalText, plainMsg) { + t.Fatalf("journal file kept %q: %s", plainMsg, journalText) + } + + debugText := rotatingFileTestContent(t, debugPath) + for _, msg := range []string{warnMsg, markedMsg, plainMsg} { + if !strings.Contains(debugText, msg) { + t.Fatalf("debug file misses %q: %s", msg, debugText) + } } } diff --git a/cmd/cli/log_writer_test.go b/cmd/cli/log_writer_test.go index 4ff4f05..6aaada2 100644 --- a/cmd/cli/log_writer_test.go +++ b/cmd/cli/log_writer_test.go @@ -2,17 +2,20 @@ package cli import ( "bytes" + "encoding/json" + "errors" "io" "os" "path/filepath" + "slices" "strings" "sync" "testing" - - "go.uber.org/zap" - "go.uber.org/zap/zapcore" + "time" "github.com/Control-D-Inc/ctrld" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) func Test_logWriter_Write(t *testing.T) { @@ -93,6 +96,280 @@ func Test_logWriter_MarkerInitEnd(t *testing.T) { } } +// testLogWriterBudget keeps the rotation tests small: four lines fill the +// current file. +var testLogWriterBudget = logBudget{maxSize: 4 * rotatingFileTestLineSize, backups: 2} + +func newTestFileLogWriter(t *testing.T, budget logBudget) (*logWriter, string) { + t.Helper() + path := filepath.Join(t.TempDir(), "test.log") + lw := newLogWriterWithSize(logWriterSize) + if err := lw.setLogFile(path, budget); err != nil { + t.Fatalf("setLogFile: %v", err) + } + t.Cleanup(lw.closeLogFile) + return lw, path +} + +func Test_logWriter_SetLogFile(t *testing.T) { + lw, path := newTestFileLogWriter(t, testLogWriterBudget) + + msg := "hello file\n" + lw.Write([]byte(msg)) + + // Verify data in memory buffer. + if lw.buf.String() != msg { + t.Fatalf("buffer: got %q, want %q", lw.buf.String(), msg) + } + // Verify data on disk. + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != msg { + t.Fatalf("file: got %q, want %q", data, msg) + } +} + +func Test_logWriter_FileRotation(t *testing.T) { + lw, path := newTestFileLogWriter(t, testLogWriterBudget) + + // Three rotations: each file takes four lines. + for seq := 1; seq <= 16; seq++ { + lw.Write(rotatingFileTestLine(1, seq)) + } + + for _, suffix := range []string{".1", ".2"} { + if _, err := os.Stat(path + suffix); err != nil { + t.Fatalf("stat %s: %v", path+suffix, err) + } + } + if _, err := os.Stat(path + ".3"); !os.IsNotExist(err) { + t.Fatalf("stat %s.3 = %v, want a not exist error", path, err) + } + + st, err := os.Stat(path) + if err != nil { + t.Fatalf("stat current: %v", err) + } + if want := testLogWriterBudget.maxSize + rotatingFileTestLineSize; st.Size() > want { + t.Fatalf("current file holds %d bytes after rotation, want at most %d", st.Size(), want) + } +} + +func Test_logWriter_RotatingPaths(t *testing.T) { + lw := newLogWriterWithSize(logWriterSize) + if lw.rotating() != nil { + t.Fatal("a writer without a file has a rotating file, want none") + } + + lw, path := newTestFileLogWriter(t, testLogWriterBudget) + if got, want := lw.rotating().paths(), []string{path}; !slices.Equal(got, want) { + t.Fatalf("paths = %v, want %v", got, want) + } + + // Two rotations, so both backups exist. + for seq := 1; seq <= 9; seq++ { + lw.Write(rotatingFileTestLine(1, seq)) + } + want := []string{path + ".2", path + ".1", path} + if got := lw.rotating().paths(); !slices.Equal(got, want) { + t.Fatalf("paths = %v, want %v", got, want) + } +} + +func Test_logWriter_FileAppendOnRestart(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.log") + + // Simulate first run. + lw1 := newLogWriterWithSize(logWriterSize) + if err := lw1.setLogFile(path, testLogWriterBudget); err != nil { + t.Fatalf("setLogFile: %v", err) + } + lw1.Write([]byte("run1\n")) + lw1.closeLogFile() + + // Simulate second run (restart) — file should be appended. + lw2 := newLogWriterWithSize(logWriterSize) + if err := lw2.setLogFile(path, testLogWriterBudget); err != nil { + t.Fatalf("setLogFile: %v", err) + } + lw2.Write([]byte("run2\n")) + lw2.closeLogFile() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + want := "run1\nrun2\n" + if string(data) != want { + t.Fatalf("file: got %q, want %q", data, want) + } +} + +func Test_logWriter_EmitsLogRotated(t *testing.T) { + lw, path := newTestFileLogWriter(t, testLogWriterBudget) + + // The logger writes back into the writer that rotates, which is the case + // that deadlocks when the writer still holds its lock. + buf := &syncBuffer{} + logger := newTestJSONLogger(lw, buf) + old := mainLog.Load() + mainLog.Store(logger) + t.Cleanup(func() { mainLog.Store(old) }) + + done := make(chan struct{}) + go func() { + defer close(done) + for seq := 1; seq <= 5; seq++ { + lw.Write(rotatingFileTestLine(1, seq)) + } + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("the writer did not return: the rotation event blocked on the writer lock") + } + + st, err := os.Stat(path + ".1") + if err != nil { + t.Fatalf("stat %s.1: %v", path, err) + } + + var rotatedLines []string + for _, line := range strings.Split(buf.String(), "\n") { + if strings.Contains(line, "Log rotated") { + rotatedLines = append(rotatedLines, line) + } + } + if len(rotatedLines) != 1 { + t.Fatalf("got %d Log rotated events, want 1: %q", len(rotatedLines), buf.String()) + } + if !strings.Contains(rotatedLines[0], string(journalMarker)) { + t.Fatalf("the event is not marked for the journal: %s", rotatedLines[0]) + } + var event struct { + File string `json:"file"` + BytesWritten int64 `json:"bytes_written"` + } + if err := json.Unmarshal([]byte(rotatedLines[0]), &event); err != nil { + t.Fatalf("unmarshal %s: %v", rotatedLines[0], err) + } + if event.File != path+".1" { + t.Fatalf("file = %q, want %q", event.File, path+".1") + } + if event.BytesWritten != st.Size() { + t.Fatalf("bytes_written = %d, want %d", event.BytesWritten, st.Size()) + } +} + +func Test_logWriter_NestedRotationStillEmitsEvent(t *testing.T) { + debugLog, debugPath := newTestFileLogWriter(t, testLogWriterBudget) + journalLog, journalPath := newTestFileLogWriter(t, logBudget{maxSize: 2 * rotatingFileTestLineSize, backups: 1}) + + buf := &syncBuffer{} + logger := newTestJSONLogger(debugLog, journalLog, buf) + old := mainLog.Load() + mainLog.Store(logger) + t.Cleanup(func() { mainLog.Store(old) }) + + // Fill the journal, so the rotation event of the debug file is the line + // that rotates the journal. That nested rotation needs its own event. + journalLog.Write(rotatingFileTestLine(1, 1)) + journalLog.Write(rotatingFileTestLine(1, 2)) + for seq := 1; seq <= 5; seq++ { + debugLog.Write(rotatingFileTestLine(1, seq)) + } + + rotatedFiles := map[string]bool{} + for _, line := range strings.Split(buf.String(), "\n") { + if !strings.Contains(line, "Log rotated") { + continue + } + var event struct { + File string `json:"file"` + } + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("parse %q: %v", line, err) + } + rotatedFiles[event.File] = true + } + for _, want := range []string{debugPath + ".1", journalPath + ".1"} { + if !rotatedFiles[want] { + t.Fatalf("no Log rotated event for %s; events: %v", want, rotatedFiles) + } + } + if len(rotatedFiles) != 2 { + t.Fatalf("got %d rotated files, want 2: %v", len(rotatedFiles), rotatedFiles) + } +} + +// rotationEventFields parses one log line into its raw fields, so a test can +// ask whether a field is there at all. +func rotationEventFields(t *testing.T, line string) map[string]json.RawMessage { + t.Helper() + fields := map[string]json.RawMessage{} + if err := json.Unmarshal([]byte(strings.TrimSpace(line)), &fields); err != nil { + t.Fatalf("unmarshal %q: %v", line, err) + } + return fields +} + +func Test_emitLogRotated_eventTimes(t *testing.T) { + + t.Run("a file with no event names no time", func(t *testing.T) { + buf := captureJSONMainLog(t) + + emitLogRotated(logRotation{file: "/tmp/ctrld.log.1", backups: 1}) + + fields := rotationEventFields(t, buf.String()) + for _, name := range []string{"first_event_at", "last_event_at"} { + if raw, ok := fields[name]; ok { + t.Errorf("%s = %s, want no field for a file with no event", name, raw) + } + } + }) + + t.Run("a file with events names both times", func(t *testing.T) { + buf := captureJSONMainLog(t) + now := time.Now() + + emitLogRotated(logRotation{file: "/tmp/ctrld.log.1", backups: 1, firstWrite: now, lastWrite: now}) + + fields := rotationEventFields(t, buf.String()) + for _, name := range []string{"first_event_at", "last_event_at"} { + if _, ok := fields[name]; !ok { + t.Errorf("%s is missing: %v", name, fields) + } + } + }) +} + +func Test_emitLogRotated_reportsAFailedRotation(t *testing.T) { + buf := captureJSONMainLog(t) + + emitLogRotated(logRotation{file: "/tmp/ctrld.log.1", err: errors.New("rename refused")}) + + var event struct { + Level string `json:"level"` + Message string `json:"message"` + Error string `json:"error"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(buf.String())), &event); err != nil { + t.Fatalf("unmarshal %q: %v", buf.String(), err) + } + if event.Message != "Log rotation failed" { + t.Errorf("message = %q, want %q", event.Message, "Log rotation failed") + } + if event.Level != "warn" { + t.Errorf("level = %q, want %q", event.Level, "warn") + } + if event.Error != "rename refused" { + t.Errorf("error = %q, want %q", event.Error, "rename refused") + } +} + // TestNoticeLevel tests that the custom NOTICE level works correctly func TestNoticeLevel(t *testing.T) { // Create a buffer to capture log output @@ -417,126 +694,3 @@ func TestNewLogReader_ANSIRegexEdgeCases(t *testing.T) { }) } } - -func Test_logWriter_SetLogFile(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.log") - lw := newLogWriterWithSize(logWriterSize) - if err := lw.setLogFile(path); err != nil { - t.Fatalf("setLogFile: %v", err) - } - defer lw.closeLogFile() - - msg := "hello file\n" - lw.Write([]byte(msg)) - - // Verify data in memory buffer. - if lw.buf.String() != msg { - t.Fatalf("buffer: got %q, want %q", lw.buf.String(), msg) - } - // Verify data on disk. - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - if string(data) != msg { - t.Fatalf("file: got %q, want %q", data, msg) - } -} - -func Test_logWriter_FileRotation(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.log") - // Use a tiny max size to trigger rotation quickly. - lw := newLogWriterWithSize(logWriterSize) - if err := lw.setLogFile(path); err != nil { - t.Fatalf("setLogFile: %v", err) - } - defer lw.closeLogFile() - - // Write enough to exceed logFileMaxSize. - chunk := strings.Repeat("X", 1024) + "\n" - written := 0 - for written < logFileMaxSize+1024 { - lw.Write([]byte(chunk)) - written += len(chunk) - } - - // Backup file should exist. - backupPath := path + ".1" - if _, err := os.Stat(backupPath); os.IsNotExist(err) { - t.Fatal("expected backup file to exist after rotation") - } - - // Current file should be smaller than max (it was rotated). - st, err := os.Stat(path) - if err != nil { - t.Fatalf("stat current: %v", err) - } - if st.Size() > logFileMaxSize { - t.Fatalf("current file too large after rotation: %d", st.Size()) - } -} - -func Test_logWriter_FilePaths(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.log") - lw := newLogWriterWithSize(logWriterSize) - - // No file configured. - c, b := lw.logFilePaths() - if c != "" || b != "" { - t.Fatalf("expected empty paths, got %q %q", c, b) - } - - if err := lw.setLogFile(path); err != nil { - t.Fatalf("setLogFile: %v", err) - } - defer lw.closeLogFile() - - // Current exists, no backup yet. - c, b = lw.logFilePaths() - if c != path { - t.Fatalf("current: got %q, want %q", c, path) - } - if b != "" { - t.Fatalf("backup should be empty, got %q", b) - } - - // Create a backup file manually. - os.WriteFile(path+".1", []byte("old"), 0600) - _, b = lw.logFilePaths() - if b != path+".1" { - t.Fatalf("backup: got %q, want %q", b, path+".1") - } -} - -func Test_logWriter_FileAppendOnRestart(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.log") - - // Simulate first run. - lw1 := newLogWriterWithSize(logWriterSize) - if err := lw1.setLogFile(path); err != nil { - t.Fatalf("setLogFile: %v", err) - } - lw1.Write([]byte("run1\n")) - lw1.closeLogFile() - - // Simulate second run (restart) — file should be appended. - lw2 := newLogWriterWithSize(logWriterSize) - if err := lw2.setLogFile(path); err != nil { - t.Fatalf("setLogFile: %v", err) - } - lw2.Write([]byte("run2\n")) - lw2.closeLogFile() - - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - want := "run1\nrun2\n" - if string(data) != want { - t.Fatalf("file: got %q, want %q", data, want) - } -} diff --git a/cmd/cli/loop.go b/cmd/cli/loop.go index d77280b..a9de6e0 100644 --- a/cmd/cli/loop.go +++ b/cmd/cli/loop.go @@ -86,8 +86,9 @@ func (p *prog) detectLoop(msg *dns.Msg) { func (p *prog) checkDnsLoop() { p.Debug().Msg("Start checking DNS loop") upstream := make(map[string]*ctrld.UpstreamConfig) - // Preserve generated-upstream identity for privacy-safe probe diagnostics. - generated := make(map[string]bool) + // The probe loop below runs on the upstream UID, and the failure report + // needs the config key. Record the reference while the key is in hand. + reference := make(map[string]string) p.loopMu.Lock() for n, uc := range p.cfg.Upstream { if p.um.isDown("upstream." + n) { @@ -101,7 +102,7 @@ func (p *prog) checkDnsLoop() { uid := uc.UID() p.loop[uid] = false upstream[uid] = uc - generated[uid] = isGeneratedInternalDomainUpstream(n, uc) + reference[uid] = upstreamPrefix + n } p.loopMu.Unlock() @@ -115,11 +116,11 @@ func (p *prog) checkDnsLoop() { } resolver, err := ctrld.NewResolver(loggerCtx, uc) if err != nil { - p.logUpstreamProbeFailure(generated[uid], uc, err, p.Warn, "Could not perform loop check") + p.logUpstreamProbeFailure(reference[uid], uc, err, p.Warn, "Could not perform loop check") continue } if _, err := resolver.Resolve(context.Background(), msg); err != nil { - p.logUpstreamProbeFailure(generated[uid], uc, err, p.Warn, "Could not send DNS loop check query") + p.logUpstreamProbeFailure(reference[uid], uc, err, p.Warn, "Could not send DNS loop check query") } } p.Debug().Msg("End checking DNS loop") diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 1711de9..2df009f 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -148,34 +148,8 @@ func initInteractiveLogging() { // wrapper instead of calling this function directly. func initLoggingWithBackup(doBackup bool) []zapcore.Core { var writers []io.Writer - if logFilePath := normalizeLogFilePath(cfg.Service.LogPath); logFilePath != "" { - // Create parent directory if necessary. - // This ensures log files can be created even if the directory doesn't exist - if err := os.MkdirAll(filepath.Dir(logFilePath), 0750); err != nil { - mainLog.Load().Error().Msgf("Failed to create log path: %v", err) - os.Exit(1) - } - - // Default open log file in append mode. - // This preserves existing log entries across restarts - flags := os.O_CREATE | os.O_RDWR | os.O_APPEND - if doBackup { - // Backup old log file with .1 suffix. - // This prevents log file corruption during rotation - if err := os.Rename(logFilePath, logFilePath+oldLogSuffix); err != nil && !os.IsNotExist(err) { - mainLog.Load().Error().Msgf("Could not backup old log file: %v", err) - } else { - // Backup was created, set flags for truncating old log file. - // This ensures a clean start for the new log file - flags = os.O_CREATE | os.O_RDWR - } - } - logFile, err := openLogFile(logFilePath, flags) - if err != nil { - mainLog.Load().Error().Msgf("Failed to create log file: %v", err) - os.Exit(1) - } - writers = append(writers, logFile) + if rf := openLogPathWriter(doBackup); rf != nil { + writers = append(writers, rf) } // Create zap cores for different writers diff --git a/cmd/cli/net_darwin.go b/cmd/cli/net_darwin.go index 7f756c4..3f2a163 100644 --- a/cmd/cli/net_darwin.go +++ b/cmd/cli/net_darwin.go @@ -3,14 +3,52 @@ package cli import ( "bufio" "bytes" + "context" + "errors" "io" "net" "os/exec" "strings" + "sync" + "time" +) + +// hardwarePortInfo holds the two names that macOS gives a network device: the +// hardware port of the adapter and the network service that uses it. +type hardwarePortInfo struct { + Port string + Service string +} + +// A network event reads the ports of every interface, so the two networksetup +// calls run at most once per window. +const hardwarePortsTTL = time.Minute + +// networksetupTimeout bounds one command. A stuck read must not hold the +// network callback that waits for the port names. +const networksetupTimeout = 5 * time.Second + +// maxNetworksetupOutput bounds one read. A host with many ports stays far +// below it, so more than this is a broken command. +const maxNetworksetupOutput = 1 << 20 + +// maxNetworksetupLine bounds one line, so one token fills no memory. +const maxNetworksetupLine = 64 * 1024 + +var errNetworksetupOutputTooLarge = errors.New("networksetup output above the bound") + +var ( + hardwarePortsMu sync.Mutex + hardwarePortsCache map[string]hardwarePortInfo + hardwarePortsReadAt time.Time + + // Tests replace these seams to avoid a networksetup call. + readHardwarePortsFn = readHardwarePorts + hardwarePortsNowFn = time.Now ) func patchNetIfaceName(iface *net.Interface) (bool, error) { - b, err := exec.Command("networksetup", "-listnetworkserviceorder").Output() + b, err := networksetupOutput("-listnetworkserviceorder") if err != nil { return false, err } @@ -24,24 +62,215 @@ func patchNetIfaceName(iface *net.Interface) (bool, error) { } func networkServiceName(ifaceName string, r io.Reader) string { - scanner := bufio.NewScanner(r) + return serviceNamesByDevice(r)[ifaceName] +} + +// serviceNamesByDevice maps each device of "networksetup +// -listnetworkserviceorder" to the name of the service that owns it. +func serviceNamesByDevice(r io.Reader) map[string]string { + names := make(map[string]string) + scanner := scanNetworksetup(r) prevLine := "" for scanner.Scan() { line := scanner.Text() if strings.Contains(line, "*") { - // Network services is disabled. + // A disabled service names no device. Without the reset, the device + // line that follows takes the name of the service before it. + prevLine = "" continue } - if !strings.Contains(line, "Device: "+ifaceName) { + device := serviceOrderDevice(line) + if device == "" { prevLine = line continue } - parts := strings.SplitN(prevLine, " ", 2) - if len(parts) == 2 { - return strings.TrimSpace(parts[1]) + if _, seen := names[device]; !seen { + names[device] = serviceOrderName(prevLine) } + prevLine = line } - return "" + if scanner.Err() != nil { + // A truncated read names fewer services than the host has. + return nil + } + return names +} + +// scanNetworksetup reads the output line by line under a bounded buffer. +func scanNetworksetup(r io.Reader) *bufio.Scanner { + scanner := bufio.NewScanner(r) + scanner.Buffer(nil, maxNetworksetupLine) + return scanner +} + +// serviceOrderDevice reads the device out of a line like +// "(Hardware Port: Wi-Fi, Device: en0)". +func serviceOrderDevice(line string) string { + _, after, found := strings.Cut(line, "Device:") + if !found { + return "" + } + return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(after), ")")) +} + +// serviceOrderName reads the service name out of a line like "(1) Wi-Fi". +func serviceOrderName(line string) string { + parts := strings.SplitN(line, " ", 2) + if len(parts) != 2 { + return "" + } + return strings.TrimSpace(parts[1]) +} + +// parseHardwarePorts maps each device of "networksetup -listallhardwareports" +// to its hardware port. Each block names the port before the device. +func parseHardwarePorts(r io.Reader) map[string]hardwarePortInfo { + ports := make(map[string]hardwarePortInfo) + scanner := scanNetworksetup(r) + port := "" + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if after, ok := strings.CutPrefix(line, "Hardware Port:"); ok { + port = strings.TrimSpace(after) + continue + } + after, ok := strings.CutPrefix(line, "Device:") + if !ok { + continue + } + device := strings.TrimSpace(after) + if device == "" { + continue + } + ports[device] = hardwarePortInfo{Port: port} + port = "" + } + if scanner.Err() != nil { + // A truncated read names fewer ports than the host has. + return nil + } + return ports +} + +// addServiceNames fills the service of every known device. A device that no +// hardware port lists is not a port of this host. +func addServiceNames(m map[string]hardwarePortInfo, r io.Reader) { + for device, service := range serviceNamesByDevice(r) { + info, known := m[device] + if !known { + continue + } + info.Service = service + m[device] = info + } +} + +// hardwarePortsByDevice returns the port and service of every device. The +// commands run outside the mutex, so a slow read blocks no other caller. +func hardwarePortsByDevice() map[string]hardwarePortInfo { + if ports, fresh := cachedHardwarePorts(); fresh { + return ports + } + return storeHardwarePorts(readHardwarePortsFn()) +} + +// cachedHardwarePorts reports whether the read window still holds. +func cachedHardwarePorts() (map[string]hardwarePortInfo, bool) { + hardwarePortsMu.Lock() + defer hardwarePortsMu.Unlock() + if hardwarePortsReadAt.IsZero() { + return nil, false + } + return hardwarePortsCache, hardwarePortsNowFn().Sub(hardwarePortsReadAt) < hardwarePortsTTL +} + +// storeHardwarePorts starts the window again and keeps the last good map. A +// failing command must run once per window, not once per lookup. +func storeHardwarePorts(ports map[string]hardwarePortInfo) map[string]hardwarePortInfo { + hardwarePortsMu.Lock() + defer hardwarePortsMu.Unlock() + hardwarePortsReadAt = hardwarePortsNowFn() + if ports != nil { + hardwarePortsCache = ports + } + return hardwarePortsCache +} + +// readHardwarePorts runs the two networksetup commands. +func readHardwarePorts() map[string]hardwarePortInfo { + b, err := networksetupOutput("-listallhardwareports") + if err != nil { + return nil + } + ports := parseHardwarePorts(bytes.NewReader(b)) + if ports == nil { + // A truncated list must keep the last good map. + return nil + } + b, err = networksetupOutput("-listnetworkserviceorder") + if err != nil { + return ports + } + addServiceNames(ports, bytes.NewReader(b)) + return ports +} + +// networksetupOutput runs one networksetup list under a time bound and a size +// bound. +func networksetupOutput(arg string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), networksetupTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "networksetup", arg) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + out, readErr := readBoundedOutput(stdout) + if readErr != nil { + // A command ends only when something reads the rest of its output. + _, _ = io.Copy(io.Discard, stdout) + } + if err := cmd.Wait(); err != nil { + return nil, err + } + return out, readErr +} + +// readBoundedOutput reads at most maxNetworksetupOutput bytes and reports an +// error above that. +func readBoundedOutput(r io.Reader) ([]byte, error) { + out, err := io.ReadAll(io.LimitReader(r, maxNetworksetupOutput+1)) + if err != nil { + return nil, err + } + if len(out) > maxNetworksetupOutput { + return nil, errNetworksetupOutputTooLarge + } + return out, nil +} + +// platformInterfaceMeta returns the macOS names of an interface. The class +// comes from the interface name, so it stays empty here. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + info := hardwarePortsByDevice()[name] + return "", info.Port, info.Service +} + +// platformVirtualInterfaces returns no set. The name rules already class the +// AirDrop and the link-local interfaces of macOS. +func platformVirtualInterfaces() map[string]struct{} { + return nil +} + +// refreshInterfaceMeta drops the read window. The cache has no port name for +// an adapter that appeared, so the next lookup reads the ports again. +func refreshInterfaceMeta() { + hardwarePortsMu.Lock() + defer hardwarePortsMu.Unlock() + hardwarePortsReadAt = time.Time{} } // validInterface reports whether the *net.Interface is a valid one. diff --git a/cmd/cli/net_darwin_test.go b/cmd/cli/net_darwin_test.go new file mode 100644 index 0000000..4bd9d43 --- /dev/null +++ b/cmd/cli/net_darwin_test.go @@ -0,0 +1,284 @@ +package cli + +import ( + "strings" + "testing" + "time" +) + +const listAllHardwarePortsFixture = ` +Hardware Port: Wi-Fi +Device: en0 +Ethernet Address: a0:78:17:68:56:3f + +Hardware Port: iPhone USB +Device: en7 +Ethernet Address: 6e:5a:6b:12:34:56 + +Hardware Port: Thunderbolt Bridge +Device: bridge0 +Ethernet Address: 36:21:bb:3a:7a:40 + +Hardware Port: Ethernet Adapter (en6) +Device: en6 +Ethernet Address: 3a:3e:fc:1e:ab:41 + +Hardware Port: com.wireguard.macos +Device: + +VLAN Configurations +=================== +` + +const listNetworkServiceOrderFixture = `An asterisk (*) denotes that a network service is disabled. +(1) Wi-Fi +(Hardware Port: Wi-Fi, Device: en0) + +(2) iPhone USB +(Hardware Port: iPhone USB, Device: en7) + +(3) Thunderbolt Bridge +(Hardware Port: Thunderbolt Bridge, Device: bridge0) + +(*) Ethernet Adapter (en6) +(Hardware Port: Ethernet Adapter (en6), Device: en6) + +(4) ca-001-stg +(Hardware Port: com.wireguard.macos, Device: ) + +` + +func Test_parseHardwarePorts(t *testing.T) { + ports := parseHardwarePorts(strings.NewReader(listAllHardwarePortsFixture)) + addServiceNames(ports, strings.NewReader(listNetworkServiceOrderFixture)) + + tests := []struct { + device string + wantPort string + wantService string + }{ + {"en0", "Wi-Fi", "Wi-Fi"}, + {"en7", "iPhone USB", "iPhone USB"}, + {"bridge0", "Thunderbolt Bridge", "Thunderbolt Bridge"}, + {"en6", "Ethernet Adapter (en6)", ""}, + } + if len(ports) != len(tests) { + t.Fatalf("parseHardwarePorts returned %d devices, want %d: %v", len(ports), len(tests), ports) + } + for _, tc := range tests { + t.Run(tc.device, func(t *testing.T) { + info := ports[tc.device] + if info.Port != tc.wantPort { + t.Errorf("port of %s = %q, want %q", tc.device, info.Port, tc.wantPort) + } + if info.Service != tc.wantService { + t.Errorf("service of %s = %q, want %q", tc.device, info.Service, tc.wantService) + } + }) + } +} + +func Test_networkServiceName(t *testing.T) { + tests := []struct { + device string + want string + }{ + {"en0", "Wi-Fi"}, + {"en7", "iPhone USB"}, + {"bridge0", "Thunderbolt Bridge"}, + {"en6", ""}, + {"en9", ""}, + } + for _, tc := range tests { + t.Run(tc.device, func(t *testing.T) { + got := networkServiceName(tc.device, strings.NewReader(listNetworkServiceOrderFixture)) + if got != tc.want { + t.Errorf("networkServiceName(%q) = %q, want %q", tc.device, got, tc.want) + } + }) + } +} + +func Test_parseHardwarePorts_dropsATruncatedRead(t *testing.T) { + longLine := strings.Repeat("x", maxNetworksetupLine+1) + + if got := parseHardwarePorts(strings.NewReader(listAllHardwarePortsFixture + longLine)); got != nil { + t.Errorf("parseHardwarePorts of a truncated read = %v, want nil", got) + } + if got := serviceNamesByDevice(strings.NewReader(listNetworkServiceOrderFixture + longLine)); got != nil { + t.Errorf("serviceNamesByDevice of a truncated read = %v, want nil", got) + } +} + +func Test_readBoundedOutput(t *testing.T) { + out, err := readBoundedOutput(strings.NewReader(listAllHardwarePortsFixture)) + if err != nil || string(out) != listAllHardwarePortsFixture { + t.Fatalf("readBoundedOutput of the fixture = %q, %v, want the fixture and no error", out, err) + } + if _, err := readBoundedOutput(strings.NewReader(strings.Repeat("x", maxNetworksetupOutput+1))); err == nil { + t.Fatal("readBoundedOutput of an output above the bound gave no error") + } +} + +// stubHardwarePorts replaces the command and clock seams of the hardware port +// cache, and restores them with the cache itself. +func stubHardwarePorts(t *testing.T, read func() map[string]hardwarePortInfo, now func() time.Time) { + t.Helper() + readBefore, nowBefore := readHardwarePortsFn, hardwarePortsNowFn + hardwarePortsMu.Lock() + cacheBefore, readAtBefore := hardwarePortsCache, hardwarePortsReadAt + hardwarePortsCache, hardwarePortsReadAt = nil, time.Time{} + hardwarePortsMu.Unlock() + t.Cleanup(func() { + readHardwarePortsFn, hardwarePortsNowFn = readBefore, nowBefore + hardwarePortsMu.Lock() + hardwarePortsCache, hardwarePortsReadAt = cacheBefore, readAtBefore + hardwarePortsMu.Unlock() + }) + readHardwarePortsFn, hardwarePortsNowFn = read, now +} + +func Test_hardwarePortsByDevice_caches(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi", Service: "Wi-Fi"}} + }, func() time.Time { return now }) + + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("first read port = %q, want Wi-Fi", got) + } + now = now.Add(59 * time.Second) + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads within the cache window = %d, want 1", reads) + } + now = now.Add(2 * time.Second) + hardwarePortsByDevice() + if reads != 2 { + t.Fatalf("reads after the cache window = %d, want 2", reads) + } +} + +func Test_hardwarePortsByDevice_keepsLastGoodRead(t *testing.T) { + now := time.Unix(1700000000, 0) + fail := false + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + if fail { + return nil + } + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi"}} + }, func() time.Time { return now }) + + hardwarePortsByDevice() + fail = true + now = now.Add(2 * time.Minute) + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("port after a failed read = %q, want Wi-Fi", got) + } +} + +func Test_hardwarePortsByDevice_keepsTheWindowAfterAFailedRead(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return nil + }, func() time.Time { return now }) + + hardwarePortsByDevice() + now = now.Add(time.Second) + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads of a failing command within the window = %d, want 1", reads) + } + now = now.Add(2 * time.Minute) + hardwarePortsByDevice() + if reads != 2 { + t.Fatalf("reads after the window = %d, want 2", reads) + } +} + +func Test_refreshInterfaceMeta_dropsTheReadWindow(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi"}} + }, func() time.Time { return now }) + + hardwarePortsByDevice() + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads within the window = %d, want 1", reads) + } + refreshInterfaceMeta() + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("port after a refresh = %q, want Wi-Fi", got) + } + if reads != 2 { + t.Fatalf("reads after a refresh = %d, want 2", reads) + } +} + +func Test_hardwarePortsByDevice_readsOutsideTheLock(t *testing.T) { + now := time.Unix(1700000000, 0) + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + // A command runs outside the mutex, so this read can take it. + if !hardwarePortsMu.TryLock() { + t.Error("the command ran under the hardware ports mutex") + } else { + hardwarePortsMu.Unlock() + } + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi"}} + }, func() time.Time { return now }) + + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("port = %q, want Wi-Fi", got) + } +} + +// fixtureHardwarePorts returns the port map of the two command fixtures. +func fixtureHardwarePorts() map[string]hardwarePortInfo { + ports := parseHardwarePorts(strings.NewReader(listAllHardwarePortsFixture)) + addServiceNames(ports, strings.NewReader(listNetworkServiceOrderFixture)) + return ports +} + +func Test_transitionTestGlobalsReadNoPorts(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return fixtureHardwarePorts() + }, func() time.Time { return now }) + + TestNetworkChangeDoesNotRestoreDownDefaultRouteSources(t) + TestNetworkChangeSourceMovedToUpInterface(t) + + if reads != 0 { + t.Fatalf("port reads of two transition tests = %d, want 0", reads) + } +} + +func Test_platformInterfaceMeta(t *testing.T) { + now := time.Unix(1700000000, 0) + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + return map[string]hardwarePortInfo{"en7": {Port: "iPhone USB", Service: "iPhone USB"}} + }, func() time.Time { return now }) + + class, hardwarePort, service := platformInterfaceMeta("en7") + if class != "" { + t.Errorf("class = %q, want empty", class) + } + if hardwarePort != "iPhone USB" || service != "iPhone USB" { + t.Errorf("port, service = %q, %q, want iPhone USB twice", hardwarePort, service) + } + if class, hardwarePort, service = platformInterfaceMeta("utun3"); class != "" || hardwarePort != "" || service != "" { + t.Errorf("unknown device gave %q, %q, %q, want empty strings", class, hardwarePort, service) + } + if platformVirtualInterfaces() != nil { + t.Error("platformVirtualInterfaces is not nil on darwin") + } +} diff --git a/cmd/cli/net_linux.go b/cmd/cli/net_linux.go index f5a07de..4f098a4 100644 --- a/cmd/cli/net_linux.go +++ b/cmd/cli/net_linux.go @@ -2,8 +2,38 @@ package cli import ( "net" + "os" + "path/filepath" + "slices" + "strconv" + "strings" ) +// The kernel gives an Ethernet-like link the type ARPHRD_ETHER. +const arphrdEther = 1 + +// Tests point the readers at a temporary directory. +var ( + sysfsNetRoot = "/sys/class/net" + sysfsVirtualNetRoot = "/sys/devices/virtual/net" +) + +// noiseInterfacePrefixes name the interfaces of a container host. They change +// every few seconds, and no traffic of the host goes through them. +var noiseInterfacePrefixes = []string{"docker", "veth", "virbr", "br-"} + +// usbTetherDrivers name the kernel modules of a phone that shares its data +// connection over the cable. +var usbTetherDrivers = []string{"ipheth", "rndis_host"} + +// sysfsInfo holds what sysfs knows about one interface. +type sysfsInfo struct { + Type int + DevType string + Driver string + Wireless bool +} + // patchNetIfaceName patches network interface names on Linux // This is a no-op on Linux as interface names don't need special handling func patchNetIfaceName(iface *net.Interface) (bool, error) { return true, nil } @@ -15,3 +45,118 @@ func validInterface(iface *net.Interface, validIfacesMap map[string]struct{}) bo _, ok := validIfacesMap[iface.Name] return ok } + +// sysfsVirtualInterfaces returns the virtual interfaces of the host. The +// nameserver code has its own reader of the fixed path; this one reads the +// root that a test can point elsewhere. +func sysfsVirtualInterfaces() map[string]struct{} { + s := make(map[string]struct{}) + entries, _ := os.ReadDir(sysfsVirtualNetRoot) + for _, entry := range entries { + if entry.IsDir() { + s[strings.TrimSpace(entry.Name())] = struct{}{} + } + } + return s +} + +// sysfsInterfaceInfo reads the sysfs entry of an interface. A missing file +// leaves its field empty. +func sysfsInterfaceInfo(name string) sysfsInfo { + dir := filepath.Join(sysfsNetRoot, name) + info := sysfsInfo{DevType: sysfsUeventValue(dir, "DEVTYPE"), Driver: sysfsDriver(dir)} + info.Type, _ = strconv.Atoi(readSysfsAttribute(dir, "type")) + if entry, err := os.Stat(filepath.Join(dir, "wireless")); err == nil && entry.IsDir() { + info.Wireless = true + } + return info +} + +// readSysfsAttribute reads one attribute without its trailing newline. +func readSysfsAttribute(dir, file string) string { + b, err := os.ReadFile(filepath.Join(dir, file)) + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} + +// sysfsUeventValue reads one entry of the uevent file. +func sysfsUeventValue(dir, key string) string { + for _, line := range strings.Split(readSysfsAttribute(dir, "uevent"), "\n") { + after, ok := strings.CutPrefix(strings.TrimSpace(line), key+"=") + if !ok { + continue + } + return strings.TrimSpace(after) + } + return "" +} + +// sysfsDriver names the kernel module behind an interface. The driver link of +// the device points at the module, and the uevent of the device names it too. +func sysfsDriver(dir string) string { + device := filepath.Join(dir, "device") + if target, err := os.Readlink(filepath.Join(device, "driver")); err == nil { + return filepath.Base(target) + } + if driver := sysfsUeventValue(device, "DRIVER"); driver != "" { + return driver + } + return sysfsUeventValue(dir, "DRIVER") +} + +// sysfsHardwarePort names the link in the words that the link type mapping +// reads. The port names of macOS use the same words. +func sysfsHardwarePort(info sysfsInfo) string { + switch { + case info.Wireless: + return "Wi-Fi" + case slices.Contains(usbTetherDrivers, info.Driver): + return hardwarePortIPhoneUSB + case info.DevType != "": + return info.DevType + case info.Type == arphrdEther: + return "Ethernet" + } + return "" +} + +// sysfsClass names the kind of an interface. The name rule comes before the +// device type, so an interface that went away keeps its class. A bridge, a +// bond, and a VLAN carry the traffic of the host, so they stay hardware. +func sysfsClass(name string, info sysfsInfo) string { + switch { + case info.DevType == "tun" || info.DevType == "tap": + return "tunnel" + case hasInterfacePrefix(name, noiseInterfacePrefixes...): + return "virtual" + case info.Wireless || info.Type == arphrdEther: + return "hardware" + } + return "" +} + +// platformInterfaceMeta returns the Linux class and link name of an interface. +// Linux has no service name. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + info := sysfsInterfaceInfo(name) + return sysfsClass(name, info), sysfsHardwarePort(info), "" +} + +// platformVirtualInterfaces returns the container interfaces of the host. The +// kernel lists the uplink bridges and the bonds beside them, and those carry +// the traffic of the host. +func platformVirtualInterfaces() map[string]struct{} { + noise := make(map[string]struct{}) + for name := range sysfsVirtualInterfaces() { + if hasInterfacePrefix(name, noiseInterfacePrefixes...) { + noise[name] = struct{}{} + } + } + return noise +} + +// refreshInterfaceMeta does no work. Each sysfs read answers from the kernel +// of this moment, so there is no cache to refresh. +func refreshInterfaceMeta() {} diff --git a/cmd/cli/net_linux_test.go b/cmd/cli/net_linux_test.go new file mode 100644 index 0000000..cbf61c5 --- /dev/null +++ b/cmd/cli/net_linux_test.go @@ -0,0 +1,201 @@ +package cli + +import ( + "maps" + "os" + "path/filepath" + "testing" +) + +// writeSysfsInterface builds one interface entry under a fake sysfs root. +func writeSysfsInterface(t *testing.T, root, name string, files map[string]string, wireless bool) { + t.Helper() + dir := filepath.Join(root, name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("create %s: %v", dir, err) + } + for file, content := range files { + if err := os.WriteFile(filepath.Join(dir, file), []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", file, err) + } + } + if !wireless { + return + } + if err := os.MkdirAll(filepath.Join(dir, "wireless"), 0o755); err != nil { + t.Fatalf("create the wireless directory: %v", err) + } +} + +// writeSysfsDriverLink points the device entry of an interface at a driver. +func writeSysfsDriverLink(t *testing.T, root, name, driver string) { + t.Helper() + device := filepath.Join(root, name, "device") + if err := os.MkdirAll(device, 0o755); err != nil { + t.Fatalf("create %s: %v", device, err) + } + if err := os.Symlink(filepath.Join(root, "drivers", driver), filepath.Join(device, "driver")); err != nil { + t.Fatalf("link the driver of %s: %v", name, err) + } +} + +// writeSysfsDriverUevent names the driver in the uevent of the device. +func writeSysfsDriverUevent(t *testing.T, root, name, driver string) { + t.Helper() + device := filepath.Join(root, name, "device") + if err := os.MkdirAll(device, 0o755); err != nil { + t.Fatalf("create %s: %v", device, err) + } + if err := os.WriteFile(filepath.Join(device, "uevent"), []byte("DRIVER="+driver+"\n"), 0o644); err != nil { + t.Fatalf("write the uevent of %s: %v", name, err) + } +} + +// stubSysfsNetRoot points the sysfs reader at a temporary directory. +func stubSysfsNetRoot(t *testing.T) string { + t.Helper() + before := sysfsNetRoot + t.Cleanup(func() { sysfsNetRoot = before }) + sysfsNetRoot = t.TempDir() + return sysfsNetRoot +} + +func Test_sysfsInterfaceInfo(t *testing.T) { + root := stubSysfsNetRoot(t) + writeSysfsInterface(t, root, "wlan0", map[string]string{ + "type": "1\n", + "uevent": "INTERFACE=wlan0\nDEVTYPE=wlan\nIFINDEX=3\n", + }, true) + writeSysfsInterface(t, root, "eth0", map[string]string{ + "type": "1\n", + "uevent": "INTERFACE=eth0\nIFINDEX=2\n", + }, false) + writeSysfsInterface(t, root, "tun0", map[string]string{ + "type": "65534\n", + "uevent": "DEVTYPE=tun\n", + }, false) + + tests := []struct { + name string + want sysfsInfo + }{ + {"wlan0", sysfsInfo{Type: 1, DevType: "wlan", Wireless: true}}, + {"eth0", sysfsInfo{Type: 1}}, + {"tun0", sysfsInfo{Type: 65534, DevType: "tun"}}, + {"absent0", sysfsInfo{}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sysfsInterfaceInfo(tc.name); got != tc.want { + t.Errorf("sysfsInterfaceInfo(%q) = %+v, want %+v", tc.name, got, tc.want) + } + }) + } +} + +func Test_sysfsClass(t *testing.T) { + tests := []struct { + name string + iface string + info sysfsInfo + want string + }{ + {"wireless", "wlan0", sysfsInfo{Wireless: true}, "hardware"}, + {"ethernet", "eth0", sysfsInfo{Type: 1}, "hardware"}, + {"tun devtype", "tun0", sysfsInfo{Type: 65534, DevType: "tun"}, "tunnel"}, + {"tap devtype", "tap0", sysfsInfo{Type: 1, DevType: "tap"}, "tunnel"}, + {"unknown", "sit0", sysfsInfo{Type: 65534}, ""}, + {"docker bridge", "docker0", sysfsInfo{Type: 1, DevType: "bridge"}, "virtual"}, + {"container link", "veth1a2b3c", sysfsInfo{Type: 1}, "virtual"}, + {"libvirt bridge", "virbr0", sysfsInfo{Type: 1, DevType: "bridge"}, "virtual"}, + {"compose bridge", "br-1a2b3c", sysfsInfo{Type: 1, DevType: "bridge"}, "virtual"}, + {"uplink bridge", "br0", sysfsInfo{Type: 1, DevType: "bridge"}, "hardware"}, + {"bond", "bond0", sysfsInfo{Type: 1, DevType: "bond"}, "hardware"}, + {"vlan uplink", "eth0.100", sysfsInfo{Type: 1, DevType: "vlan"}, "hardware"}, + {"removed container link", "veth9f8e7d", sysfsInfo{}, "virtual"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sysfsClass(tc.iface, tc.info); got != tc.want { + t.Errorf("sysfsClass(%q, %+v) = %q, want %q", tc.iface, tc.info, got, tc.want) + } + }) + } +} + +func Test_platformVirtualInterfaces_namesTheContainerInterfaces(t *testing.T) { + root := t.TempDir() + before := sysfsVirtualNetRoot + t.Cleanup(func() { sysfsVirtualNetRoot = before }) + sysfsVirtualNetRoot = root + for _, name := range []string{"lo", "docker0", "veth1a2b3c", "virbr0", "br-1a2b3c", "br0", "bond0"} { + if err := os.MkdirAll(filepath.Join(root, name), 0o755); err != nil { + t.Fatalf("create %s: %v", name, err) + } + } + + want := map[string]struct{}{"docker0": {}, "veth1a2b3c": {}, "virbr0": {}, "br-1a2b3c": {}} + if got := platformVirtualInterfaces(); !maps.Equal(got, want) { + t.Errorf("platformVirtualInterfaces = %v, want %v", got, want) + } + if got := sysfsVirtualInterfaces(); len(got) != 7 { + t.Errorf("virtualInterfaces = %v, want every entry of the kernel directory", got) + } +} + +func Test_platformInterfaceMeta(t *testing.T) { + root := stubSysfsNetRoot(t) + writeSysfsInterface(t, root, "ctrld-test-eth0", map[string]string{ + "type": "1\n", + "uevent": "DEVTYPE=bridge\n", + }, false) + + class, hardwarePort, service := platformInterfaceMeta("ctrld-test-eth0") + if class != "hardware" { + t.Errorf("class = %q, want hardware", class) + } + if hardwarePort != "bridge" { + t.Errorf("hardware port = %q, want bridge", hardwarePort) + } + if service != "" { + t.Errorf("service = %q, want empty", service) + } +} + +func Test_platformInterfaceMeta_namesTheLink(t *testing.T) { + root := stubSysfsNetRoot(t) + writeSysfsInterface(t, root, "wlan0", map[string]string{"type": "1\n", "uevent": "DEVTYPE=wlan\n"}, true) + writeSysfsInterface(t, root, "eth0", map[string]string{"type": "1\n", "uevent": "INTERFACE=eth0\n"}, false) + writeSysfsInterface(t, root, "br0", map[string]string{"type": "1\n", "uevent": "DEVTYPE=bridge\n"}, false) + writeSysfsInterface(t, root, "tun0", map[string]string{"type": "65534\n", "uevent": "DEVTYPE=tun\n"}, false) + writeSysfsInterface(t, root, "usb0", map[string]string{"type": "1\n"}, false) + writeSysfsDriverLink(t, root, "usb0", "ipheth") + writeSysfsInterface(t, root, "usb1", map[string]string{"type": "1\n"}, false) + writeSysfsDriverUevent(t, root, "usb1", "rndis_host") + writeSysfsInterface(t, root, "eth1", map[string]string{"type": "1\n"}, false) + writeSysfsDriverUevent(t, root, "eth1", "e1000e") + + tests := []struct { + iface, class, hardwarePort, linkType string + }{ + {"wlan0", "hardware", "Wi-Fi", "wifi"}, + {"eth0", "hardware", "Ethernet", "ethernet"}, + {"eth1", "hardware", "Ethernet", "ethernet"}, + {"br0", "hardware", "bridge", "unknown"}, + {"tun0", "tunnel", "tun", "tunnel"}, + {"usb0", "hardware", hardwarePortIPhoneUSB, "usb_tether"}, + {"usb1", "hardware", hardwarePortIPhoneUSB, "usb_tether"}, + } + for _, tc := range tests { + t.Run(tc.iface, func(t *testing.T) { + class, hardwarePort, service := platformInterfaceMeta(tc.iface) + if class != tc.class || hardwarePort != tc.hardwarePort || service != "" { + t.Errorf("platformInterfaceMeta(%q) = %q, %q, %q, want %q, %q, empty", + tc.iface, class, hardwarePort, service, tc.class, tc.hardwarePort) + } + if got := linkTypeFor(hardwarePort, class); got != tc.linkType { + t.Errorf("linkTypeFor(%q, %q) = %q, want %q", hardwarePort, class, got, tc.linkType) + } + }) + } +} diff --git a/cmd/cli/net_others.go b/cmd/cli/net_others.go index 4ab96de..03c80dd 100644 --- a/cmd/cli/net_others.go +++ b/cmd/cli/net_others.go @@ -11,3 +11,18 @@ func patchNetIfaceName(iface *net.Interface) (bool, error) { return true, nil } // validInterface checks if an interface is valid on non-Linux/Darwin platforms func validInterface(iface *net.Interface, validIfacesMap map[string]struct{}) bool { return true } + +// platformInterfaceMeta returns no names. These platforms have no source for +// them, so the interface name decides the class. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + return "", "", "" +} + +// platformVirtualInterfaces returns no set. +func platformVirtualInterfaces() map[string]struct{} { + return nil +} + +// refreshInterfaceMeta does no work. These platforms keep no interface names, +// so an adapter that appears leaves nothing stale. +func refreshInterfaceMeta() {} diff --git a/cmd/cli/net_windows.go b/cmd/cli/net_windows.go index bdd6dcf..d77858e 100644 --- a/cmd/cli/net_windows.go +++ b/cmd/cli/net_windows.go @@ -1,7 +1,44 @@ package cli import ( + "io" + "log" "net" + "os" + "strings" + "sync" + "time" + + "github.com/microsoft/wmi/pkg/base/host" + "github.com/microsoft/wmi/pkg/base/instance" + "github.com/microsoft/wmi/pkg/base/query" + "github.com/microsoft/wmi/pkg/constant" + "github.com/microsoft/wmi/pkg/hardware/network/netadapter" +) + +// adapterInfo holds what one WMI query knows about a network adapter. +type adapterInfo struct { + Name string + Description string + Virtual bool + ConnectorPresent bool + Hardware bool +} + +// Every network change reads the adapter set, so one query serves a window. +const adapterCacheTTL = time.Minute + +var ( + adapterCacheMu sync.Mutex + adapterCache []adapterInfo + // The set before the last read still names the adapter that went away, + // so a removed adapter keeps its class in the journal. + adapterCachePrevious []adapterInfo + adapterCacheReadAt time.Time + + // Tests replace these seams to avoid a WMI query. + readAdaptersFn = readAdapters + adapterCacheNowFn = time.Now ) func patchNetIfaceName(iface *net.Interface) (bool, error) { @@ -14,3 +51,168 @@ func validInterface(iface *net.Interface, validIfacesMap map[string]struct{}) bo _, ok := validIfacesMap[iface.Name] return ok } + +// adapters returns the adapter set of the host, from the cache while it is +// younger than the window. The query runs under the mutex, so two callers +// start one query. +func adapters() []adapterInfo { + adapterCacheMu.Lock() + defer adapterCacheMu.Unlock() + now := adapterCacheNowFn() + if !adapterCacheReadAt.IsZero() && now.Sub(adapterCacheReadAt) < adapterCacheTTL { + return adapterCache + } + read := readAdaptersFn() + // A failing query waits for the window too, so it starts no query storm. + adapterCacheReadAt = now + if read == nil { + // Keep the last good read, so a failed query loses no adapter. + return adapterCache + } + adapterCachePrevious, adapterCache = adapterCache, read + return read +} + +// refreshInterfaceMeta drops the read window. The cache still describes the +// adapter that appeared or went away, and the next lookup reads the set. +func refreshInterfaceMeta() { + adapterCacheMu.Lock() + defer adapterCacheMu.Unlock() + adapterCacheReadAt = time.Time{} +} + +// tunnelAdapterWords name the VPN drivers of Windows. The description is the +// only signal, because WMI describes a tunnel adapter as an Ethernet adapter. +var tunnelAdapterWords = []string{"tap", "wintun", "wireguard", "vpn"} + +// adapterClass names the kind of an adapter. WMI marks a software adapter +// virtual and gives it no connector. +func adapterClass(adapter adapterInfo) string { + switch { + case tunnelAdapter(adapter.Description): + return "tunnel" + case adapter.Virtual && !adapter.ConnectorPresent: + return "virtual" + } + return "hardware" +} + +// tunnelAdapter reports a description that names a VPN driver. +func tunnelAdapter(description string) bool { + lower := strings.ToLower(description) + for _, word := range tunnelAdapterWords { + if strings.Contains(lower, word) { + return true + } + } + return false +} + +// platformInterfaceMeta returns the Windows class and adapter description of +// an interface. Windows has no service name. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + adapter, known := adapterByName(name) + if !known { + return "", "", "" + } + return adapterClass(adapter), adapter.Description, "" +} + +// adapterByName finds an adapter in the current set, or in the set before the +// last read. An adapter that went away keeps its names that way. +func adapterByName(name string) (adapterInfo, bool) { + if adapter, known := findAdapter(adapters(), name); known { + return adapter, true + } + adapterCacheMu.Lock() + defer adapterCacheMu.Unlock() + return findAdapter(adapterCachePrevious, name) +} + +func findAdapter(set []adapterInfo, name string) (adapterInfo, bool) { + for _, adapter := range set { + if adapter.Name == name { + return adapter, true + } + } + return adapterInfo{}, false +} + +// platformVirtualInterfaces returns the adapters that carry no traffic of the +// host network. +func platformVirtualInterfaces() map[string]struct{} { + virtual := make(map[string]struct{}) + for _, adapter := range adapters() { + if adapterClass(adapter) != "virtual" { + continue + } + virtual[adapter.Name] = struct{}{} + } + return virtual +} + +// readAdapters runs one WMI query for every network adapter. It returns nil +// when the query fails. +func readAdapters() []adapterInfo { + log.SetOutput(io.Discard) + defer log.SetOutput(os.Stderr) + whost := host.NewWmiLocalHost() + q := query.NewWmiQuery("MSFT_NetAdapter") + instances, err := instance.GetWmiInstancesFromHost(whost, string(constant.StadardCimV2), q) + if instances != nil { + defer instances.Close() + } + if err != nil { + mainLog.Load().Warn().Err(err).Msg("failed to get wmi network adapter") + return nil + } + var read []adapterInfo + for _, i := range instances { + adapter, err := netadapter.NewNetworkAdapter(i) + if err != nil { + mainLog.Load().Warn().Err(err).Msg("failed to get network adapter") + continue + } + info, ok := readAdapterInfo(adapter) + if !ok { + continue + } + read = append(read, info) + } + return read +} + +// readAdapterInfo reads the properties of one adapter. It reports false when a +// property that decides validity is unreadable. +func readAdapterInfo(adapter *netadapter.NetworkAdapter) (adapterInfo, bool) { + name, err := adapter.GetPropertyName() + if err != nil { + mainLog.Load().Warn().Err(err).Msg("failed to get interface name") + return adapterInfo{}, false + } + + // From: https://learn.microsoft.com/en-us/previous-versions/windows/desktop/legacy/hh968170(v=vs.85) + // + // "Indicates if a connector is present on the network adapter. This value is set to TRUE + // if this is a physical adapter or FALSE if this is not a physical adapter." + connectorPresent, err := adapter.GetPropertyConnectorPresent() + if err != nil { + mainLog.Load().Debug().Str("method", "readAdapterInfo").Str("interface", name).Msg("failed to get network adapter connector present property") + return adapterInfo{}, false + } + + // Check if it's a hardware interface. Checking only for connector present is not enough + // because some interfaces are not physical but have a connector. + hardware, err := adapter.GetPropertyHardwareInterface() + if err != nil { + mainLog.Load().Debug().Str("method", "readAdapterInfo").Str("interface", name).Msg("failed to get network adapter hardware interface property") + return adapterInfo{}, false + } + + info := adapterInfo{Name: name, ConnectorPresent: connectorPresent, Hardware: hardware} + // The class and the description describe the adapter only, so an + // unreadable one costs no validity. + info.Description, _ = adapter.GetPropertyInterfaceDescription() + info.Virtual, _ = adapter.GetPropertyVirtual() + return info, true +} diff --git a/cmd/cli/net_windows_test.go b/cmd/cli/net_windows_test.go index 551fe78..e52b52b 100644 --- a/cmd/cli/net_windows_test.go +++ b/cmd/cli/net_windows_test.go @@ -7,6 +7,7 @@ import ( "maps" "slices" "strings" + "sync" "testing" "time" @@ -45,3 +46,143 @@ func validInterfacesPowershell() []string { } return res } + +func Test_adapterClass(t *testing.T) { + tests := []struct { + name string + adapter adapterInfo + want string + }{ + {"virtual without a connector", adapterInfo{Virtual: true}, "virtual"}, + {"virtual with a connector", adapterInfo{Virtual: true, ConnectorPresent: true}, "hardware"}, + {"physical adapter", adapterInfo{ConnectorPresent: true, Hardware: true}, "hardware"}, + {"unknown adapter", adapterInfo{}, "hardware"}, + {"tap driver", adapterInfo{Description: "TAP-Windows Adapter V9", Virtual: true}, "tunnel"}, + {"wintun driver", adapterInfo{Description: "Wintun Userspace Tunnel", Virtual: true}, "tunnel"}, + {"wireguard driver", adapterInfo{Description: "WireGuard Tunnel #1", Virtual: true}, "tunnel"}, + {"vpn description", adapterInfo{Description: "Cisco AnyConnect VPN Adapter", ConnectorPresent: true}, "tunnel"}, + {"lower case description", adapterInfo{Description: "openvpn tap-windows6"}, "tunnel"}, + {"ethernet adapter", adapterInfo{Description: "Intel(R) Ethernet Connection I219-V", ConnectorPresent: true, Hardware: true}, "hardware"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := adapterClass(tc.adapter); got != tc.want { + t.Errorf("adapterClass(%+v) = %q, want %q", tc.adapter, got, tc.want) + } + }) + } +} + +// stubAdapters replaces the WMI and clock seams of the adapter cache, and +// restores them with the cache itself. +func stubAdapters(t *testing.T, read func() []adapterInfo, now func() time.Time) { + t.Helper() + readBefore, nowBefore := readAdaptersFn, adapterCacheNowFn + adapterCacheMu.Lock() + cacheBefore, readAtBefore := adapterCache, adapterCacheReadAt + adapterCache, adapterCacheReadAt = nil, time.Time{} + adapterCacheMu.Unlock() + t.Cleanup(func() { + readAdaptersFn, adapterCacheNowFn = readBefore, nowBefore + adapterCacheMu.Lock() + adapterCache, adapterCacheReadAt = cacheBefore, readAtBefore + adapterCacheMu.Unlock() + }) + readAdaptersFn, adapterCacheNowFn = read, now +} + +func Test_adapterCache(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubAdapters(t, func() []adapterInfo { + reads++ + return []adapterInfo{ + {Name: "Ethernet", Description: "Intel I219-V", ConnectorPresent: true, Hardware: true}, + {Name: "Hyper-V Switch", Description: "Hyper-V Virtual Switch", Virtual: true}, + } + }, func() time.Time { return now }) + + if got := adapters(); len(got) != 2 || got[0].Name != "Ethernet" { + t.Fatalf("adapters = %v, want the two fixture adapters", got) + } + now = now.Add(59 * time.Second) + adapters() + if reads != 1 { + t.Fatalf("reads within the cache window = %d, want 1", reads) + } + now = now.Add(2 * time.Second) + adapters() + if reads != 2 { + t.Fatalf("reads after the cache window = %d, want 2", reads) + } + refreshInterfaceMeta() + if reads != 2 { + t.Fatalf("reads for a refresh alone = %d, want 2", reads) + } + adapters() + if reads != 3 { + t.Fatalf("reads after a refresh = %d, want 3", reads) + } + + virtual := platformVirtualInterfaces() + if _, ok := virtual["Hyper-V Switch"]; !ok || len(virtual) != 1 { + t.Fatalf("platformVirtualInterfaces = %v, want the virtual switch only", virtual) + } + class, hardwarePort, service := platformInterfaceMeta("Ethernet") + if class != "hardware" || hardwarePort != "Intel I219-V" || service != "" { + t.Fatalf("platformInterfaceMeta = %q, %q, %q, want hardware, Intel I219-V, empty", class, hardwarePort, service) + } +} + +func Test_adapterCacheReadsOnceForTwoCallers(t *testing.T) { + now := time.Unix(1700000000, 0) + var mu sync.Mutex + reads := 0 + stubAdapters(t, func() []adapterInfo { + mu.Lock() + reads++ + mu.Unlock() + time.Sleep(10 * time.Millisecond) + return []adapterInfo{{Name: "Ethernet", Description: "Intel I219-V", ConnectorPresent: true, Hardware: true}} + }, func() time.Time { return now }) + + var wg sync.WaitGroup + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + adapters() + }() + } + wg.Wait() + + mu.Lock() + defer mu.Unlock() + if reads != 1 { + t.Fatalf("queries for four callers = %d, want 1", reads) + } +} + +func Test_adapterCacheKeepsTheRemovedAdapter(t *testing.T) { + now := time.Unix(1700000000, 0) + removed := false + stubAdapters(t, func() []adapterInfo { + read := []adapterInfo{{Name: "Ethernet", Description: "Intel I219-V", ConnectorPresent: true, Hardware: true}} + if removed { + return read + } + return append(read, adapterInfo{Name: "WireGuard", Description: "WireGuard Tunnel #1", Virtual: true}) + }, func() time.Time { return now }) + + adapters() + removed = true + refreshInterfaceMeta() + + class, hardwarePort, _ := platformInterfaceMeta("WireGuard") + if class != "tunnel" || hardwarePort != "WireGuard Tunnel #1" { + t.Fatalf("meta of the removed adapter = %q, %q, want tunnel, WireGuard Tunnel #1", class, hardwarePort) + } + if class, _, _ := platformInterfaceMeta("Absent"); class != "" { + t.Fatalf("meta of an adapter that no read named = %q, want empty", class) + } +} diff --git a/cmd/cli/network_delta.go b/cmd/cli/network_delta.go new file mode 100644 index 0000000..18ea6bd --- /dev/null +++ b/cmd/cli/network_delta.go @@ -0,0 +1,301 @@ +package cli + +import ( + "net" + "net/netip" + "slices" + "strings" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// interfaceChange describes one interface that a network delta changed. +type interfaceChange struct { + Name string + Action string + Class string + HardwarePort string + Service string + IPsBefore []string + IPsAfter []string + Flags string + MTU int + IsDefaultRoute bool +} + +// interfaceMeta holds the platform description of one interface. An empty +// Class asks the caller to read the class from the interface name. +type interfaceMeta struct { + Class string + HardwarePort string + Service string + LinkType string +} + +// interfaceMetaFunc reads the platform description of one interface. +type interfaceMetaFunc func(name string) interfaceMeta + +// virtualInterfaceSetFn names the virtual adapters of the host. Each platform +// file fills it, because a name alone does not tell a container bridge from a +// physical port. +var virtualInterfaceSetFn = func() map[string]struct{} { return nil } + +// interfaceSnapshot holds the part of one interface state that the diff +// compares. +type interfaceSnapshot struct { + present bool + up bool + mac string + mtu int + flags string + ips []string +} + +// interfaceClass groups an interface by the traffic it carries. The class +// decides which changes are noise. +func interfaceClass(name string, virtual bool) string { + switch { + case strings.HasPrefix(name, "lo"): + return "loopback" + case hasInterfacePrefix(name, "awdl", "llw"): + return "airdrop" + case hasInterfacePrefix(name, "utun", "tun", "tap", "wg", "ipsec", "ppp"): + return "tunnel" + case virtual: + return "virtual" + default: + return "hardware" + } +} + +// deltaBeforeState picks the state that the diff of one callback compares +// against, and makes a current callback the baseline of the next one. netmon +// caches major snapshots only, so every minor callback of an epoch carries the +// same Old, and a difference that lasts would look new in each of them. A +// callback that a newer snapshot replaced reads the baseline and leaves it. +// The caller holds networkSourceMu. +// +// netmon runs its callbacks in their own goroutines, so a minor callback of a +// new epoch can run before the major callback that opened the epoch. That +// major then reports its own changes against its Old and leaves the newer +// baseline in place. A major whose Old is the cache of the epoch before +// continues from the last reported state of that epoch, so a change that a +// minor callback reported does not appear again. +func (p *prog) deltaBeforeState(delta *netmon.ChangeDelta, epoch *netmon.State, current, major bool) *netmon.State { + before := p.networkDeltaState + switch { + case before == nil: + before = delta.Old + case current && major && p.networkDeltaEpoch == epoch: + return delta.Old + case p.networkDeltaEpoch != epoch && delta.Old != p.networkDeltaEpoch: + before = delta.Old + } + if current { + p.networkDeltaState, p.networkDeltaEpoch = delta.New, epoch + } + return before +} + +// diffNetworkDelta describes every interface that changed between two states. +// It walks every interface, not only the valid hardware ports, because the +// journal must show the port that went away too. +func diffNetworkDelta(beforeState, afterState *netmon.State) []interfaceChange { + before, after := networkStateOrEmpty(beforeState), networkStateOrEmpty(afterState) + virtual := virtualInterfaceSetFn() + var changes []interfaceChange + for _, name := range interfaceNameUnion(before, after) { + snapBefore, snapAfter := readInterfaceSnapshot(before, name), readInterfaceSnapshot(after, name) + action := interfaceAction(snapBefore, snapAfter) + if action == "" { + continue + } + known := snapAfter + if !snapAfter.present { + known = snapBefore + } + _, isVirtual := virtual[name] + changes = append(changes, interfaceChange{ + Name: name, + Action: action, + Class: interfaceClass(name, isVirtual), + IPsBefore: snapBefore.ips, + IPsAfter: snapAfter.ips, + Flags: known.flags, + MTU: known.mtu, + IsDefaultRoute: name == after.DefaultRouteInterface, + }) + } + return changes +} + +// noiseDelta reports a delta that no part of the daemon must act on. AirDrop +// and the virtual adapters of a container host change every few seconds, and +// each change would otherwise start the whole reconcile chain. A time jump +// means the host woke, so the daemon acts on the delta whatever it holds. +func noiseDelta(beforeState, afterState *netmon.State, timeJumped bool, changes []interfaceChange) bool { + if len(changes) == 0 || timeJumped { + return false + } + before, after := networkStateOrEmpty(beforeState), networkStateOrEmpty(afterState) + if before.DefaultRouteInterface != after.DefaultRouteInterface { + return false + } + if before.HaveV4 != after.HaveV4 || before.HaveV6 != after.HaveV6 { + return false + } + source4, source6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + for _, change := range changes { + if !noiseClass(change.Class) || change.IsDefaultRoute { + return false + } + if carriesHostTraffic(change.IPsBefore, source4, source6) { + return false + } + if carriesHostTraffic(change.IPsAfter, source4, source6) { + return false + } + } + return true +} + +// noiseClass names the classes that change on their own. The kernel moves the +// link-local address of an AirDrop or a container adapter every few seconds. +func noiseClass(class string) bool { + return class == "airdrop" || class == "virtual" +} + +// carriesHostTraffic reports an address list that reaches another host. A +// bridge, a bond, or a tunnel adapter can own the traffic of the host, so a +// routable address or the resolver source address makes the interface real +// whatever its name says. +func carriesHostTraffic(ips []string, source4, source6 net.IP) bool { + for _, ip := range ips { + prefix, err := netip.ParsePrefix(ip) + if err != nil { + continue + } + addr := prefix.Addr() + if addr.IsGlobalUnicast() || sameAddress(addr, source4) || sameAddress(addr, source6) { + return true + } + } + return false +} + +// sameAddress compares one interface address with a resolver source address. +func sameAddress(addr netip.Addr, ip net.IP) bool { + source, ok := netip.AddrFromSlice(ip) + return ok && source.Unmap() == addr +} + +// describeInterfaceChanges fills the platform fields of the changes that reach +// the journal. macOS reads them with two subprocesses, so only a delta that +// the daemon acts on pays for them. The platform class wins where it knows +// one, because a name rule cannot see a virtual adapter on Windows. +func describeInterfaceChanges(changes []interfaceChange, meta interfaceMetaFunc) { + if meta == nil { + return + } + for i := range changes { + info := meta(changes[i].Name) + changes[i].HardwarePort, changes[i].Service = info.HardwarePort, info.Service + if info.Class != "" { + changes[i].Class = info.Class + } + } +} + +// interfaceAction names the first difference that the journal reports. The +// order runs from the coarse change to the fine one, so one interface gives +// one action. +func interfaceAction(before, after interfaceSnapshot) string { + switch { + case before.present && !after.present: + return "removed" + case !before.present && after.present: + return "added" + case before.up && !after.up: + return "down" + case !before.up && after.up: + return "up" + case hasExtraEntry(before.ips, after.ips): + return "ip_removed" + case hasExtraEntry(after.ips, before.ips): + return "ip_added" + case before.mac != after.mac: + return "mac_changed" + case before.mtu != after.mtu: + return "mtu_changed" + default: + return "" + } +} + +func readInterfaceSnapshot(state *netmon.State, name string) interfaceSnapshot { + snapshot := interfaceSnapshot{ips: sortedPrefixStrings(state.InterfaceIPs[name])} + iface, ok := state.Interface[name] + if !ok { + return snapshot + } + snapshot.present = true + if iface.Interface == nil { + return snapshot + } + snapshot.up = iface.IsUp() + snapshot.mtu = iface.MTU + snapshot.flags = iface.Flags.String() + if len(iface.HardwareAddr) > 0 { + snapshot.mac = iface.HardwareAddr.String() + } + return snapshot +} + +func interfaceNameUnion(before, after *netmon.State) []string { + names := make([]string, 0, len(before.Interface)+len(after.Interface)) + for name := range before.Interface { + names = append(names, name) + } + for name := range after.Interface { + if _, both := before.Interface[name]; !both { + names = append(names, name) + } + } + slices.Sort(names) + return names +} + +func sortedPrefixStrings(prefixes []netip.Prefix) []string { + ips := prefixStrings(prefixes) + slices.Sort(ips) + return ips +} + +// networkStateOrEmpty keeps the diff free of nil checks. The first delta after +// start carries no old state. +func networkStateOrEmpty(state *netmon.State) *netmon.State { + if state != nil { + return state + } + return &netmon.State{} +} + +func hasInterfacePrefix(name string, prefixes ...string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} + +// hasExtraEntry reports an entry of list that other does not hold. +func hasExtraEntry(list, other []string) bool { + for _, entry := range list { + if !slices.Contains(other, entry) { + return true + } + } + return false +} diff --git a/cmd/cli/network_delta_test.go b/cmd/cli/network_delta_test.go new file mode 100644 index 0000000..4b847eb --- /dev/null +++ b/cmd/cli/network_delta_test.go @@ -0,0 +1,489 @@ +package cli + +import ( + "context" + "net" + "net/netip" + "slices" + "testing" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// deltaTestInterface describes one interface of a test network state. +type deltaTestInterface struct { + name string + up bool + mac string + mtu int + ips []string +} + +var ( + deltaTestUpFlags = net.FlagUp | net.FlagBroadcast | net.FlagMulticast + deltaTestDownFlags = net.Flags(0) +) + +func deltaTestState(t *testing.T, route string, ifaces ...deltaTestInterface) *netmon.State { + t.Helper() + state := &netmon.State{ + DefaultRouteInterface: route, + Interface: map[string]netmon.Interface{}, + InterfaceIPs: map[string][]netip.Prefix{}, + } + for _, iface := range ifaces { + flags := deltaTestDownFlags + if iface.up { + flags = deltaTestUpFlags + } + var mac net.HardwareAddr + if iface.mac != "" { + parsed, err := net.ParseMAC(iface.mac) + if err != nil { + t.Fatalf("ParseMAC(%q) error: %v", iface.mac, err) + } + mac = parsed + } + state.Interface[iface.name] = netmon.Interface{Interface: &net.Interface{ + Name: iface.name, + Flags: flags, + MTU: iface.mtu, + HardwareAddr: mac, + }} + for _, ip := range iface.ips { + state.InterfaceIPs[iface.name] = append(state.InterfaceIPs[iface.name], netip.MustParsePrefix(ip)) + } + } + return state +} + +func sameInterfaceChange(got, want interfaceChange) bool { + return got.Name == want.Name && + got.Action == want.Action && + got.Class == want.Class && + got.HardwarePort == want.HardwarePort && + got.Service == want.Service && + slices.Equal(got.IPsBefore, want.IPsBefore) && + slices.Equal(got.IPsAfter, want.IPsAfter) && + got.Flags == want.Flags && + got.MTU == want.MTU && + got.IsDefaultRoute == want.IsDefaultRoute +} + +func assertInterfaceChanges(t *testing.T, got, want []interfaceChange) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("changes = %+v, want %+v", got, want) + } + for i := range want { + if !sameInterfaceChange(got[i], want[i]) { + t.Errorf("change %d = %+v, want %+v", i, got[i], want[i]) + } + } +} + +func Test_diffNetworkDelta(t *testing.T) { + en0 := deltaTestInterface{name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}} + awdl0 := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:02", mtu: 1484, ips: []string{"fe80::1/64"}} + awdl0NewMAC := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:03", mtu: 1484, ips: []string{"fe80::1/64"}} + en5 := deltaTestInterface{name: "en5", up: true, mac: "aa:bb:cc:dd:ee:05", mtu: 1500, ips: []string{"192.0.2.50/24"}} + + for _, tt := range []struct { + name string + routeOld string + routeNew string + nilBefore bool + before []deltaTestInterface + after []deltaTestInterface + meta interfaceMetaFunc + want []interfaceChange + }{ + { + name: "airdrop_mac_change_with_link_local_only", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + want: []interfaceChange{{ + Name: "awdl0", Action: "mac_changed", Class: "airdrop", + IPsBefore: []string{"fe80::1/64"}, IPsAfter: []string{"fe80::1/64"}, + Flags: deltaTestUpFlags.String(), MTU: 1484, + }}, + }, + { + name: "airdrop_mac_and_link_local_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, { + name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:03", mtu: 1484, ips: []string{"fe80::2/64"}, + }}, + want: []interfaceChange{{ + Name: "awdl0", Action: "ip_removed", Class: "airdrop", + IPsBefore: []string{"fe80::1/64"}, IPsAfter: []string{"fe80::2/64"}, + Flags: deltaTestUpFlags.String(), MTU: 1484, + }}, + }, + { + name: "hardware_address_added_with_airdrop_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "192.0.2.11/24"}, + }, awdl0NewMAC}, + want: []interfaceChange{ + { + Name: "awdl0", Action: "mac_changed", Class: "airdrop", + IPsBefore: []string{"fe80::1/64"}, IPsAfter: []string{"fe80::1/64"}, + Flags: deltaTestUpFlags.String(), MTU: 1484, + }, + { + Name: "en0", Action: "ip_added", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24", "192.0.2.11/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }, + }, + }, + { + name: "interface_removed", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, en5}, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en5", Action: "removed", Class: "hardware", + IPsBefore: []string{"192.0.2.50/24"}, IPsAfter: nil, + Flags: deltaTestUpFlags.String(), MTU: 1500, + }}, + }, + { + name: "interface_added", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{en0, en5}, + want: []interfaceChange{{ + Name: "en5", Action: "added", Class: "hardware", + IPsBefore: nil, IPsAfter: []string{"192.0.2.50/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, + }}, + }, + { + name: "address_removed", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "192.0.2.11/24"}, + }}, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en0", Action: "ip_removed", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24", "192.0.2.11/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "interface_down", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{{ + name: "en0", up: false, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, + }}, + want: []interfaceChange{{ + Name: "en0", Action: "down", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestDownFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "interface_up", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{{ + name: "en0", up: false, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, + }}, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en0", Action: "up", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "mtu_changed", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1400, ips: []string{"192.0.2.10/24"}, + }}, + want: []interfaceChange{{ + Name: "en0", Action: "mtu_changed", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1400, IsDefaultRoute: true, + }}, + }, + { + name: "default_route_moves_to_the_new_interface", + routeOld: "en0", routeNew: "en5", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{{ + name: "en0", up: false, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, + }, en5}, + want: []interfaceChange{ + { + Name: "en0", Action: "down", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestDownFlags.String(), MTU: 1500, + }, + { + Name: "en5", Action: "added", Class: "hardware", + IPsBefore: nil, IPsAfter: []string{"192.0.2.50/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }, + }, + }, + { + name: "nil_old_state_adds_every_interface", + routeNew: "en0", + nilBefore: true, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en0", Action: "added", Class: "hardware", + IPsBefore: nil, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "no_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0}, + want: nil, + }, + { + name: "meta_fills_the_platform_fields", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, {name: "utun3", up: true, mtu: 1400}}, + after: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1400, ips: []string{"192.0.2.10/24"}, + }, {name: "utun3", up: true, mtu: 1380}}, + meta: func(name string) interfaceMeta { + if name == "utun3" { + return interfaceMeta{Class: "virtual"} + } + return interfaceMeta{HardwarePort: "Wi-Fi", Service: "Wi-Fi"} + }, + want: []interfaceChange{ + { + Name: "en0", Action: "mtu_changed", Class: "hardware", + HardwarePort: "Wi-Fi", Service: "Wi-Fi", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1400, IsDefaultRoute: true, + }, + { + Name: "utun3", Action: "mtu_changed", Class: "virtual", + Flags: deltaTestUpFlags.String(), MTU: 1380, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + var before *netmon.State + if !tt.nilBefore { + before = deltaTestState(t, tt.routeOld, tt.before...) + } + changes := diffNetworkDelta(before, deltaTestState(t, tt.routeNew, tt.after...)) + describeInterfaceChanges(changes, tt.meta) + assertInterfaceChanges(t, changes, tt.want) + }) + } +} + +func Test_diffNetworkDeltaVirtualSet(t *testing.T) { + virtual := virtualInterfaceSetFn + t.Cleanup(func() { virtualInterfaceSetFn = virtual }) + virtualInterfaceSetFn = func() map[string]struct{} { return map[string]struct{}{"docker0": {}} } + + before := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1500, ips: []string{"192.0.2.10/24"}}, + deltaTestInterface{name: "docker0", up: true, mtu: 1500}, + ) + after := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1400, ips: []string{"192.0.2.10/24"}}, + deltaTestInterface{name: "docker0", up: true, mtu: 1400}, + ) + assertInterfaceChanges(t, diffNetworkDelta(before, after), []interfaceChange{ + { + Name: "docker0", Action: "mtu_changed", Class: "virtual", + Flags: deltaTestUpFlags.String(), MTU: 1400, + }, + { + Name: "en0", Action: "mtu_changed", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1400, IsDefaultRoute: true, + }, + }) +} + +func Test_interfaceClass(t *testing.T) { + for _, tt := range []struct { + name string + virtual bool + want string + }{ + {name: "lo0", want: "loopback"}, + {name: "awdl0", want: "airdrop"}, + {name: "llw0", want: "airdrop"}, + {name: "utun3", want: "tunnel"}, + {name: "tun0", want: "tunnel"}, + {name: "tap0", want: "tunnel"}, + {name: "wg0", want: "tunnel"}, + {name: "ipsec0", want: "tunnel"}, + {name: "ppp0", want: "tunnel"}, + {name: "docker0", virtual: true, want: "virtual"}, + {name: "docker0", want: "hardware"}, + {name: "en0", want: "hardware"}, + } { + t.Run(tt.name+"_"+tt.want, func(t *testing.T) { + if got := interfaceClass(tt.name, tt.virtual); got != tt.want { + t.Errorf("interfaceClass(%q, %v) = %q, want %q", tt.name, tt.virtual, got, tt.want) + } + }) + } +} + +func Test_noiseDelta(t *testing.T) { + en0 := deltaTestInterface{name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}} + en5 := deltaTestInterface{name: "en5", up: true, mac: "aa:bb:cc:dd:ee:05", mtu: 1500, ips: []string{"192.0.2.50/24"}} + awdl0 := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:02", mtu: 1484, ips: []string{"fe80::1/64"}} + awdl0NewMAC := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:03", mtu: 1484, ips: []string{"fe80::1/64"}} + en0Extra := deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "192.0.2.11/24"}, + } + + docker0 := deltaTestInterface{name: "docker0", up: true, mtu: 1500} + docker0NewMTU := deltaTestInterface{name: "docker0", up: true, mtu: 1400} + + for _, tt := range []struct { + name string + routeOld string + routeNew string + before []deltaTestInterface + after []deltaTestInterface + virtual []string + timeJumped bool + beforeV4, beforeV6 bool + afterV4, afterV6 bool + source4 string + want bool + }{ + { + name: "airdrop_only", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + want: true, + }, + { + name: "hardware_and_airdrop", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0Extra, awdl0NewMAC}, + want: false, + }, + { + name: "no_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0}, + want: false, + }, + { + name: "default_route_moved", + routeOld: "en0", routeNew: "en5", + before: []deltaTestInterface{en0, en5, awdl0}, + after: []deltaTestInterface{en0, en5, awdl0NewMAC}, + want: false, + }, + { + name: "virtual_only", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, docker0}, + after: []deltaTestInterface{en0, docker0NewMTU}, + virtual: []string{"docker0"}, + want: true, + }, + { + name: "a_time_jump_is_never_noise", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + timeJumped: true, + want: false, + }, + { + name: "the_default_route_runs_over_a_virtual_interface", + routeOld: "docker0", routeNew: "docker0", + before: []deltaTestInterface{en0, docker0}, + after: []deltaTestInterface{en0, docker0NewMTU}, + virtual: []string{"docker0"}, + want: false, + }, + { + name: "a_virtual_interface_holds_a_routable_address", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, {name: "docker0", up: true, mtu: 1500, ips: []string{"172.17.0.1/16"}}}, + after: []deltaTestInterface{en0, {name: "docker0", up: true, mtu: 1400, ips: []string{"172.17.0.1/16"}}}, + virtual: []string{"docker0"}, + want: false, + }, + { + name: "an_airdrop_interface_holds_the_resolver_source", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, {name: "awdl0", up: true, mtu: 1484, ips: []string{"169.254.10.5/16"}}}, + after: []deltaTestInterface{en0, {name: "awdl0", up: true, mtu: 1400, ips: []string{"169.254.10.5/16"}}}, + source4: "169.254.10.5", + want: false, + }, + { + name: "have_v4_flips", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + beforeV4: true, + want: false, + }, + { + name: "have_v6_flips", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + afterV6: true, + want: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + stubSnapshotVirtualSet(t, tt.virtual...) + stubResolverSource(t, tt.source4) + before := deltaTestState(t, tt.routeOld, tt.before...) + after := deltaTestState(t, tt.routeNew, tt.after...) + before.HaveV4, before.HaveV6 = tt.beforeV4, tt.beforeV6 + after.HaveV4, after.HaveV6 = tt.afterV4, tt.afterV6 + changes := diffNetworkDelta(before, after) + if got := noiseDelta(before, after, tt.timeJumped, changes); got != tt.want { + t.Errorf("noiseDelta = %v, want %v, changes %+v", got, tt.want, changes) + } + }) + } +} + +// stubResolverSource sets the address that the resolver sends its queries +// from. The tests own that global, so each case starts from a known source. +func stubResolverSource(t *testing.T, source4 string) { + t.Helper() + original4, original6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + t.Cleanup(func() { + ctrld.SetDefaultLocalIPv4(context.Background(), original4) + ctrld.SetDefaultLocalIPv6(context.Background(), original6) + }) + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP(source4)) + ctrld.SetDefaultLocalIPv6(context.Background(), nil) +} diff --git a/cmd/cli/network_events.go b/cmd/cli/network_events.go new file mode 100644 index 0000000..f73cc99 --- /dev/null +++ b/cmd/cli/network_events.go @@ -0,0 +1,167 @@ +package cli + +import ( + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// networkEventsNowFn is the clock of the network events. Tests replace it. +var networkEventsNowFn = time.Now + +const ( + hostWokeMessage = "Host woke" + + // noiseDeltaMessage names the deltas that no part of the daemon acts on. + noiseDeltaMessage = "Network delta noise" + + // interfaceChangedMessage names one interface that a delta changed. + interfaceChangedMessage = "Network interface changed" + + // networkTransitionMessage names the outcome of one delta. + networkTransitionMessage = "Network transition" + + // transitionOutcomeIgnored names a delta that left every usable interface + // as it was. + transitionOutcomeIgnored = "ignored" + + // transitionOutcomeSnapshotSuperseded names a delta whose snapshot netmon + // already replaced. A newer callback reports the same interfaces. + transitionOutcomeSnapshotSuperseded = "snapshot_superseded" +) + +// noteHostWoke reports one wake to the wake reporter. The caller passes the +// network of the wake, because the state that ctrld holds still describes the +// network before the sleep. netmon reports a time jump and measures no gap, so +// its report waits for the detector, and the journal gets one event per wake. +func (p *prog) noteHostWoke(source string, gap time.Duration, state *netmon.State) { + now := networkEventsNowFn() + // A resume moves the resolvers of the host, so every wake source starts the + // fast poll. + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(now) + } + p.wake.note(now, wakeReport{source: source, gap: gap, state: state}, logHostWoke) +} + +// logHostWoke writes the Host woke journal event of one report. +func logHostWoke(report wakeReport) { + gapKnown := report.gap > 0 + event := journal(mainLog.Load().Info()).Str("source", report.source).Bool("gap_known", gapKnown) + if gapKnown { + event.Int64("gap_ms", report.gap.Milliseconds()) + } + event.Str("default_route", defaultRouteOf(report.state)). + Int("interfaces_up", upInterfaceCount(report.state)). + Msg(hostWokeMessage) +} + +// noteNoiseDelta records one delta of a noise storm. AirDrop and the virtual +// adapters of a container host change every few seconds, so the journal keeps +// one line per window while the debug stream keeps them all. +func (p *prog) noteNoiseDelta(changes []interfaceChange) { + names := changedInterfaceNames(changes) + mainLog.Load().Debug().Strs("interfaces", names).Msg(noiseDeltaMessage) + if summary, emit := p.networkNoise.add(names, networkEventsNowFn()); emit { + logNoiseSummary(summary) + } +} + +// flushNoiseSummary closes the open noise window, because the delta that the +// daemon acts on ends the storm. +func (p *prog) flushNoiseSummary() { + if summary, emit := p.networkNoise.flush(networkEventsNowFn()); emit { + logNoiseSummary(summary) + } +} + +// logNoiseSummary writes one journal line for a window of noise deltas. +func logNoiseSummary(summary noiseSummary) { + journal(mainLog.Load().Info()). + Int("count", summary.Count). + Strs("interfaces", summary.Interfaces). + Str("first_at", summary.First.Format(time.RFC3339)). + Str("last_at", summary.Last.Format(time.RFC3339)). + Msg(noiseDeltaMessage) +} + +// logInterfaceChanges puts the interfaces of one transition in the journal. +// Support reads back the port that lost its address at the time of an outage. +// An AirDrop or a container adapter changes every few seconds, so the +// transition names it and the journal keeps the interfaces that carry the +// traffic of the host. +func logInterfaceChanges(transitionID uint64, changes []interfaceChange) { + for _, change := range changes { + if noiseClass(change.Class) { + continue + } + journal(mainLog.Load().Info()). + Uint64("transition_id", transitionID). + Str("interface", change.Name). + Str("action", change.Action). + Str("class", change.Class). + Str("hardware_port", change.HardwarePort). + Str("service", change.Service). + Strs("ips_before", change.IPsBefore). + Strs("ips_after", change.IPsAfter). + Str("flags", change.Flags). + Int("mtu", change.MTU). + Bool("is_default_route", change.IsDefaultRoute). + Msg(interfaceChangedMessage) + } +} + +// networkTransitionEvent picks the level of one transition line. An ignored +// delta and a superseded snapshot arrive many times per minute, so the journal +// keeps the outcomes that can change the resolver state only. +func networkTransitionEvent(outcome string) *ctrld.LogEvent { + if outcome == transitionOutcomeIgnored || outcome == transitionOutcomeSnapshotSuperseded { + return mainLog.Load().Debug() + } + return journal(mainLog.Load().Info()) +} + +// changedInterfaceNames names the interfaces of one delta, in the order of the +// diff. +func changedInterfaceNames(changes []interfaceChange) []string { + names := make([]string, 0, len(changes)) + for _, change := range changes { + names = append(names, change.Name) + } + return names +} + +// hasAddOrRemove reports a delta that changed the adapter set of the host. The +// platform names of an adapter come from a cache, and that cache holds nothing +// for an adapter that just appeared. +func hasAddOrRemove(changes []interfaceChange) bool { + for _, change := range changes { + if change.Action == "added" || change.Action == "removed" { + return true + } + } + return false +} + +func defaultRouteOf(state *netmon.State) string { + if state == nil { + return "" + } + return state.DefaultRouteInterface +} + +// upInterfaceCount counts the interfaces with the up flag. A netmon entry +// without an embedded interface counts as down. +func upInterfaceCount(state *netmon.State) int { + if state == nil { + return 0 + } + up := 0 + for _, iface := range state.Interface { + if iface.Interface != nil && iface.IsUp() { + up++ + } + } + return up +} diff --git a/cmd/cli/network_events_test.go b/cmd/cli/network_events_test.go new file mode 100644 index 0000000..20133bd --- /dev/null +++ b/cmd/cli/network_events_test.go @@ -0,0 +1,809 @@ +package cli + +import ( + "context" + "fmt" + "net" + "net/netip" + "slices" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// hostWokeTestState names the network that a wake event reports. +func hostWokeTestState() *netmon.State { + return &netmon.State{ + DefaultRouteInterface: "en0", + Interface: map[string]netmon.Interface{ + "en0": {Interface: &net.Interface{Name: "en0", Flags: net.FlagUp}}, + "en5": {Interface: &net.Interface{Name: "en5"}}, + "utun": {Interface: &net.Interface{Name: "utun", Flags: net.FlagUp}}, + }, + InterfaceIPs: map[string][]netip.Prefix{}, + } +} + +// hostWokeTestClock hands the wake events a clock that the test moves. +func hostWokeTestClock(t *testing.T, clock *time.Time) { + t.Helper() + original := networkEventsNowFn + t.Cleanup(func() { networkEventsNowFn = original }) + networkEventsNowFn = func() time.Time { return *clock } +} + +// hostWokeTestHold captures the hold timer of the wake reporter, so a test +// decides when the hold ends. +func hostWokeTestHold(p *prog) *[]func() { + var armed []func() + p.wake.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + return &armed +} + +// Test_noteHostWokeNetmonFirstWaitsForTheDetectorGap covers the order of the +// 2026-09-14 capture. netmon sees the time jump first and measures no gap, so +// its report waits for the detector, and the journal gets one event with the +// gap. +func Test_noteHostWokeNetmonFirstWaitsForTheDetectorGap(t *testing.T) { + logs := captureDebugMainLog(t) + clock := time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC) + hostWokeTestClock(t, &clock) + p := &prog{dnsConfig: newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil })} + p.logger.Store(mainLog.Load()) + armed := hostWokeTestHold(p) + + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if events := jsonLogEvents(t, logs, hostWokeMessage); len(events) != 0 { + t.Fatalf("got %d Host woke events before the hold ended, want 0", len(events)) + } + if got := p.dnsConfig.nextDelay(clock); got != dnsConfigFastInterval { + t.Fatalf("DNS configuration poll delay after a wake = %s, want %s", got, dnsConfigFastInterval) + } + clock = clock.Add(2 * time.Second) + p.noteHostWoke("detector", 90*time.Second, hostWokeTestState()) + for _, fire := range *armed { + fire() + } + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 { + t.Fatalf("got %d Host woke events, want one report with the gap of the detector", len(events)) + } + wantField(t, events[0], "source", "detector") + wantField(t, events[0], "gap_known", true) + wantField(t, events[0], "gap_ms", float64(90000)) + wantField(t, events[0], "default_route", "en0") + wantField(t, events[0], "interfaces_up", float64(2)) + wantField(t, events[0], "journal", true) + + clock = clock.Add(5 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if events = jsonLogEvents(t, logs, hostWokeMessage); len(events) != 1 { + t.Fatalf("got %d Host woke events, want no second report inside the window", len(events)) + } +} + +// Test_noteHostWokeNetmonAloneReportsAfterTheHold covers a host without the +// detector. The netmon report goes out without a gap when the hold ends. +func Test_noteHostWokeNetmonAloneReportsAfterTheHold(t *testing.T) { + logs := captureDebugMainLog(t) + clock := time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC) + hostWokeTestClock(t, &clock) + p := &prog{} + p.logger.Store(mainLog.Load()) + armed := hostWokeTestHold(p) + + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if len(*armed) != 1 { + t.Fatalf("holds armed = %d, want 1", len(*armed)) + } + (*armed)[0]() + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 { + t.Fatalf("got %d Host woke events after the hold, want 1", len(events)) + } + wantField(t, events[0], "source", "netmon") + wantField(t, events[0], "gap_known", false) + if _, measured := events[0]["gap_ms"]; measured { + t.Fatalf("the netmon report carries a gap that it cannot measure: %v", events[0]) + } +} + +// Test_noteHostWokeDetectorFirstReportsOneEvent covers the other order. The +// gap is known with the first report, so netmon adds nothing. +func Test_noteHostWokeDetectorFirstReportsOneEvent(t *testing.T) { + logs := captureDebugMainLog(t) + resume := time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC) + clock := resume + hostWokeTestClock(t, &clock) + p := &prog{} + p.logger.Store(mainLog.Load()) + armed := hostWokeTestHold(p) + + p.noteHostWoke("detector", 90*time.Second, hostWokeTestState()) + clock = resume.Add(2 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 || len(*armed) != 0 { + t.Fatalf("got %d Host woke events and %d holds inside the window, want 1 and 0", len(events), len(*armed)) + } + wantField(t, events[0], "source", "detector") + wantField(t, events[0], "gap_ms", float64(90000)) + + // netmon polls every 15 s, so the window has to outlast one poll. + clock = resume.Add(29 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if events = jsonLogEvents(t, logs, hostWokeMessage); len(events) != 1 || len(*armed) != 0 { + t.Fatalf("got %d Host woke events and %d holds, want one report for a wake that netmon reports late", len(events), len(*armed)) + } + + clock = resume.Add(31 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + for _, fire := range *armed { + fire() + } + if events = jsonLogEvents(t, logs, hostWokeMessage); len(events) != 2 { + t.Fatalf("got %d Host woke events after the window, want 2", len(events)) + } +} + +// networkEventsHarness drives the network callback on a clock that the test +// owns. Every host read stays behind a stub, so no test starts networksetup or +// reads the route table of the machine. +type networkEventsHarness struct { + prog *prog + logs *syncBuffer + now time.Time + reconciled int + ignored int +} + +func newNetworkEventsHarness(t *testing.T, valid ...string) *networkEventsHarness { + t.Helper() + sourceTestGlobals(t) + stubHeaderSnapshotSources(t) + stubSnapshotVirtualSet(t) + h := &networkEventsHarness{ + prog: &prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}, + logs: captureDebugMainLog(t), + now: time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC), + } + h.prog.logger.Store(mainLog.Load()) + oldNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return h.now } + t.Cleanup(func() { networkEventsNowFn = oldNow }) + validSet := make(map[string]struct{}, len(valid)) + for _, name := range valid { + validSet[name] = struct{}{} + } + networkChangeValidInterfacesFn = func(context.Context) map[string]struct{} { return validSet } + networkChangeReconcileFn = func(_ *prog, _ context.Context, _ uint64) { h.reconciled++ } + networkChangeIgnoredInterceptFn = func(_ *prog, _ *netmon.ChangeDelta, _ time.Time) { h.ignored++ } + return h +} + +func (h *networkEventsHarness) handle(before, after *netmon.State, timeJumped bool) { + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: before, New: after, TimeJumped: timeJumped}, true) +} + +// stubNetmonCache gives every callback of the harness one cache snapshot. +// netmon caches major snapshots only, so the epoch of a minor callback stays +// at that snapshot. +func (h *networkEventsHarness) stubNetmonCache(t *testing.T, cache *netmon.State) { + t.Helper() + oldCurrent, oldRead := networkChangeCurrentStateFn, readNetworkSourceStateFn + t.Cleanup(func() { networkChangeCurrentStateFn, readNetworkSourceStateFn = oldCurrent, oldRead }) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return cache } + readNetworkSourceStateFn = func() (*netmon.State, error) { return cache, nil } +} + +// handleMinor drives one minor callback. netmon puts the cached major snapshot +// in Old, whatever the callback before it reported. +func (h *networkEventsHarness) handleMinor(cache, after *netmon.State) { + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: netmon.NewStatic(), Old: cache, New: after}, false) +} + +// interfaceEventsFor counts the journal events of one interface. +func (h *networkEventsHarness) interfaceEventsFor(t *testing.T, name string) int { + t.Helper() + count := 0 + for _, event := range jsonLogEvents(t, h.logs, interfaceChangedMessage) { + if event["interface"] == name { + count++ + } + } + return count +} + +// eventsAtLevel selects the captured events of one message and one level. +func (h *networkEventsHarness) eventsAtLevel(t *testing.T, message, level string) []map[string]any { + t.Helper() + var selected []map[string]any + for _, event := range jsonLogEvents(t, h.logs, message) { + if event["level"] == level { + selected = append(selected, event) + } + } + return selected +} + +var networkEventsHardwareIface = deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, +} + +// networkEventsAddressChange returns a delta that moves the address of en0. +func networkEventsAddressChange(t *testing.T) (before, after *netmon.State) { + t.Helper() + return deltaTestState(t, "en0", networkEventsHardwareIface), + deltaTestState(t, "en0", deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.20/24"}, + }) +} + +// transitionEvent returns the one transition event of a callback. +func (h *networkEventsHarness) transitionEvent(t *testing.T) map[string]any { + t.Helper() + events := jsonLogEvents(t, h.logs, networkTransitionMessage) + if len(events) != 1 { + t.Fatalf("got %d %q events, want 1", len(events), networkTransitionMessage) + } + return events[0] +} + +// wantNoField fails when an event carries a field that it must not. +func wantNoField(t *testing.T, event map[string]any, field string) { + t.Helper() + if value, carried := event[field]; carried { + t.Fatalf("field %q: got %v, want no field", field, value) + } +} + +// noiseDeltaStates returns a delta that moves the AirDrop link-local address. +func noiseDeltaStates(t *testing.T, step int) (before, after *netmon.State) { + t.Helper() + airdrop := func(address int) deltaTestInterface { + return deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{fmt.Sprintf("fe80::%d/64", address)}} + } + return deltaTestState(t, "en0", networkEventsHardwareIface, airdrop(step)), + deltaTestState(t, "en0", networkEventsHardwareIface, airdrop(step+1)) +} + +func wantStringsField(t *testing.T, event map[string]any, field string, want ...string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("field %q: got %v, want a list", field, event[field]) + } + got := make([]string, 0, len(values)) + for _, value := range values { + text, isText := value.(string) + if !isText { + t.Fatalf("field %q: got %v, want strings", field, event[field]) + } + got = append(got, text) + } + if !slices.Equal(got, want) { + t.Fatalf("field %q: got %v, want %v", field, got, want) + } +} + +func Test_networkEventsNoiseDeltaSkipsTheHandler(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + + before, after := noiseDeltaStates(t, 1) + h.handle(before, after, false) + + if h.reconciled != 0 || h.ignored != 0 { + t.Fatalf("noise delta ran the handler: reconciled=%d ignored=%d, want 0/0", h.reconciled, h.ignored) + } + debugLines := h.eventsAtLevel(t, noiseDeltaMessage, "debug") + if len(debugLines) != 1 { + t.Fatalf("got %d debug %q lines, want 1", len(debugLines), noiseDeltaMessage) + } + wantStringsField(t, debugLines[0], "interfaces", "awdl0") + if events := jsonLogEvents(t, h.logs, networkTransitionMessage); len(events) != 0 { + t.Fatalf("got %d %q events for a noise delta, want 0", len(events), networkTransitionMessage) + } +} + +func Test_networkEventsNoiseStormEmitsOneSummaryPerWindow(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + start := h.now + + for step := 0; step <= 10; step++ { + h.now = start.Add(time.Duration(step) * time.Minute) + before, after := noiseDeltaStates(t, step) + h.handle(before, after, false) + } + + summaries := h.eventsAtLevel(t, noiseDeltaMessage, "info") + if len(summaries) != 2 { + t.Fatalf("got %d journal %q events, want 2", len(summaries), noiseDeltaMessage) + } + wantField(t, summaries[0], "journal", true) + wantField(t, summaries[0], "count", float64(1)) + wantField(t, summaries[1], "count", float64(10)) + wantField(t, summaries[1], "first_at", start.Add(time.Minute).Format(time.RFC3339)) + wantField(t, summaries[1], "last_at", start.Add(10*time.Minute).Format(time.RFC3339)) + wantStringsField(t, summaries[1], "interfaces", "awdl0") +} + +func Test_networkEventsLogsEveryChangedInterface(t *testing.T) { + h := newNetworkEventsHarness(t, "en0", "en5") + before := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1500, ips: []string{"192.0.2.10/24", "2001:db8::10/64"}}, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.10/24"}}, + ) + before.HaveV4, before.HaveV6 = true, true + after := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1500, ips: []string{"192.0.2.20/24"}}, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.20/24"}}, + ) + after.HaveV4, after.HaveV6 = true, false + + h.handle(before, after, false) + + changes := jsonLogEvents(t, h.logs, interfaceChangedMessage) + if len(changes) != 2 { + t.Fatalf("got %d %q events, want 2", len(changes), interfaceChangedMessage) + } + for index, name := range []string{"en0", "en5"} { + wantField(t, changes[index], "journal", true) + wantField(t, changes[index], "level", "info") + wantField(t, changes[index], "interface", name) + wantField(t, changes[index], "action", "ip_removed") + wantField(t, changes[index], "class", "hardware") + wantField(t, changes[index], "transition_id", float64(1)) + wantField(t, changes[index], "mtu", float64(1500)) + } + wantField(t, changes[0], "is_default_route", true) + wantField(t, changes[1], "is_default_route", false) + wantStringsField(t, changes[0], "ips_before", "192.0.2.10/24", "2001:db8::10/64") + wantStringsField(t, changes[0], "ips_after", "192.0.2.20/24") + + transitions := jsonLogEvents(t, h.logs, networkTransitionMessage) + if len(transitions) != 1 { + t.Fatalf("got %d %q events, want 1", len(transitions), networkTransitionMessage) + } + wantField(t, transitions[0], "journal", true) + wantField(t, transitions[0], "level", "info") + wantField(t, transitions[0], "outcome", "accepted") + wantField(t, transitions[0], "time_jumped", false) + wantField(t, transitions[0], "have_v4_before", true) + wantField(t, transitions[0], "have_v4_after", true) + wantField(t, transitions[0], "have_v6_before", true) + wantField(t, transitions[0], "have_v6_after", false) + wantStringsField(t, transitions[0], "changed_interfaces", "en0", "en5") + if h.reconciled != 1 { + t.Fatalf("reconciled = %d, want 1", h.reconciled) + } + if snapshots := jsonLogEvents(t, h.logs, networkSnapshotMessage); len(snapshots) != 1 { + t.Fatalf("got %d %q events, want 1", len(snapshots), networkSnapshotMessage) + } +} + +func Test_networkEventsTimeJumpReportsTheWake(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + armed := hostWokeTestHold(h.prog) + before, after := noiseDeltaStates(t, 1) + + h.handle(before, after, true) + for _, fire := range *armed { + fire() + } + + wakes := jsonLogEvents(t, h.logs, hostWokeMessage) + if len(wakes) != 1 { + t.Fatalf("got %d %q events, want 1", len(wakes), hostWokeMessage) + } + wantField(t, wakes[0], "journal", true) + wantField(t, wakes[0], "source", "netmon") + wantField(t, wakes[0], "gap_known", false) + wantField(t, wakes[0], "default_route", "en0") + if _, measured := wakes[0]["gap_ms"]; measured { + t.Fatalf("the netmon report carries a gap that it cannot measure: %v", wakes[0]) + } +} + +func Test_networkEventsStoreTheIPv6Flag(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + // The one-time probe wins over the first store, so consume it first. + available := ctrld.HasIPv6(context.Background()) + t.Cleanup(func() { ctrld.SetIPv6Available(available) }) + ctrld.SetIPv6Available(true) + + before := deltaTestState(t, "en0", networkEventsHardwareIface) + before.HaveV6 = true + after := deltaTestState(t, "en0", deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.20/24"}, + }) + after.HaveV6 = false + + h.handle(before, after, false) + + if ctrld.HasIPv6(context.Background()) { + t.Fatal("HasIPv6 = true after a callback with have_v6 false, want false") + } +} + +// Test_networkEventsMinorDeltaDiffsAgainstTheStateBeforeIt covers a difference +// that lasts. The cached major snapshot holds no link-local address of en0, so +// a diff against the cache reports the same change in every minor callback. +func Test_networkEventsMinorDeltaDiffsAgainstTheStateBeforeIt(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + en0WithLinkLocal := deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "fe80::a/64"}, + } + airdrop := func(address int) deltaTestInterface { + return deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{fmt.Sprintf("fe80::%d/64", address)}} + } + cache := deltaTestState(t, "en0", networkEventsHardwareIface, airdrop(1)) + h.stubNetmonCache(t, cache) + + h.handleMinor(cache, deltaTestState(t, "en0", en0WithLinkLocal, airdrop(2))) + h.handleMinor(cache, deltaTestState(t, "en0", en0WithLinkLocal, airdrop(3))) + + if got := h.interfaceEventsFor(t, "en0"); got != 1 { + t.Fatalf("got %d %q events for en0, want 1", got, interfaceChangedMessage) + } + if noise := h.eventsAtLevel(t, noiseDeltaMessage, "debug"); len(noise) != 1 { + t.Fatalf("got %d debug %q lines, want 1 for the second delta", len(noise), noiseDeltaMessage) + } +} + +// Test_networkEventsNoiseDeltaReadsNoPlatformMeta pins the class-first order. +// The macOS meta read starts two subprocesses per call, so a storm of noise +// deltas must not reach it. +func Test_networkEventsNoiseDeltaReadsNoPlatformMeta(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + reads := countPlatformMetaReads(t) + + before, after := noiseDeltaStates(t, 1) + h.handle(before, after, false) + + if *reads != 0 { + t.Fatalf("a noise delta read the platform meta %d times, want 0", *reads) + } +} + +// countPlatformMetaReads counts the platform meta reads of a test. One read +// starts two subprocesses on macOS, so the count is the cost of a delta. +func countPlatformMetaReads(t *testing.T) *int { + t.Helper() + reads := 0 + original := platformInterfaceMetaFn + t.Cleanup(func() { platformInterfaceMetaFn = original }) + platformInterfaceMetaFn = func(string) (class, hardwarePort, service string) { + reads++ + return "", "", "" + } + return &reads +} + +// Test_networkEventsNoiseStormEndsWithTheNextTransition covers the last window +// of a storm. The count of the deltas that a real change follows must not wait +// for the next storm. +func Test_networkEventsNoiseStormEndsWithTheNextTransition(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + start := h.now + + for step := 0; step < 4; step++ { + h.now = start.Add(time.Duration(step) * time.Minute) + before, after := noiseDeltaStates(t, step) + h.handle(before, after, false) + } + h.now = start.Add(5 * time.Minute) + before, after := networkEventsAddressChange(t) + h.handle(before, after, false) + + summaries := h.eventsAtLevel(t, noiseDeltaMessage, "info") + if len(summaries) != 2 { + t.Fatalf("got %d journal %q events, want the first delta and the end of the storm", len(summaries), noiseDeltaMessage) + } + wantField(t, summaries[1], "count", float64(3)) + wantField(t, summaries[1], "first_at", start.Add(time.Minute).Format(time.RFC3339)) + wantField(t, summaries[1], "last_at", start.Add(3*time.Minute).Format(time.RFC3339)) + wantStringsField(t, summaries[1], "interfaces", "awdl0") +} + +// Test_networkEventsStoreTheNetworkBeforeTheHeaderRender proves that the +// callback stores its network under the source lock and renders the header +// after the unlock, so the header read of the hardware ports starts no +// process under the lock. +func Test_networkEventsStoreTheNetworkBeforeTheHeaderRender(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + lockHeld, renders := false, 0 + original := defaultRoutesFn + t.Cleanup(func() { defaultRoutesFn = original }) + // The header render reads the routes, so this seam runs where the + // callback renders the header. + defaultRoutesFn = func() (defaultRoute, defaultRoute) { + renders++ + if h.prog.networkSourceMu.TryLock() { + h.prog.networkSourceMu.Unlock() + } else { + lockHeld = true + } + return defaultRoute{Gateway: "192.0.2.1"}, defaultRoute{} + } + + before, after := networkEventsAddressChange(t) + h.handle(before, after, false) + + if renders == 0 { + t.Fatal("the callback rendered no header") + } + if lockHeld { + t.Fatal("the header render ran under the source lock") + } + if h.prog.lastNetworkState.Load() != after { + t.Fatal("the callback did not store its network") + } +} + +// Test_networkEventsIgnoredTransitionStaysOutOfTheJournal pins the level of an +// ignored delta. Such a delta arrives many times per minute, so it belongs in +// the debug stream and not in the journal. +func Test_networkEventsIgnoredTransitionStaysOutOfTheJournal(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + before := deltaTestState(t, "en0", networkEventsHardwareIface, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.10/24"}}) + after := deltaTestState(t, "en0", networkEventsHardwareIface, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.20/24"}}) + + h.handle(before, after, false) + + transition := h.transitionEvent(t) + wantField(t, transition, "outcome", transitionOutcomeIgnored) + wantField(t, transition, "level", "debug") + wantNoField(t, transition, "journal") + if snapshots := jsonLogEvents(t, h.logs, networkSnapshotMessage); len(snapshots) != 0 { + t.Fatalf("got %d %q events for an ignored delta, want 0", len(snapshots), networkSnapshotMessage) + } + if h.ignored != 1 || h.reconciled != 0 { + t.Fatalf("ignored delta: ignored=%d reconciled=%d, want 1/0", h.ignored, h.reconciled) + } +} + +// Test_networkEventsSupersededTransitionEntersTheJournal pins the level of a +// delta that a newer accepted delta replaced. Support reads it to tell a lost +// reconcile from one that never started. +func Test_networkEventsSupersededTransitionEntersTheJournal(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + h.prog.networkAcceptedGen.Store(5) + + before, after := networkEventsAddressChange(t) + h.handle(before, after, false) + + transition := h.transitionEvent(t) + wantField(t, transition, "outcome", "superseded") + wantField(t, transition, "level", "info") + wantField(t, transition, "journal", true) + if h.reconciled != 0 { + t.Fatalf("a superseded delta reconciled %d times, want 0", h.reconciled) + } +} + +// Test_networkEventsSupersededSnapshotReportsTheOutcome covers the delta whose +// snapshot netmon already replaced. The callback logged nothing at all, so the +// journal showed a silent gap between two deltas. +func Test_networkEventsSupersededSnapshotReportsTheOutcome(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + cache := deltaTestState(t, "en0", networkEventsHardwareIface) + h.stubNetmonCache(t, cache) + reads := countPlatformMetaReads(t) + stale, after := networkEventsAddressChange(t) + + // netmon holds a newer snapshot than the one that this callback carries. + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: netmon.NewStatic(), Old: stale, New: after}, false) + + transition := h.transitionEvent(t) + wantField(t, transition, "outcome", "snapshot_superseded") + wantField(t, transition, "level", "debug") + wantNoField(t, transition, "journal") + wantStringsField(t, transition, "changed_interfaces", "en0") + if events := jsonLogEvents(t, h.logs, interfaceChangedMessage); len(events) != 0 { + t.Fatalf("got %d %q events for a superseded snapshot, want 0", len(events), interfaceChangedMessage) + } + if h.reconciled != 0 || h.ignored != 0 { + t.Fatalf("superseded snapshot ran the handler: reconciled=%d ignored=%d, want 0/0", h.reconciled, h.ignored) + } + if *reads != 0 { + t.Fatalf("a superseded snapshot read the platform meta %d times, want 0", *reads) + } +} + +// Test_networkEventsLogsTheInterfacesOutsideTheNoiseClass covers a mixed +// delta. An AirDrop change beside a real one must not fill the journal, but +// the transition still names every interface that changed. +func Test_networkEventsLogsTheInterfacesOutsideTheNoiseClass(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + before := deltaTestState(t, "en0", networkEventsHardwareIface, + deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{"fe80::1/64"}}) + after := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.20/24"}}, + deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{"fe80::2/64"}}) + + h.handle(before, after, false) + + events := jsonLogEvents(t, h.logs, interfaceChangedMessage) + if len(events) != 1 { + t.Fatalf("got %d %q events, want one for the hardware port", len(events), interfaceChangedMessage) + } + wantField(t, events[0], "interface", "en0") + wantField(t, events[0], "class", "hardware") + wantStringsField(t, h.transitionEvent(t), "changed_interfaces", "awdl0", "en0") +} + +// reorderTestState builds a state with en0 on one address and awdl0 on one +// link-local address, so a test can replay callbacks out of order. +func reorderTestState(en0 string, awdl string) *netmon.State { + return &netmon.State{ + Interface: map[string]netmon.Interface{ + "en0": {Interface: &net.Interface{Name: "en0", Flags: net.FlagUp}}, + "awdl0": {Interface: &net.Interface{Name: "awdl0", Flags: net.FlagUp}}, + }, + InterfaceIPs: map[string][]netip.Prefix{ + "en0": {netip.MustParsePrefix(en0)}, + "awdl0": {netip.MustParsePrefix(awdl)}, + }, + DefaultRouteInterface: "en0", + HaveV4: true, + } +} + +// firstV4Of returns the first IPv4 address of one interface of a state. +func firstV4Of(state *netmon.State, name string) net.IP { + for _, prefix := range state.InterfaceIPs[name] { + if prefix.Addr().Is4() { + return net.IP(prefix.Addr().AsSlice()) + } + } + return nil +} + +// Test_networkEventsSupersededCallbackKeepsTheBaseline replays the callbacks +// out of order: B runs, a late A arrives while the monitor already caches C, +// then C runs. The late A must not become the baseline of C, or the address +// change from B to C looks like AirDrop noise and the stale source stays. +func Test_networkEventsSupersededCallbackKeepsTheBaseline(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + stateA := reorderTestState("192.0.2.20/24", "fe80::1/64") + stateB := reorderTestState("192.0.2.10/24", "fe80::1/64") + stateC := reorderTestState("192.0.2.20/24", "fe80::2/64") + cache := stateB + oldCurrent := networkChangeCurrentStateFn + t.Cleanup(func() { networkChangeCurrentStateFn = oldCurrent }) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return cache } + networkChangeReconcileFn = func(p *prog, _ context.Context, _ uint64) { + h.reconciled++ + ctrld.SetDefaultLocalIPv4(context.Background(), firstV4Of(p.networkSourceState, "en0")) + } + monitor := netmon.NewStatic() + + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: monitor, Old: stateA, New: stateB}, true) + cache = stateC + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: monitor, Old: &netmon.State{}, New: stateA}, true) + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: monitor, Old: stateB, New: stateC}, true) + + if h.reconciled != 2 { + t.Fatalf("reconciles = %d, want 2: the late callback must not hide the change from B to C", h.reconciled) + } + if got := ctrld.GetDefaultLocalIPv4().String(); got != "192.0.2.20" { + t.Fatalf("resolver source = %s, want 192.0.2.20", got) + } +} + +// orderTestState builds a state for the callback order tests: en0 on one +// address, plus extra interfaces with their addresses and their up flag. +func orderTestState(route, en0 string, extra map[string][]string, down map[string]bool) *netmon.State { + state := &netmon.State{ + Interface: map[string]netmon.Interface{}, + InterfaceIPs: map[string][]netip.Prefix{}, + DefaultRouteInterface: route, + HaveV4: true, + } + add := func(name string, up bool, ips []string) { + flags := net.Flags(0) + if up { + flags = net.FlagUp + } + state.Interface[name] = netmon.Interface{Interface: &net.Interface{Name: name, Flags: flags}} + for _, ip := range ips { + state.InterfaceIPs[name] = append(state.InterfaceIPs[name], netip.MustParsePrefix(ip)) + } + } + if en0 != "" { + add("en0", true, []string{en0}) + } + for name, ips := range extra { + add(name, !down[name], ips) + } + return state +} + +// Test_networkEventsMajorAfterItsMinorKeepsItsChanges runs a minor callback +// of the new epoch before the major callback that opened the epoch. netmon +// starts one goroutine per callback, so this order happens. The major must +// still report the address change and reconcile. +func Test_networkEventsMajorAfterItsMinorKeepsItsChanges(t *testing.T) { + for _, order := range []string{"in_order", "minor_first", "minor_first_time_jumped"} { + t.Run(order, func(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + hostWokeTestHold(h.prog) + s0 := orderTestState("en0", "192.0.2.10/24", map[string][]string{"awdl0": {"fe80::1/64"}}, nil) + s2 := orderTestState("en0", "192.0.2.20/24", map[string][]string{"awdl0": {"fe80::1/64"}}, nil) + s3 := orderTestState("en0", "192.0.2.20/24", map[string][]string{"awdl0": {"fe80::2/64"}}, nil) + h.stubNetmonCache(t, s2) + monitor := netmon.NewStatic() + major := &netmon.ChangeDelta{Monitor: monitor, Old: s0, New: s2, TimeJumped: order == "minor_first_time_jumped"} + minor := &netmon.ChangeDelta{Monitor: monitor, Old: s2, New: s3} + if order == "in_order" { + h.prog.handleNetworkChange(context.Background(), major, true) + h.prog.handleNetworkChange(context.Background(), minor, false) + } else { + h.prog.handleNetworkChange(context.Background(), minor, false) + h.prog.handleNetworkChange(context.Background(), major, true) + } + + if h.reconciled != 1 { + t.Fatalf("reconciled = %d, want 1 for the move of en0 from .10 to .20", h.reconciled) + } + if got := h.interfaceEventsFor(t, "en0"); got != 1 { + t.Fatalf("en0 interface events = %d, want 1", got) + } + if h.prog.networkDeltaState != s3 { + t.Fatal("the baseline must stay at the newest state of the epoch") + } + }) + } +} + +// Test_networkEventsEpochChangeReportsAMinorChangeOnce closes an epoch with a +// major callback whose Old is the cache of the epoch. The interface that a +// minor callback of that epoch reported must not appear again. +func Test_networkEventsEpochChangeReportsAMinorChangeOnce(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + s0 := orderTestState("en0", "192.0.2.10/24", nil, nil) + s1 := orderTestState("en0", "192.0.2.10/24", map[string][]string{"ipsec0": {"10.0.0.2/32"}}, nil) + s2 := orderTestState("en0", "192.0.2.20/24", map[string][]string{"ipsec0": {"10.0.0.2/32"}}, nil) + cache := s0 + oldCurrent, oldRead := networkChangeCurrentStateFn, readNetworkSourceStateFn + t.Cleanup(func() { networkChangeCurrentStateFn, readNetworkSourceStateFn = oldCurrent, oldRead }) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return cache } + readNetworkSourceStateFn = func() (*netmon.State, error) { return cache, nil } + monitor := netmon.NewStatic() + + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: monitor, Old: s0, New: s1}, false) + cache = s2 + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: monitor, Old: s0, New: s2}, true) + + if got := h.interfaceEventsFor(t, "ipsec0"); got != 1 { + t.Fatalf("ipsec0 interface events = %d, want 1", got) + } + if got := h.interfaceEventsFor(t, "en0"); got != 1 { + t.Fatalf("en0 interface events = %d, want 1", got) + } +} + +// Test_networkEventsNewValidInterfaceReconciles plugs a valid interface in +// with its address in one delta while the other valid interface is down. The +// new interface is active, so the change must reconcile. +func Test_networkEventsNewValidInterfaceReconciles(t *testing.T) { + h := newNetworkEventsHarness(t, "en0", "en7") + before := orderTestState("", "", map[string][]string{"en0": nil}, map[string]bool{"en0": true}) + before.HaveV4 = false + after := orderTestState("en7", "", map[string][]string{"en0": nil, "en7": {"192.0.2.30/24"}}, map[string]bool{"en0": true}) + + h.handle(before, after, false) + + if h.reconciled != 1 { + event := h.transitionEvent(t) + t.Fatalf("reconciled = %d, want 1; outcome %v", h.reconciled, event["outcome"]) + } +} diff --git a/cmd/cli/network_snapshot.go b/cmd/cli/network_snapshot.go new file mode 100644 index 0000000..61cadaf --- /dev/null +++ b/cmd/cli/network_snapshot.go @@ -0,0 +1,301 @@ +package cli + +import ( + "cmp" + "net/netip" + "slices" + "strings" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" + "tailscale.com/net/tsaddr" +) + +// hardwarePortIPhoneUSB is the macOS port name of a phone that shares its +// connection over the cable. +const hardwarePortIPhoneUSB = "iPhone USB" + +// snapshotInterfaceLimit bounds the interface list of one snapshot. A +// container host holds hundreds of interfaces, and each entry costs bytes in +// every journal line and in every log file header. +const snapshotInterfaceLimit = 32 + +// v6GlobalPrefix holds the v6 addresses that reach the Internet. +var v6GlobalPrefix = netip.MustParsePrefix("2000::/3") + +// hotspotGatewayPrefix is the range that an iPhone hotspot hands out. A +// gateway inside it names a tethered host on every platform. +var hotspotGatewayPrefix = netip.MustParsePrefix("172.20.10.0/28") + +// ethernetPortWords name a wired port. macOS spells the same link in several +// ways, one per adapter family. +var ethernetPortWords = []string{"Ethernet", "LAN", "Thunderbolt"} + +// defaultRoute names the next hop and the interface of one default route. Both +// values come from the same entry of the route table, so a reader can trust +// the pair. +type defaultRoute struct { + Gateway string + Interface string +} + +// snapshotInterface is one interface of a network snapshot. +type snapshotInterface struct { + Name string + Class string + Up bool + IPs []string + HardwarePort string + Service string +} + +// networkSnapshot describes the host network at one moment. The journal event +// and the header of a log file render the same value. +type networkSnapshot struct { + DefaultRouteV4 string + DefaultRouteV6 string + GatewayV4 string + GatewayV6 string + HaveV4 bool + HaveV6 bool + Interfaces []snapshotInterface + InterfacesOmitted int + Resolvers []string + SourceIPv4 string + SourceIPv6 string + InterceptTarget string + BypassActive bool + RecoveryRunning bool + PFStabilizing bool + LinkType string + Tethered bool + CLATPresent bool + NAT64Prefix string + DNSLessTarget bool +} + +// snapshotInputs holds every value a snapshot needs. The caller reads them, so +// the builder stays free of locks and of system calls. +type snapshotInputs struct { + State *netmon.State + RouteV4 defaultRoute + RouteV6 defaultRoute + Resolvers []string + SourceIPv4 string + SourceIPv6 string + InterceptTarget string + BypassActive bool + RecoveryRunning bool + PFStabilizing bool + NAT64Prefix string + CLATPresent bool + Meta interfaceMetaFunc +} + +// buildNetworkSnapshot makes the snapshot of one moment. It is pure, so a +// caller can take the inputs under a lock and build outside it. +func buildNetworkSnapshot(in snapshotInputs) networkSnapshot { + snapshot := networkSnapshot{ + GatewayV4: in.RouteV4.Gateway, + GatewayV6: in.RouteV6.Gateway, + DefaultRouteV4: in.RouteV4.Interface, + DefaultRouteV6: in.RouteV6.Interface, + Resolvers: in.Resolvers, + SourceIPv4: in.SourceIPv4, + SourceIPv6: in.SourceIPv6, + InterceptTarget: in.InterceptTarget, + BypassActive: in.BypassActive, + RecoveryRunning: in.RecoveryRunning, + PFStabilizing: in.PFStabilizing, + NAT64Prefix: in.NAT64Prefix, + CLATPresent: in.CLATPresent, + DNSLessTarget: in.InterceptTarget != "", + } + + routeMeta := interfaceMeta{} + var routeAddresses []netip.Prefix + if in.State != nil { + meta := interfaceMetaWithClass(in.Meta) + routeMeta = meta(in.State.DefaultRouteInterface) + routeAddresses = in.State.InterfaceIPs[in.State.DefaultRouteInterface] + snapshot.HaveV4, snapshot.HaveV6 = in.State.HaveV4, in.State.HaveV6 + snapshot.Interfaces, snapshot.InterfacesOmitted = snapshotInterfaces(in.State, meta) + if snapshot.DefaultRouteV4 == "" { + snapshot.DefaultRouteV4 = defaultRouteV4Interface(in.State) + } + } + snapshot.LinkType = linkTypeFor(routeMeta.HardwarePort, routeMeta.Class) + snapshot.Tethered = tetheredNetwork(in.RouteV4.Gateway, routeMeta.HardwarePort, routeAddresses) + return snapshot +} + +// interfaceMetaWithClass fills the class that the platform metadata leaves +// empty. A host that names no port still tells a tunnel from a hardware port +// by the interface name. +func interfaceMetaWithClass(meta interfaceMetaFunc) interfaceMetaFunc { + return func(name string) interfaceMeta { + info := interfaceMeta{} + if meta != nil { + info = meta(name) + } + if info.Class == "" { + info.Class = interfaceClass(name, false) + } + return info + } +} + +// snapshotInterfaces lists the interfaces of the state, sorted by name, and +// counts the ones it leaves out. A reader compares two snapshots line by line, +// so the order must not move. An interface that is down and holds no address +// carries no traffic, and a container host holds hundreds of them. +func snapshotInterfaces(state *netmon.State, meta interfaceMetaFunc) (list []snapshotInterface, omitted int) { + list = make([]snapshotInterface, 0, len(state.Interface)) + for name, iface := range state.Interface { + up := iface.Interface != nil && iface.IsUp() + ips := prefixStrings(state.InterfaceIPs[name]) + if !up && len(ips) == 0 { + omitted++ + continue + } + info := meta(name) + list = append(list, snapshotInterface{ + Name: name, + Class: info.Class, + Up: up, + IPs: ips, + HardwarePort: info.HardwarePort, + Service: info.Service, + }) + } + slices.SortFunc(list, func(a, b snapshotInterface) int { + return cmp.Compare(a.Name, b.Name) + }) + if len(list) > snapshotInterfaceLimit { + omitted += len(list) - snapshotInterfaceLimit + list = list[:snapshotInterfaceLimit] + } + return list, omitted +} + +// defaultRouteV4Interface names the interface that the network monitor holds +// for the default route. It fills the v4 route of a platform that reads no +// route table. An interface without a usable v4 address carries no v4 traffic, +// whatever the monitor says. +func defaultRouteV4Interface(state *netmon.State) string { + route := state.DefaultRouteInterface + if route == "" { + return "" + } + for _, prefix := range state.InterfaceIPs[route] { + if usableV4(prefix.Addr().Unmap()) { + return route + } + } + return "" +} + +// globalAddress reports whether an address reaches another host. +func globalAddress(address netip.Addr) bool { + return address.IsGlobalUnicast() && !address.IsLoopback() && !address.IsLinkLocalUnicast() +} + +// usableV4 and usableV6 grade one address the way the network monitor grades +// it. A fresh read of the interfaces must report the same families as a +// monitor callback, or two snapshots of one moment disagree. The monitor +// counts a unique local address of the v6 family, because some networks reach +// the Internet from it. The range of the Tailscale mesh is the exception, +// because that mesh carries no Internet traffic. +func usableV4(address netip.Addr) bool { + return address.Is4() && globalAddress(address) +} + +func usableV6(address netip.Addr) bool { + if !address.Is6() { + return false + } + if v6GlobalPrefix.Contains(address) { + return true + } + return address.IsPrivate() && !tsaddr.TailscaleULARange().Contains(address) +} + +// tetheredNetwork reports whether the host runs on a phone connection. A +// tethered host pays for its traffic and loses the link on every phone move. +// A platform that reads no route table gives no gateway, so the address that +// the phone handed to the host tells the same story. +func tetheredNetwork(gatewayV4, hardwarePort string, routeAddresses []netip.Prefix) bool { + if hardwarePort == hardwarePortIPhoneUSB { + return true + } + if gateway, err := netip.ParseAddr(gatewayV4); err == nil && hotspotGatewayPrefix.Contains(gateway.Unmap()) { + return true + } + for _, prefix := range routeAddresses { + if hotspotGatewayPrefix.Contains(prefix.Addr().Unmap()) { + return true + } + } + return false +} + +// linkTypeFor names the link of the default route. The port name tells more +// than the interface name, which is only a number on macOS. +func linkTypeFor(hardwarePort, class string) string { + switch { + case hardwarePort == "Wi-Fi": + return "wifi" + case hardwarePort == hardwarePortIPhoneUSB: + return "usb_tether" + case ethernetPort(hardwarePort): + return "ethernet" + case class == "tunnel": + return "tunnel" + } + return "unknown" +} + +func ethernetPort(hardwarePort string) bool { + for _, word := range ethernetPortWords { + if strings.Contains(hardwarePort, word) { + return true + } + } + return false +} + +// snapshotDict renders the snapshot with the field names of the spec. The +// journal event and the header line share it, so both stay in one shape. +func snapshotDict(snapshot networkSnapshot) *ctrld.LogEvent { + interfaces := ctrld.Arr() + for _, iface := range snapshot.Interfaces { + interfaces = interfaces.Dict(ctrld.Dict(). + Str("name", iface.Name). + Str("class", iface.Class). + Bool("up", iface.Up). + Strs("ips", iface.IPs). + Str("hardware_port", iface.HardwarePort). + Str("service", iface.Service)) + } + return ctrld.Dict(). + Str("default_route_v4", snapshot.DefaultRouteV4). + Str("default_route_v6", snapshot.DefaultRouteV6). + Str("gateway_v4", snapshot.GatewayV4). + Str("gateway_v6", snapshot.GatewayV6). + Bool("have_v4", snapshot.HaveV4). + Bool("have_v6", snapshot.HaveV6). + Array("interfaces", interfaces). + Int("interfaces_omitted", snapshot.InterfacesOmitted). + Strs("resolvers", snapshot.Resolvers). + Str("source_ipv4", snapshot.SourceIPv4). + Str("source_ipv6", snapshot.SourceIPv6). + Str("intercept_target", snapshot.InterceptTarget). + Bool("bypass_active", snapshot.BypassActive). + Bool("recovery_running", snapshot.RecoveryRunning). + Bool("pf_stabilizing", snapshot.PFStabilizing). + Str("link_type", snapshot.LinkType). + Bool("tethered", snapshot.Tethered). + Bool("clat_present", snapshot.CLATPresent). + Str("nat64_prefix", snapshot.NAT64Prefix). + Bool("dns_less_target", snapshot.DNSLessTarget) +} diff --git a/cmd/cli/network_snapshot_prog.go b/cmd/cli/network_snapshot_prog.go new file mode 100644 index 0000000..22b1f43 --- /dev/null +++ b/cmd/cli/network_snapshot_prog.go @@ -0,0 +1,226 @@ +package cli + +import ( + "net" + "reflect" + "slices" + "sync" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// networkSnapshotMessage is the message of the snapshot event. A log tool +// selects every snapshot of a run by this message. +const networkSnapshotMessage = "Network snapshot" + +// snapshotTriggerStart names the snapshot of the start of a run. +const snapshotTriggerStart = "start" + +// snapshotMinInterval bounds the snapshots of a flapping network. +const snapshotMinInterval = 60 * time.Second + +// snapshotNowFn is the clock of the snapshot limit. A test replaces it. +var snapshotNowFn = time.Now + +// defaultRoutesFn reads the default route of each family. A test replaces it, +// because the route table of the host is not a fixture. +var defaultRoutesFn = defaultRoutes + +// platformInterfaceMetaFn reads the platform names of one interface. A test +// replaces it, because the ports of the host are not a fixture. +var platformInterfaceMetaFn = platformInterfaceMeta + +// The delta diff has to know the virtual adapters of the host, and only the +// platform file reads them. +func init() { + virtualInterfaceSetFn = platformVirtualInterfaces +} + +// interfaceMetaFor describes one interface for the snapshot and for the delta +// diff. +func interfaceMetaFor(name string) interfaceMeta { + platformClass, hardwarePort, service := platformInterfaceMetaFn(name) + meta := interfaceMeta{ + Class: interfaceClassFor(name, platformClass), + HardwarePort: hardwarePort, + Service: service, + } + meta.LinkType = linkTypeFor(hardwarePort, meta.Class) + return meta +} + +// interfaceClassFor picks the class of one interface. A name rule wins for the +// loopback, the AirDrop, and the tunnel interfaces, because Windows describes +// a tunnel adapter as an Ethernet adapter. +func interfaceClassFor(name, platformClass string) string { + if class := interfaceClass(name, false); class != "hardware" { + return class + } + if platformClass != "" { + return platformClass + } + if _, virtual := virtualInterfaceSetFn()[name]; virtual { + return "virtual" + } + return "hardware" +} + +// snapshotInputs reads the sources of one snapshot from the network that the +// last monitor callback reported. No callback ran before the first header of a +// run, so that header reads the network itself. +func (p *prog) snapshotInputs() snapshotInputs { + state := p.lastNetworkState.Load() + if state == nil { + state = freshNetworkState() + } + return p.snapshotInputsFor(state) +} + +// freshSnapshotInputs reads the sources of one snapshot from the network of +// this moment. Support reads the header of an upload to learn the network of +// the moment the user asked, which can differ from the last callback. +func (p *prog) freshSnapshotInputs() snapshotInputs { + return p.snapshotInputsFor(freshNetworkState()) +} + +// networkSnapshot describes the host network of this moment. +func (p *prog) networkSnapshot() networkSnapshot { + return buildNetworkSnapshot(p.snapshotInputs()) +} + +// logNetworkSnapshot puts the host network in the journal. Support reads the +// snapshot of each trigger to learn the network at the time of an outage. +func (p *prog) logNetworkSnapshot(trigger string) { + snapshot := p.networkSnapshot() + if !p.snapshotWrites.allow(trigger, snapshot, snapshotNowFn()) { + return + } + journal(mainLog.Load().Info()). + Str("trigger", trigger). + Dict("network", snapshotDict(snapshot)). + Msg(networkSnapshotMessage) +} + +// snapshotLimiter holds the snapshot that reached the journal last, so a burst +// of triggers writes one line. +type snapshotLimiter struct { + mu sync.Mutex + last networkSnapshot + written time.Time + seen bool +} + +// allow reports whether one snapshot reaches the journal, and records the +// snapshots that do. +func (s *snapshotLimiter) allow(trigger string, snapshot networkSnapshot, now time.Time) bool { + s.mu.Lock() + defer s.mu.Unlock() + if !s.permits(trigger, snapshot, now) { + return false + } + s.last, s.written, s.seen = snapshot, now, true + return true +} + +// permits decides one snapshot. The first snapshot of a run tells a reader +// where the run started. A snapshot equal to the one before it carries +// nothing. A new default route and a new resolver set explain an outage, so +// they pass at once. Every other snapshot waits out the interval, because a +// flapping interface writes some kilobytes per flap. +func (s *snapshotLimiter) permits(trigger string, snapshot networkSnapshot, now time.Time) bool { + if trigger == snapshotTriggerStart || !s.seen { + return true + } + if sameNetworkSnapshot(s.last, snapshot) { + return false + } + if routeOrResolversChanged(s.last, snapshot) { + return true + } + return now.Sub(s.written) >= snapshotMinInterval +} + +// sameNetworkSnapshot compares two snapshots field by field. The trigger is no +// part of a snapshot, so two triggers of one network give one line. +func sameNetworkSnapshot(before, after networkSnapshot) bool { + return reflect.DeepEqual(before, after) +} + +// routeOrResolversChanged names the changes that a reader of an outage needs +// at once. +func routeOrResolversChanged(before, after networkSnapshot) bool { + return before.DefaultRouteV4 != after.DefaultRouteV4 || + before.DefaultRouteV6 != after.DefaultRouteV6 || + !slices.Equal(before.Resolvers, after.Resolvers) +} + +// snapshotInputsFor reads every source of one snapshot. It takes no writer +// lock, so any path of the daemon can log a snapshot. +func (p *prog) snapshotInputsFor(state *netmon.State) snapshotInputs { + routeV4, routeV6 := defaultRoutesFn() + inputs := snapshotInputs{ + State: state, + RouteV4: routeV4, + RouteV6: routeV6, + Resolvers: ctrld.OsResolverNameservers(), + SourceIPv4: addressString(ctrld.GetDefaultLocalIPv4()), + SourceIPv6: addressString(ctrld.GetDefaultLocalIPv6()), + InterceptTarget: p.interceptTargetSnapshot(), + BypassActive: p.recoveryBypass.Load(), + RecoveryRunning: p.recoveryRunning.Load(), + PFStabilizing: p.pfStabilizing.Load(), + Meta: interfaceMetaFor, + } + if prefix, ok := p.dns64PrefixSnapshot(); ok { + inputs.NAT64Prefix = prefix.String() + } + if state != nil { + inputs.CLATPresent = dns64StateHasCLAT(state) + } + return inputs +} + +// freshNetworkState reads the interfaces of the host now. netmon fills the +// route and the families of a callback state, so a read of this package has to +// fill them itself. +func freshNetworkState() *netmon.State { + state, err := readNetworkSourceStateFn() + if err != nil || state == nil { + return nil + } + fresh := *state + if route, err := netmon.DefaultRouteInterface(); err == nil { + fresh.DefaultRouteInterface = route + } + fresh.HaveV4, fresh.HaveV6 = reachableAddressFamilies(&fresh) + return &fresh +} + +// reachableAddressFamilies reports which families the host can reach. It +// grades each address with the rule of the network monitor, so the first +// header of a run reports the same families as the first callback. +func reachableAddressFamilies(state *netmon.State) (haveV4, haveV6 bool) { + for name, iface := range state.Interface { + if iface.Interface == nil || !iface.IsUp() { + continue + } + for _, prefix := range state.InterfaceIPs[name] { + address := prefix.Addr().Unmap() + haveV4 = haveV4 || usableV4(address) + haveV6 = haveV6 || usableV6(address) + } + } + return haveV4, haveV6 +} + +// addressString names one source address. An address that ctrld does not hold +// leaves an empty field, because net.IP prints an empty value as the word for +// a nil address. +func addressString(ip net.IP) string { + if len(ip) == 0 { + return "" + } + return ip.String() +} diff --git a/cmd/cli/network_snapshot_prog_test.go b/cmd/cli/network_snapshot_prog_test.go new file mode 100644 index 0000000..881e568 --- /dev/null +++ b/cmd/cli/network_snapshot_prog_test.go @@ -0,0 +1,351 @@ +package cli + +import ( + "context" + "net" + "net/netip" + "reflect" + "slices" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// stubSnapshotPlatformMeta makes the platform names of each interface a +// fixture, so no test depends on the ports of the host. +func stubSnapshotPlatformMeta(t *testing.T, meta map[string]interfaceMeta) { + t.Helper() + orig := platformInterfaceMetaFn + t.Cleanup(func() { platformInterfaceMetaFn = orig }) + platformInterfaceMetaFn = func(name string) (class, hardwarePort, service string) { + info := meta[name] + return info.Class, info.HardwarePort, info.Service + } +} + +// stubSnapshotVirtualSet names the virtual adapters of the fixture host. +func stubSnapshotVirtualSet(t *testing.T, names ...string) { + t.Helper() + orig := virtualInterfaceSetFn + t.Cleanup(func() { virtualInterfaceSetFn = orig }) + virtual := make(map[string]struct{}, len(names)) + for _, name := range names { + virtual[name] = struct{}{} + } + virtualInterfaceSetFn = func() map[string]struct{} { return virtual } +} + +// stubSnapshotGateways replaces the route table read, because the gateways of +// the host are not a fixture. +func stubSnapshotGateways(t *testing.T, v4, v6 string) { + t.Helper() + stubSnapshotRoutes(t, defaultRoute{Gateway: v4}, defaultRoute{Gateway: v6}) +} + +// stubSnapshotRoutes replaces the route table read with one default route per +// family. +func stubSnapshotRoutes(t *testing.T, v4, v6 defaultRoute) { + t.Helper() + orig := defaultRoutesFn + t.Cleanup(func() { defaultRoutesFn = orig }) + defaultRoutesFn = func() (defaultRoute, defaultRoute) { return v4, v6 } +} + +// stubSnapshotDNS64 reports a dual-stack host, so the NAT64 check starts no +// discovery and touches no name server. +func stubSnapshotDNS64(t *testing.T) { + t.Helper() + orig := dns64NetworkClassFn + t.Cleanup(func() { dns64NetworkClassFn = orig }) + dns64NetworkClassFn = func() (hasUsableIPv4, hasCLAT bool, err error) { return true, false, nil } +} + +// stubSnapshotSources sets the local source addresses and puts the values of +// the host back when the test ends. +func stubSnapshotSources(t *testing.T, v4, v6 string) { + t.Helper() + origV4, origV6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + t.Cleanup(func() { + ctrld.SetDefaultLocalIPv4(context.Background(), origV4) + ctrld.SetDefaultLocalIPv6(context.Background(), origV6) + }) + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP(v4)) + ctrld.SetDefaultLocalIPv6(context.Background(), net.ParseIP(v6)) +} + +// tetheredSnapshotProg returns a prog on a phone hotspot over the cable. +func tetheredSnapshotProg(t *testing.T) *prog { + t.Helper() + stubSnapshotDNS64(t) + stubSnapshotGateways(t, "172.20.10.1", "") + stubSnapshotPlatformMeta(t, map[string]interfaceMeta{ + "en7": {HardwarePort: hardwarePortIPhoneUSB, Service: "iPhone USB"}, + }) + stubSnapshotVirtualSet(t) + stubSnapshotSources(t, "172.20.10.5", "") + + p := &prog{} + p.logger.Store(mainLog.Load()) + p.lastNetworkState.Store(&netmon.State{ + DefaultRouteInterface: "en7", + HaveV4: true, + Interface: map[string]netmon.Interface{ + "en7": {Interface: &net.Interface{Name: "en7", Flags: net.FlagUp}}, + "lo0": {Interface: &net.Interface{Name: "lo0", Flags: net.FlagUp}}, + }, + InterfaceIPs: map[string][]netip.Prefix{ + "en7": {netip.MustParsePrefix("172.20.10.5/28")}, + "lo0": {netip.MustParsePrefix("127.0.0.1/8")}, + }, + }) + p.recoveryBypass.Store(true) + p.pfStabilizing.Store(true) + p.interceptDNSTargetSetValue = "127.0.0.1" + p.publishInterceptTarget("127.0.0.1") + return p +} + +func Test_networkSnapshotFromTheStoredState(t *testing.T) { + p := tetheredSnapshotProg(t) + + snapshot := p.networkSnapshot() + + checks := []struct { + field string + got, want any + }{ + {"default_route_v4", snapshot.DefaultRouteV4, "en7"}, + {"default_route_v6", snapshot.DefaultRouteV6, ""}, + {"gateway_v4", snapshot.GatewayV4, "172.20.10.1"}, + {"have_v4", snapshot.HaveV4, true}, + {"source_ipv4", snapshot.SourceIPv4, "172.20.10.5"}, + {"source_ipv6", snapshot.SourceIPv6, ""}, + {"intercept_target", snapshot.InterceptTarget, "127.0.0.1"}, + {"dns_less_target", snapshot.DNSLessTarget, true}, + {"bypass_active", snapshot.BypassActive, true}, + {"recovery_running", snapshot.RecoveryRunning, false}, + {"pf_stabilizing", snapshot.PFStabilizing, true}, + {"link_type", snapshot.LinkType, "usb_tether"}, + {"tethered", snapshot.Tethered, true}, + {"nat64_prefix", snapshot.NAT64Prefix, ""}, + {"clat_present", snapshot.CLATPresent, false}, + } + for _, c := range checks { + if c.got != c.want { + t.Errorf("%s = %v, want %v", c.field, c.got, c.want) + } + } + + want := []snapshotInterface{ + {Name: "en7", Class: "hardware", Up: true, IPs: []string{"172.20.10.5/28"}, HardwarePort: hardwarePortIPhoneUSB, Service: "iPhone USB"}, + {Name: "lo0", Class: "loopback", Up: true, IPs: []string{"127.0.0.1/8"}}, + } + if !reflect.DeepEqual(snapshot.Interfaces, want) { + t.Errorf("interfaces = %+v, want %+v", snapshot.Interfaces, want) + } +} + +// Test_networkSnapshotReadsTheNetworkWithoutAStoredState covers the first +// header of a run. No network callback reached ctrld yet, so the snapshot must +// read the interfaces itself and name the families it finds. +func Test_networkSnapshotReadsTheNetworkWithoutAStoredState(t *testing.T) { + stubSnapshotDNS64(t) + stubSnapshotGateways(t, "192.0.2.1", "") + stubSnapshotPlatformMeta(t, nil) + stubSnapshotSources(t, "", "") + stubLogHeaderNetworkRead(t, "en9") + p := &prog{} + p.logger.Store(mainLog.Load()) + + snapshot := p.networkSnapshot() + + if len(snapshot.Interfaces) != 1 || snapshot.Interfaces[0].Name != "en9" || !snapshot.Interfaces[0].Up { + t.Fatalf("interfaces = %+v, want one up en9", snapshot.Interfaces) + } + if !snapshot.HaveV4 { + t.Error("have_v4 = false, want true for a global unicast v4 address") + } + if snapshot.HaveV6 { + t.Error("have_v6 = true, want false without a v6 address") + } +} + +// Test_reachableAddressFamiliesGradesTheV6AddressLikeNetmon covers a host on a +// unique local address. The network monitor counts such an address as usable, +// so a fresh read must count it too. +func Test_reachableAddressFamiliesGradesTheV6AddressLikeNetmon(t *testing.T) { + for _, tt := range []struct { + name string + ips []string + wantV6 bool + }{ + {name: "unique_local", ips: []string{"fd00::2/64"}, wantV6: true}, + {name: "tailscale_range", ips: []string{"fd7a:115c:a1e0::2/128"}}, + {name: "global", ips: []string{"2001:db8::2/64"}, wantV6: true}, + {name: "link_local", ips: []string{"fe80::2/64"}}, + } { + t.Run(tt.name, func(t *testing.T) { + state := snapshotTestState("en0", snapshotTestInterface{"en0", true, tt.ips}) + + _, haveV6 := reachableAddressFamilies(state) + + if haveV6 != tt.wantV6 { + t.Errorf("have_v6 = %v, want %v for %v", haveV6, tt.wantV6, tt.ips) + } + }) + } +} + +func Test_networkSnapshotLogsOneJournalLine(t *testing.T) { + logs := captureDebugMainLog(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("start") + + events := jsonLogEvents(t, logs, networkSnapshotMessage) + if len(events) != 1 { + t.Fatalf("%q lines = %d, want 1", networkSnapshotMessage, len(events)) + } + wantField(t, events[0], "trigger", "start") + wantField(t, events[0], journalField, true) + wantField(t, events[0], "level", "info") + network, ok := events[0]["network"].(map[string]any) + if !ok { + t.Fatalf("network = %v, want an object", events[0]["network"]) + } + wantField(t, network, "default_route_v4", "en7") + wantField(t, network, "tethered", true) + wantField(t, network, "link_type", "usb_tether") +} + +// snapshotTestClock hands out the time that a test sets, because the snapshot +// limit counts seconds between two lines. +type snapshotTestClock struct { + now time.Time +} + +// stubSnapshotClock puts the clock of the snapshot limit under the control of +// one test. +func stubSnapshotClock(t *testing.T) *snapshotTestClock { + t.Helper() + clock := &snapshotTestClock{now: time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC)} + orig := snapshotNowFn + t.Cleanup(func() { snapshotNowFn = orig }) + snapshotNowFn = func() time.Time { return clock.now } + return clock +} + +func (c *snapshotTestClock) advance(d time.Duration) { c.now = c.now.Add(d) } + +// snapshotTriggerLines names the trigger of every snapshot line, in order. +func snapshotTriggerLines(t *testing.T, logs *syncBuffer) []string { + t.Helper() + var triggers []string + for _, event := range jsonLogEvents(t, logs, networkSnapshotMessage) { + trigger, _ := event["trigger"].(string) + triggers = append(triggers, trigger) + } + return triggers +} + +// Test_logNetworkSnapshotSkipsAnEqualSnapshot covers a recovery on a network +// that did not change. The begin and the end of that recovery render the same +// object, so the second line carries nothing. +func Test_logNetworkSnapshotSkipsAnEqualSnapshot(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("recovery_begin") + clock.advance(5 * time.Minute) + p.logNetworkSnapshot("recovery_end") + + if got := snapshotTriggerLines(t, logs); !slices.Equal(got, []string{"recovery_begin"}) { + t.Errorf("snapshot triggers = %v, want the first line alone", got) + } +} + +// Test_logNetworkSnapshotCapsTheRate covers a flapping interface. Two +// snapshots that differ in no route and in no resolver give one line per +// minute. +func Test_logNetworkSnapshotCapsTheRate(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("transition") + clock.advance(30 * time.Second) + p.recoveryRunning.Store(true) + p.logNetworkSnapshot("transition") + clock.advance(30 * time.Second) + p.logNetworkSnapshot("transition") + + if got := snapshotTriggerLines(t, logs); len(got) != 2 { + t.Errorf("snapshot lines = %d, want 2: %v", len(got), got) + } +} + +// Test_logNetworkSnapshotWritesARouteChangeAtOnce covers the change that +// explains an outage. The cap must not hold it back. +func Test_logNetworkSnapshotWritesARouteChangeAtOnce(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("transition") + clock.advance(time.Second) + stubSnapshotRoutes(t, defaultRoute{Gateway: "192.168.1.1", Interface: "en0"}, defaultRoute{}) + p.logNetworkSnapshot("transition") + clock.advance(time.Second) + p.recoveryRunning.Store(true) + p.logNetworkSnapshot("recovery_begin") + + if got := snapshotTriggerLines(t, logs); !slices.Equal(got, []string{"transition", "transition"}) { + t.Errorf("snapshot triggers = %v, want the two transitions", got) + } +} + +// Test_logNetworkSnapshotAlwaysWritesTheStartLine covers the first line of a +// run. A reader needs the network of the start, whatever came before it. +func Test_logNetworkSnapshotAlwaysWritesTheStartLine(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("transition") + clock.advance(time.Second) + p.logNetworkSnapshot(snapshotTriggerStart) + + if got := snapshotTriggerLines(t, logs); !slices.Equal(got, []string{"transition", "start"}) { + t.Errorf("snapshot triggers = %v, want the transition and the start", got) + } +} + +func Test_interfaceMetaFor(t *testing.T) { + stubSnapshotVirtualSet(t, "bridge100") + stubSnapshotPlatformMeta(t, map[string]interfaceMeta{ + "lo0": {Class: "hardware"}, + "en0": {HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + "eth0": {Class: "hardware", HardwarePort: "Intel I219-V"}, + }) + + for _, tt := range []struct { + name string + want interfaceMeta + }{ + {"lo0", interfaceMeta{Class: "loopback", LinkType: "unknown"}}, + {"awdl0", interfaceMeta{Class: "airdrop", LinkType: "unknown"}}, + {"utun3", interfaceMeta{Class: "tunnel", LinkType: "tunnel"}}, + {"en0", interfaceMeta{Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi", LinkType: "wifi"}}, + {"bridge100", interfaceMeta{Class: "virtual", LinkType: "unknown"}}, + {"eth0", interfaceMeta{Class: "hardware", HardwarePort: "Intel I219-V", LinkType: "unknown"}}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := interfaceMetaFor(tt.name); got != tt.want { + t.Errorf("interfaceMetaFor(%q) = %+v, want %+v", tt.name, got, tt.want) + } + }) + } +} diff --git a/cmd/cli/network_snapshot_test.go b/cmd/cli/network_snapshot_test.go new file mode 100644 index 0000000..1c45180 --- /dev/null +++ b/cmd/cli/network_snapshot_test.go @@ -0,0 +1,558 @@ +package cli + +import ( + "encoding/json" + "fmt" + "net" + "net/netip" + "slices" + "strings" + "testing" + + "tailscale.com/net/netmon" +) + +// snapshotTestInterface names one interface of a test network state. +type snapshotTestInterface struct { + name string + up bool + ips []string +} + +// snapshotTestState builds a netmon state with several interfaces, because the +// shared helper of the transition tests holds one interface only. +func snapshotTestState(route string, interfaces ...snapshotTestInterface) *netmon.State { + state := &netmon.State{ + DefaultRouteInterface: route, + Interface: map[string]netmon.Interface{}, + InterfaceIPs: map[string][]netip.Prefix{}, + } + for _, iface := range interfaces { + flags := net.Flags(0) + if iface.up { + flags = net.FlagUp + } + state.Interface[iface.name] = netmon.Interface{Interface: &net.Interface{Name: iface.name, Flags: flags}} + for _, ip := range iface.ips { + state.InterfaceIPs[iface.name] = append(state.InterfaceIPs[iface.name], netip.MustParsePrefix(ip)) + } + } + return state +} + +func snapshotTestMeta(meta map[string]interfaceMeta) interfaceMetaFunc { + return func(name string) interfaceMeta { return meta[name] } +} + +func Test_tetheredNetwork(t *testing.T) { + for _, tt := range []struct { + name string + gatewayV4 string + hardwarePort string + hostIPs []string + want bool + }{ + {name: "hotspot_gateway", gatewayV4: "172.20.10.1", hardwarePort: "Wi-Fi", want: true}, + {name: "hotspot_range_end", gatewayV4: "172.20.10.15", hardwarePort: "Wi-Fi", want: true}, + {name: "above_hotspot_range", gatewayV4: "172.20.10.16", hardwarePort: "Wi-Fi"}, + {name: "below_hotspot_range", gatewayV4: "172.20.9.255", hardwarePort: "Wi-Fi"}, + {name: "home_gateway", gatewayV4: "192.168.1.1", hardwarePort: "Wi-Fi"}, + {name: "iphone_usb_port", gatewayV4: "192.168.1.1", hardwarePort: "iPhone USB", want: true}, + {name: "no_gateway_no_port"}, + {name: "unparsable_gateway", gatewayV4: "not-an-address"}, + {name: "v6_gateway", gatewayV4: "fe80::1", hardwarePort: "Ethernet"}, + {name: "hotspot_host_address_without_a_gateway", hostIPs: []string{"172.20.10.4/28"}, want: true}, + {name: "hotspot_host_address_of_a_second_address", hostIPs: []string{"fe80::1/64", "172.20.10.4/28"}, want: true}, + {name: "home_host_address_without_a_gateway", hostIPs: []string{"192.168.1.20/24"}}, + } { + t.Run(tt.name, func(t *testing.T) { + hostIPs := make([]netip.Prefix, 0, len(tt.hostIPs)) + for _, ip := range tt.hostIPs { + hostIPs = append(hostIPs, netip.MustParsePrefix(ip)) + } + if got := tetheredNetwork(tt.gatewayV4, tt.hardwarePort, hostIPs); got != tt.want { + t.Fatalf("tetheredNetwork(%q, %q, %v) = %v, want %v", tt.gatewayV4, tt.hardwarePort, tt.hostIPs, got, tt.want) + } + }) + } +} + +// Test_buildNetworkSnapshot_tetheredWithoutAGateway covers Linux and Windows. +// Both read no route table in this release, so the address of the host is the +// only sign of a phone hotspot. +func Test_buildNetworkSnapshot_tetheredWithoutAGateway(t *testing.T) { + state := snapshotTestState("wlan0", snapshotTestInterface{"wlan0", true, []string{"172.20.10.4/28"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + if !snapshot.Tethered { + t.Error("tethered = false, want true for a host address of the hotspot range") + } +} + +// Test_usableV6 pins the grade of a v6 address to the rule of the network +// monitor. A fresh read and a monitor callback must report the same have_v6. +func Test_usableV6(t *testing.T) { + for _, tt := range []struct { + address string + want bool + }{ + {"2001:db8::1", true}, + {"2606:4700:4700::1111", true}, + {"fd00::1", true}, + {"fd7a:115c:a1e0::1", false}, + {"fe80::1", false}, + {"::1", false}, + {"192.0.2.10", false}, + } { + t.Run(tt.address, func(t *testing.T) { + if got := usableV6(netip.MustParseAddr(tt.address).Unmap()); got != tt.want { + t.Errorf("usableV6(%q) = %v, want %v", tt.address, got, tt.want) + } + }) + } +} + +func Test_linkTypeFor(t *testing.T) { + for _, tt := range []struct { + name string + hardwarePort string + class string + want string + }{ + {"wifi_port", "Wi-Fi", "hardware", "wifi"}, + {"ethernet_port", "Ethernet", "hardware", "ethernet"}, + {"thunderbolt_port", "Thunderbolt Ethernet Slot 1", "hardware", "ethernet"}, + {"usb_lan_port", "USB 10/100/1000 LAN", "hardware", "ethernet"}, + {"thunderbolt_bridge", "Thunderbolt Bridge", "hardware", "ethernet"}, + {"iphone_usb_port", "iPhone USB", "hardware", "usb_tether"}, + {"tunnel_class", "", "tunnel", "tunnel"}, + // The Windows adapter description reaches linkTypeFor as the port + // name, and a wired word in it wins over the tunnel class. + {"ethernet_word_wins_over_tunnel_class", "SSL VPN Virtual Ethernet Adapter", "tunnel", "ethernet"}, + {"no_port_no_class", "", "", "unknown"}, + {"unknown_port", "Bluetooth PAN", "hardware", "unknown"}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := linkTypeFor(tt.hardwarePort, tt.class); got != tt.want { + t.Fatalf("linkTypeFor(%q, %q) = %q, want %q", tt.hardwarePort, tt.class, got, tt.want) + } + }) + } +} + +func Test_buildNetworkSnapshot_tetheredWiFiHotspot(t *testing.T) { + state := snapshotTestState("en0", snapshotTestInterface{"en0", true, []string{"172.20.10.2/28"}}) + state.HaveV4 = true + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "172.20.10.1", Interface: "en0"}, + Meta: snapshotTestMeta(map[string]interfaceMeta{"en0": {Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi"}}), + }) + + if !snapshot.Tethered { + t.Error("tethered = false, want true for the hotspot gateway") + } + if snapshot.LinkType != "wifi" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "wifi") + } + if snapshot.DefaultRouteV4 != "en0" || snapshot.DefaultRouteV6 != "" { + t.Errorf("default routes = %q/%q, want en0 and empty", snapshot.DefaultRouteV4, snapshot.DefaultRouteV6) + } + if !snapshot.HaveV4 || snapshot.HaveV6 { + t.Errorf("have_v4/have_v6 = %v/%v, want true/false", snapshot.HaveV4, snapshot.HaveV6) + } + want := []snapshotInterface{{Name: "en0", Class: "hardware", Up: true, IPs: []string{"172.20.10.2/28"}, HardwarePort: "Wi-Fi", Service: "Wi-Fi"}} + if !slices.EqualFunc(snapshot.Interfaces, want, sameSnapshotInterface) { + t.Errorf("interfaces = %+v, want %+v", snapshot.Interfaces, want) + } +} + +func Test_buildNetworkSnapshot_usbTether(t *testing.T) { + state := snapshotTestState("en7", snapshotTestInterface{"en7", true, []string{"172.20.10.3/28"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{"en7": {Class: "hardware", HardwarePort: "iPhone USB", Service: "iPhone USB"}}), + }) + + if snapshot.LinkType != "usb_tether" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "usb_tether") + } + if !snapshot.Tethered { + t.Error("tethered = false, want true for the iPhone USB port") + } +} + +func Test_buildNetworkSnapshot_tunnelDefaultRoute(t *testing.T) { + state := snapshotTestState("utun3", snapshotTestInterface{"utun3", true, []string{"100.64.0.2/32"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{"utun3": {Class: "tunnel"}}), + }) + + if snapshot.LinkType != "tunnel" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "tunnel") + } + if snapshot.Tethered { + t.Error("tethered = true, want false without a hotspot gateway") + } +} + +// Test_buildNetworkSnapshot_defaultRouteV4WithoutARouteTable covers the +// platforms that read no route table. The v4 route falls back to the interface +// of the network monitor, and that interface must hold a usable v4 address. +func Test_buildNetworkSnapshot_defaultRouteV4WithoutARouteTable(t *testing.T) { + for _, tt := range []struct { + name string + ips []string + wantV4 string + otherIPs []string + }{ + {name: "dual_stack", ips: []string{"192.0.2.10/24", "2001:db8::10/64"}, wantV4: "en0"}, + {name: "v4_only", ips: []string{"192.0.2.10/24", "fe80::1/64"}, wantV4: "en0"}, + {name: "v6_only", ips: []string{"2001:db8::10/64"}}, + {name: "link_local_only", ips: []string{"169.254.1.2/16", "fe80::1/64"}}, + {name: "loopback_only", ips: []string{"127.0.0.1/8"}}, + {name: "address_on_another_interface", ips: nil, otherIPs: []string{"192.0.2.20/24", "2001:db8::20/64"}}, + } { + t.Run(tt.name, func(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, tt.ips}, + snapshotTestInterface{"en1", true, tt.otherIPs}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + if snapshot.DefaultRouteV4 != tt.wantV4 { + t.Errorf("default_route_v4 = %q, want %q", snapshot.DefaultRouteV4, tt.wantV4) + } + if snapshot.DefaultRouteV6 != "" { + t.Errorf("default_route_v6 = %q, want an empty value without a route table", snapshot.DefaultRouteV6) + } + }) + } +} + +// Test_buildNetworkSnapshot_defaultRoutesComeFromTheRouteTable covers a host +// whose v6 default route leaves over a tunnel. The v6 route must name that +// tunnel, not the interface of the v4 route. +func Test_buildNetworkSnapshot_defaultRoutesComeFromTheRouteTable(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24", "2001:db8::10/64"}}, + snapshotTestInterface{"utun4", true, []string{"2001:db8:1::2/64"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "192.0.2.1", Interface: "en0"}, + RouteV6: defaultRoute{Gateway: "fe80::1%utun4", Interface: "utun4"}, + }) + + if snapshot.DefaultRouteV4 != "en0" || snapshot.DefaultRouteV6 != "utun4" { + t.Errorf("default routes = %q/%q, want en0 and utun4", snapshot.DefaultRouteV4, snapshot.DefaultRouteV6) + } + if snapshot.GatewayV4 != "192.0.2.1" || snapshot.GatewayV6 != "fe80::1%utun4" { + t.Errorf("gateways = %q/%q, want the gateways of the route table", snapshot.GatewayV4, snapshot.GatewayV6) + } +} + +func Test_buildNetworkSnapshot_sortsInterfacesAndTakesMeta(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en1", false, []string{"192.0.2.11/24"}}, + snapshotTestInterface{"awdl0", true, []string{"fe80::2/64"}}, + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{ + "en0": {Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + "awdl0": {Class: "airdrop"}, + "en1": {Class: "hardware", HardwarePort: "Ethernet", Service: "Ethernet"}, + }), + }) + + want := []snapshotInterface{ + {Name: "awdl0", Class: "airdrop", Up: true, IPs: []string{"fe80::2/64"}}, + {Name: "en0", Class: "hardware", Up: true, IPs: []string{"192.0.2.10/24"}, HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + {Name: "en1", Class: "hardware", Up: false, IPs: []string{"192.0.2.11/24"}, HardwarePort: "Ethernet", Service: "Ethernet"}, + } + if !slices.EqualFunc(snapshot.Interfaces, want, sameSnapshotInterface) { + t.Fatalf("interfaces = %+v, want %+v", snapshot.Interfaces, want) + } +} + +// Test_buildNetworkSnapshot_capsTheInterfaceList covers a container host. Such +// a host holds hundreds of interfaces, and a snapshot of all of them fills the +// journal. +func Test_buildNetworkSnapshot_capsTheInterfaceList(t *testing.T) { + interfaces := make([]snapshotTestInterface, 0, 40) + for i := range 36 { + interfaces = append(interfaces, snapshotTestInterface{fmt.Sprintf("veth%02d", i), true, nil}) + } + for i := range 4 { + interfaces = append(interfaces, snapshotTestInterface{fmt.Sprintf("down%d", i), false, nil}) + } + state := snapshotTestState("veth00", interfaces...) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + if len(snapshot.Interfaces) != snapshotInterfaceLimit { + t.Fatalf("interfaces = %d entries, want %d", len(snapshot.Interfaces), snapshotInterfaceLimit) + } + if snapshot.InterfacesOmitted != 8 { + t.Errorf("interfaces_omitted = %d, want 8", snapshot.InterfacesOmitted) + } + if first, last := snapshot.Interfaces[0].Name, snapshot.Interfaces[snapshotInterfaceLimit-1].Name; first != "veth00" || last != "veth31" { + t.Errorf("interfaces run from %q to %q, want veth00 to veth31", first, last) + } +} + +// Test_buildNetworkSnapshot_keepsUpOrAddressedInterfaces covers the rule that +// selects the interfaces of the list. +func Test_buildNetworkSnapshot_keepsUpOrAddressedInterfaces(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}, + snapshotTestInterface{"en1", true, nil}, + snapshotTestInterface{"en2", false, []string{"192.0.2.20/24"}}, + snapshotTestInterface{"en3", false, nil}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + var names []string + for _, iface := range snapshot.Interfaces { + names = append(names, iface.Name) + } + if !slices.Equal(names, []string{"en0", "en1", "en2"}) { + t.Errorf("interfaces = %v, want en0, en1, and en2", names) + } + if snapshot.InterfacesOmitted != 1 { + t.Errorf("interfaces_omitted = %d, want 1", snapshot.InterfacesOmitted) + } +} + +func Test_buildNetworkSnapshot_classesByNameWithoutMeta(t *testing.T) { + state := snapshotTestState("utun3", + snapshotTestInterface{"utun3", true, []string{"100.64.0.2/32"}}, + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + classes := map[string]string{} + for _, iface := range snapshot.Interfaces { + classes[iface.Name] = iface.Class + } + if classes["utun3"] != "tunnel" || classes["en0"] != "hardware" { + t.Fatalf("classes = %v, want utun3 tunnel and en0 hardware", classes) + } + if snapshot.LinkType != "tunnel" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "tunnel") + } +} + +func Test_buildNetworkSnapshot_fillsTheEmptyClassOfTheMeta(t *testing.T) { + state := snapshotTestState("utun3", + snapshotTestInterface{"utun3", true, []string{"100.64.0.2/32"}}, + snapshotTestInterface{"awdl0", true, []string{"fe80::2/64"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{"utun3": {HardwarePort: ""}}), + }) + + classes := map[string]string{} + for _, iface := range snapshot.Interfaces { + classes[iface.Name] = iface.Class + } + if classes["utun3"] != "tunnel" || classes["awdl0"] != "airdrop" { + t.Fatalf("classes = %v, want utun3 tunnel and awdl0 airdrop", classes) + } + if snapshot.LinkType != "tunnel" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "tunnel") + } +} + +func Test_buildNetworkSnapshot_nilStateKeepsFlags(t *testing.T) { + snapshot := buildNetworkSnapshot(snapshotInputs{ + RouteV4: defaultRoute{Gateway: "192.168.1.1"}, + RouteV6: defaultRoute{Gateway: "fe80::1"}, + Resolvers: []string{"192.168.1.1:53"}, + SourceIPv4: "192.168.1.20", + SourceIPv6: "2001:db8::20", + InterceptTarget: "127.0.0.1", + BypassActive: true, + RecoveryRunning: true, + PFStabilizing: true, + NAT64Prefix: "64:ff9b::/96", + CLATPresent: true, + }) + + if len(snapshot.Interfaces) != 0 { + t.Errorf("interfaces = %+v, want none", snapshot.Interfaces) + } + if snapshot.DefaultRouteV4 != "" || snapshot.DefaultRouteV6 != "" || snapshot.HaveV4 || snapshot.HaveV6 { + t.Errorf("state fields are set without a state: %+v", snapshot) + } + if snapshot.LinkType != "unknown" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "unknown") + } + if !snapshot.BypassActive || !snapshot.RecoveryRunning || !snapshot.PFStabilizing || !snapshot.CLATPresent { + t.Errorf("flags dropped: %+v", snapshot) + } + if !snapshot.DNSLessTarget { + t.Error("dns_less_target = false, want true with an intercept target") + } + if snapshot.GatewayV4 != "192.168.1.1" || snapshot.GatewayV6 != "fe80::1" { + t.Errorf("gateways = %q/%q, want the input values", snapshot.GatewayV4, snapshot.GatewayV6) + } + if snapshot.SourceIPv4 != "192.168.1.20" || snapshot.SourceIPv6 != "2001:db8::20" { + t.Errorf("sources = %q/%q, want the input values", snapshot.SourceIPv4, snapshot.SourceIPv6) + } + if snapshot.NAT64Prefix != "64:ff9b::/96" || !slices.Equal(snapshot.Resolvers, []string{"192.168.1.1:53"}) { + t.Errorf("prefix and resolvers dropped: %+v", snapshot) + } +} + +func Test_buildNetworkSnapshot_noInterceptTargetIsNotDNSLess(t *testing.T) { + snapshot := buildNetworkSnapshot(snapshotInputs{State: snapshotTestState("en0", snapshotTestInterface{"en0", true, nil})}) + if snapshot.DNSLessTarget { + t.Error("dns_less_target = true, want false without an intercept target") + } +} + +// snapshotDictKeys are the field names that spec R2.2 names for the snapshot, +// plus the count of the interfaces that the list leaves out. +var snapshotDictKeys = []string{ + "default_route_v4", "default_route_v6", "gateway_v4", "gateway_v6", + "have_v4", "have_v6", "interfaces", "interfaces_omitted", "resolvers", + "source_ipv4", "source_ipv6", "intercept_target", "bypass_active", + "recovery_running", "pf_stabilizing", "link_type", "tethered", + "clat_present", "nat64_prefix", "dns_less_target", +} + +var snapshotInterfaceDictKeys = []string{"class", "hardware_port", "ips", "name", "service", "up"} + +// renderSnapshotDict writes the dict under one field and parses the line, so +// the test reads the keys that a reader of the journal sees. +func renderSnapshotDict(t *testing.T, snapshot networkSnapshot) (string, map[string]any) { + t.Helper() + buf := &syncBuffer{} + newTestJSONLogger(buf).Info().Dict("network", snapshotDict(snapshot)).Msg("Network snapshot") + + line := strings.TrimSpace(buf.String()) + event := map[string]any{} + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("log line is not JSON: %q: %v", line, err) + } + network, ok := event["network"].(map[string]any) + if !ok { + t.Fatalf("line has no network object: %q", line) + } + return line, network +} + +func Test_snapshotDict_rendersEveryKeyOnce(t *testing.T) { + line, network := renderSnapshotDict(t, buildNetworkSnapshot(snapshotInputs{})) + + if got := sortedKeys(network); !slices.Equal(got, slices.Sorted(slices.Values(snapshotDictKeys))) { + t.Fatalf("keys = %v, want %v", got, snapshotDictKeys) + } + for _, key := range snapshotDictKeys { + if count := strings.Count(line, `"`+key+`":`); count != 1 { + t.Errorf("key %q appears %d times, want 1: %s", key, count, line) + } + } + if network["interfaces"] == nil || len(network["interfaces"].([]any)) != 0 { + t.Errorf("interfaces = %v, want an empty array", network["interfaces"]) + } + if network["resolvers"] == nil || len(network["resolvers"].([]any)) != 0 { + t.Errorf("resolvers = %v, want an empty array", network["resolvers"]) + } +} + +func Test_snapshotDict_rendersSortedInterfaces(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en1", true, nil}, + snapshotTestInterface{"awdl0", true, []string{"fe80::2/64"}}, + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "192.0.2.1", Interface: "en0"}, + Resolvers: []string{"192.0.2.1:53"}, + Meta: snapshotTestMeta(map[string]interfaceMeta{"en0": {Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi"}}), + }) + _, network := renderSnapshotDict(t, snapshot) + + interfaces, ok := network["interfaces"].([]any) + if !ok || len(interfaces) != 3 { + t.Fatalf("interfaces = %v, want three entries", network["interfaces"]) + } + var names []string + for _, entry := range interfaces { + object, ok := entry.(map[string]any) + if !ok { + t.Fatalf("interface entry is not an object: %v", entry) + } + if got := sortedKeys(object); !slices.Equal(got, snapshotInterfaceDictKeys) { + t.Fatalf("interface keys = %v, want %v", got, snapshotInterfaceDictKeys) + } + names = append(names, object["name"].(string)) + } + if !slices.Equal(names, []string{"awdl0", "en0", "en1"}) { + t.Errorf("interface order = %v, want awdl0, en0, en1", names) + } + if ips, ok := interfaces[2].(map[string]any)["ips"].([]any); !ok || len(ips) != 0 { + t.Errorf("ips of en1 = %v, want an empty array", interfaces[2].(map[string]any)["ips"]) + } + wantValues := map[string]any{"gateway_v4": "192.0.2.1", "link_type": "wifi", "default_route_v4": "en0", "tethered": false} + for key, want := range wantValues { + wantField(t, network, key, want) + } + if resolvers, ok := network["resolvers"].([]any); !ok || len(resolvers) != 1 || resolvers[0] != "192.0.2.1:53" { + t.Errorf("resolvers = %v, want one entry", network["resolvers"]) + } +} + +func Test_defaultRoutes_returnAddressesOrNothing(t *testing.T) { + v4, v6 := defaultRoutes() + // A host without a route of a family gives an empty value, so the test + // logs the pair and holds every value it gets to the format. + t.Logf("default routes: v4 %+v, v6 %+v", v4, v6) + + if v4.Gateway != "" { + address, err := netip.ParseAddr(v4.Gateway) + if err != nil || !address.Is4() { + t.Errorf("gateway_v4 = %q, want an IPv4 address or nothing: %v", v4.Gateway, err) + } + } + if v6.Gateway != "" { + address, err := netip.ParseAddr(v6.Gateway) + if err != nil || !address.Is6() { + t.Errorf("gateway_v6 = %q, want an IPv6 address or nothing: %v", v6.Gateway, err) + } + } + // A tunnel default has no next hop, so an interface without a gateway is a + // valid pair. A gateway without an interface is not. + if v4.Gateway != "" && v4.Interface == "" { + t.Errorf("gateway_v4 = %q without an interface, want the interface of the route", v4.Gateway) + } + if v6.Gateway != "" && v6.Interface == "" { + t.Errorf("gateway_v6 = %q without an interface, want the interface of the route", v6.Gateway) + } +} + +func sameSnapshotInterface(a, b snapshotInterface) bool { + return a.Name == b.Name && a.Class == b.Class && a.Up == b.Up && + a.HardwarePort == b.HardwarePort && a.Service == b.Service && + slices.Equal(a.IPs, b.IPs) +} + +func sortedKeys(object map[string]any) []string { + keys := make([]string, 0, len(object)) + for key := range object { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} diff --git a/cmd/cli/network_source_state_test.go b/cmd/cli/network_source_state_test.go new file mode 100644 index 0000000..dfb197e --- /dev/null +++ b/cmd/cli/network_source_state_test.go @@ -0,0 +1,147 @@ +package cli + +import ( + "context" + "errors" + "net" + "net/netip" + "slices" + "testing" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +func Test_interfaceAddressPrefix(t *testing.T) { + for _, tc := range []struct { + name string + address net.Addr + want string + }{ + {"IPv4 with a 4-byte mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10).To4(), Mask: net.CIDRMask(24, 32)}, "192.0.2.10/24"}, + {"IPv4 in the 16-byte form with a 4-byte mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10), Mask: net.CIDRMask(24, 32)}, "192.0.2.10/24"}, + {"IPv4 in the 16-byte form with a 16-byte mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10), Mask: net.CIDRMask(120, 128)}, "192.0.2.10/24"}, + {"IPv6 with a 64-bit mask", &net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(64, 128)}, "2001:db8::1/64"}, + {"IPv4 without a mask", &net.IPAddr{IP: net.IPv4(192, 0, 2, 10)}, "192.0.2.10/32"}, + {"IPv6 without a mask", &net.IPAddr{IP: net.ParseIP("2001:db8::1")}, "2001:db8::1/128"}, + {"IPv4 with a nil mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10)}, "192.0.2.10/32"}, + {"IPv4 with an irregular mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10), Mask: net.IPMask{255, 0, 255, 0}}, "192.0.2.10/32"}, + } { + t.Run(tc.name, func(t *testing.T) { + prefix, err := interfaceAddressPrefix(tc.address) + if err != nil { + t.Fatalf("interfaceAddressPrefix: %v", err) + } + if prefix.String() != tc.want { + t.Fatalf("prefix = %s, want %s", prefix, tc.want) + } + }) + } + if _, err := interfaceAddressPrefix(&net.TCPAddr{IP: net.IPv4(192, 0, 2, 10), Port: 53}); err == nil { + t.Fatal("a TCP address gave a prefix, want an error") + } +} + +// sourceTestInterfaces returns one up interface with a /24 and a /64 address, +// through the reader seam of networkSourceState. +func sourceTestInterfaces() ([]net.Interface, func(*net.Interface) ([]net.Addr, error)) { + interfaces := []net.Interface{{Index: 1, Name: "en0", Flags: net.FlagUp}} + addresses := func(iface *net.Interface) ([]net.Addr, error) { + if iface.Name != "en0" { + return nil, errors.New("unknown interface") + } + return []net.Addr{ + &net.IPNet{IP: net.IPv4(192, 0, 2, 10), Mask: net.CIDRMask(24, 32)}, + &net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(64, 128)}, + }, nil + } + return interfaces, addresses +} + +func Test_networkSourceState_keepsTheMasks(t *testing.T) { + state, err := networkSourceState(sourceTestInterfaces()) + if err != nil { + t.Fatalf("networkSourceState: %v", err) + } + if !state.Interface["en0"].IsUp() { + t.Fatal("en0 is not up") + } + if got := prefixStrings(state.InterfaceIPs["en0"]); !slices.Equal(got, []string{"192.0.2.10/24", "2001:db8::1/64"}) { + t.Fatalf("prefixes = %v, want the /24 and the /64 with the host address", got) + } +} + +// stubSourceStateRead makes the fresh network read return the test interfaces +// through the real state builder. +func stubSourceStateRead(t *testing.T) { + t.Helper() + origRead := readNetworkSourceStateFn + t.Cleanup(func() { readNetworkSourceStateFn = origRead }) + readNetworkSourceStateFn = func() (*netmon.State, error) { return networkSourceState(sourceTestInterfaces()) } +} + +// Test_freshSnapshotAndHeaderKeepTheMasks proves that the send-time header and +// the fresh snapshot show the /24 and the /64 of the host, as the network +// monitor does. +func Test_freshSnapshotAndHeaderKeepTheMasks(t *testing.T) { + stubHeaderSnapshotSources(t) + stubSourceStateRead(t) + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + want := []string{"192.0.2.10/24", "2001:db8::1/64"} + + snapshot := buildNetworkSnapshot(p.freshSnapshotInputs()) + index := slices.IndexFunc(snapshot.Interfaces, func(i snapshotInterface) bool { return i.Name == "en0" }) + if index < 0 { + t.Fatalf("snapshot holds no en0: %+v", snapshot.Interfaces) + } + if got := snapshot.Interfaces[index].IPs; !slices.Equal(got, want) { + t.Fatalf("snapshot ips = %v, want %v", got, want) + } + + parsed, _ := parseLogHeader(t, p.sendLogHeader()) + var headerIPs []string + for _, iface := range parsed.Network.Interfaces { + if iface.Name == "en0" { + headerIPs = iface.IPs + } + } + if !slices.Equal(headerIPs, want) { + t.Fatalf("header ips = %v, want %v", headerIPs, want) + } +} + +// Test_sourceValidationKeepsTheHostAddress proves that a state with masked +// prefixes still names the host address, not the subnet address, as the +// source of the process. +func Test_sourceValidationKeepsTheHostAddress(t *testing.T) { + before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + t.Cleanup(func() { + ctrld.SetDefaultLocalIPv4(context.Background(), before4) + ctrld.SetDefaultLocalIPv6(context.Background(), before6) + }) + state, err := networkSourceState(sourceTestInterfaces()) + if err != nil { + t.Fatalf("networkSourceState: %v", err) + } + + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP("192.0.2.10")) + ctrld.SetDefaultLocalIPv6(context.Background(), net.ParseIP("2001:db8::1")) + validateDefaultLocalIPsFromDelta(context.Background(), state, 1) + if got := ctrld.GetDefaultLocalIPv4(); !got.Equal(net.ParseIP("192.0.2.10")) { + t.Fatalf("source IPv4 = %v, want the host address kept", got) + } + if got := ctrld.GetDefaultLocalIPv6(); !got.Equal(net.ParseIP("2001:db8::1")) { + t.Fatalf("source IPv6 = %v, want the host address kept", got) + } + + // The subnet address is not an address of the host. + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP("192.0.2.0")) + validateDefaultLocalIPsFromDelta(context.Background(), state, 2) + if got := ctrld.GetDefaultLocalIPv4(); got != nil { + t.Fatalf("source IPv4 = %v, want the subnet address cleared", got) + } + if reason := sourceInvalidReason(state, netip.MustParseAddr("192.0.2.10").AsSlice()); reason != "" { + t.Fatalf("host address reported %q, want a valid source", reason) + } +} diff --git a/cmd/cli/network_transition.go b/cmd/cli/network_transition.go index 1a7075d..2c9a8df 100644 --- a/cmd/cli/network_transition.go +++ b/cmd/cli/network_transition.go @@ -49,34 +49,71 @@ func readNetworkSourceState() (*netmon.State, error) { if err != nil { return nil, err } + return networkSourceState(interfaces, (*net.Interface).Addrs) +} + +// networkSourceState builds the state of the given interfaces. The address +// reader is a parameter, so a test feeds addresses without a host network. +func networkSourceState(interfaces []net.Interface, addressesOf func(*net.Interface) ([]net.Addr, error)) (*netmon.State, error) { state := &netmon.State{Interface: map[string]netmon.Interface{}, InterfaceIPs: map[string][]netip.Prefix{}} for _, iface := range interfaces { - addresses, err := iface.Addrs() + addresses, err := addressesOf(&iface) if err != nil { return nil, err } state.Interface[iface.Name] = netmon.Interface{Interface: &iface} for _, address := range addresses { - var ip net.IP - switch address := address.(type) { - case *net.IPNet: - ip = address.IP - case *net.IPAddr: - ip = address.IP - default: - return nil, errors.New("unsupported interface address type") + prefix, err := interfaceAddressPrefix(address) + if err != nil { + return nil, err } - addr, ok := netip.AddrFromSlice(ip) - if !ok { - return nil, errors.New("invalid interface address") - } - addr = addr.Unmap() - state.InterfaceIPs[iface.Name] = append(state.InterfaceIPs[iface.Name], netip.PrefixFrom(addr, addr.BitLen())) + state.InterfaceIPs[iface.Name] = append(state.InterfaceIPs[iface.Name], prefix) } } return state, nil } +// interfaceAddressPrefix turns one interface address into a prefix. An address +// with a subnet mask keeps the mask and its host bits, so a fresh state shows +// the same /24 or /64 as the network monitor. An address without a mask is a +// host prefix. +func interfaceAddressPrefix(address net.Addr) (netip.Prefix, error) { + var ip net.IP + var mask net.IPMask + switch address := address.(type) { + case *net.IPNet: + ip, mask = address.IP, address.Mask + case *net.IPAddr: + ip = address.IP + default: + return netip.Prefix{}, errors.New("unsupported interface address type") + } + addr, ok := netip.AddrFromSlice(ip) + if !ok { + return netip.Prefix{}, errors.New("invalid interface address") + } + addr = addr.Unmap() + return netip.PrefixFrom(addr, prefixBits(addr, mask)), nil +} + +// prefixBits reads the length of a mask for an unmapped address. The mask of +// an IPv4 address is 4 or 16 bytes long, and a 16-byte mask counts the 96 +// bits of the IPv6 mapping too. A missing or irregular mask means a host +// prefix. +func prefixBits(addr netip.Addr, mask net.IPMask) int { + bits, total := mask.Size() + if total == 0 { + return addr.BitLen() + } + if addr.Is4() && total == 128 { + bits -= 96 + } + if bits < 0 || bits > addr.BitLen() { + return addr.BitLen() + } + return bits +} + // netmon v1.74.0 caches only major snapshots. Minor and time-jump events // belong to their Old snapshot, not their distinct New snapshot. func networkSnapshotCurrent(delta *netmon.ChangeDelta, major bool, current *netmon.State) bool { @@ -154,6 +191,13 @@ type recoveryDiagnostic struct { started time.Time firstFailure sync.Once failed atomic.Bool + + // The flow fills these while it runs. The end event reports them, so one + // journal line tells what the recovery found and what it changed. + recoveredUpstream string + dhcpServers []string + interceptTargetAction string + bypassActive bool } func recoveryReasonName(reason RecoveryReason) string { @@ -192,12 +236,36 @@ func (d *recoveryDiagnostic) failure(err error) { }) } +// The outcomes that one recovery pass ends with. +const ( + recoveryOutcomeCompleted = "completed" + recoveryOutcomeCanceled = "canceled" + recoveryOutcomeSuperseded = "superseded" + recoveryOutcomeFailed = "failed" +) + +// journalRecoveredUpstream bounds the upstream name of the end event. An empty +// name means that no upstream recovered, so it stays empty. +func journalRecoveredUpstream(upstream string) string { + if upstream == "" { + return "" + } + return journalUpstreamName(upstream) +} + func (d *recoveryDiagnostic) end(outcome string) { - e := d.log().Debug() - if outcome == "canceled" || d.failed.Load() { + e := d.log().Info() + if outcome == recoveryOutcomeCanceled || outcome == recoveryOutcomeFailed || d.failed.Load() { e = d.log().Warn() } - d.event(e).Str("outcome", outcome).Bool("had_failure", d.failed.Load()).Int64("duration_ms", time.Since(d.started).Milliseconds()).Msg("Recovery end") + journal(d.event(e)).Str("outcome", outcome).Bool("had_failure", d.failed.Load()). + Int64("duration_ms", time.Since(d.started).Milliseconds()). + Str("recovered_upstream", journalRecoveredUpstream(d.recoveredUpstream)). + Strs("dhcp_servers", d.dhcpServers). + Int("dhcp_server_count", len(d.dhcpServers)). + Str("intercept_target_action", d.interceptTargetAction). + Bool("bypass_active", d.bypassActive). + Msg("Recovery end") } func (d *recoveryDiagnostic) log() *ctrld.Logger { diff --git a/cmd/cli/network_transition_test.go b/cmd/cli/network_transition_test.go index 9807bd8..809e6b7 100644 --- a/cmd/cli/network_transition_test.go +++ b/cmd/cli/network_transition_test.go @@ -46,6 +46,10 @@ func captureTransitionLogs(t *testing.T) *syncBuffer { func sourceTestGlobals(t *testing.T) { t.Helper() + // A transition reads the hardware ports and the virtual adapters, and no + // test may start the commands behind them. + stubHeaderSnapshotSources(t) + stubSnapshotVirtualSet(t) v4, v6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() valid, route := networkChangeValidInterfacesFn, networkChangeDefaultRouteIPFn reconcile, ignored := networkChangeReconcileFn, networkChangeIgnoredInterceptFn @@ -191,6 +195,7 @@ func TestNetworkChangeSourceMovedToUpInterface(t *testing.T) { func TestRecoveryTransitionDiagnostics(t *testing.T) { logs := captureTransitionLogs(t) + stubHeaderSnapshotSources(t) original, intercept := recoveryResetDNSFn, dnsIntercept dnsIntercept = false t.Cleanup(func() { recoveryResetDNSFn = original; dnsIntercept = intercept }) @@ -260,7 +265,7 @@ func TestTransitionWarningsSurviveDebugRotation(t *testing.T) { config.MessageKey = "message" logger := &ctrld.Logger{Logger: zap.New(zapcore.NewTee( zapcore.NewCore(zapcore.NewJSONEncoder(config), zapcore.AddSync(debug), zap.DebugLevel), - zapcore.NewCore(zapcore.NewJSONEncoder(config), zapcore.AddSync(warnings), zap.WarnLevel), + newJournalCore(warnings), ))} mainLog.Store(logger) t.Cleanup(func() { mainLog.Store(old) }) @@ -279,6 +284,8 @@ func TestTransitionWarningsSurviveDebugRotation(t *testing.T) { wg.Wait() d.end("completed") (&recoveryDiagnostic{transitionID: 8, generation: 10, started: time.Now()}).end("completed") + journal(logger.Info()).Str("interface", "en0").Msg("Network snapshot") + logger.Info().Msg("plain info line") for range 100 { logger.Debug().Msg(strings.Repeat("noise", 100)) } @@ -291,12 +298,18 @@ func TestTransitionWarningsSurviveDebugRotation(t *testing.T) { t.Fatalf("debug buffer did not rotate %q", msg) } } - if strings.Count(text, "Recovery end") != 1 || !strings.Contains(text, `"outcome":"completed"`) { + if strings.Count(text, "Recovery end") != 2 || !strings.Contains(text, `"outcome":"completed"`) { t.Fatal("recovery completion must survive rotation beside its retained failure") } if strings.Contains(text, "private.customer") || strings.Contains(text, "secret payload") { t.Fatalf("raw payload or normal debug elevated: %s", text) } + if strings.Count(text, "Network snapshot") != 1 { + t.Fatalf("expected one retained marked info line: %s", text) + } + if strings.Contains(text, "plain info line") { + t.Fatalf("unmarked info line must not be retained: %s", text) + } } func TestRecoveryWaitLogsFirstFailure(t *testing.T) { diff --git a/cmd/cli/pf_state.go b/cmd/cli/pf_state.go new file mode 100644 index 0000000..2b10cc6 --- /dev/null +++ b/cmd/cli/pf_state.go @@ -0,0 +1,82 @@ +package cli + +import ( + "bufio" + "io" + "maps" + "slices" + "strings" +) + +// parsePFAnchorNames returns the anchor names that the running ruleset refers to, +// unique and sorted. It takes both pfctl show commands because each one holds half +// of the list: "pfctl -sr" the scrub and filter anchors, "pfctl -sn" the +// translation anchors. +func parsePFAnchorNames(rules, nat io.Reader) []string { + unique := make(map[string]struct{}) + // A read that stopped early gives a shorter list, and a shorter list reads + // as an anchor that went away. + if !collectPFAnchorNames(rules, unique) || !collectPFAnchorNames(nat, unique) { + return nil + } + return slices.Sorted(maps.Keys(unique)) +} + +// collectPFAnchorNames reports whether it read the whole output. +func collectPFAnchorNames(r io.Reader, into map[string]struct{}) bool { + if r == nil { + return true + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + name := pfAnchorNameOf(scanner.Text()) + if name == "" { + continue + } + into[name] = struct{}{} + } + return scanner.Err() == nil +} + +// pfAnchorNameOf returns the anchor name of one pfctl rule line. The rule verb is +// anchor, nat-anchor, rdr-anchor, scrub-anchor or dummynet-anchor, and the quoted +// name follows it. Every other line, including the merged ALTQ warnings, gives "". +func pfAnchorNameOf(line string) string { + verb, rest, ok := strings.Cut(strings.TrimSpace(line), " ") + if !ok || !strings.HasSuffix(verb, "anchor") { + return "" + } + _, quoted, ok := strings.Cut(rest, `"`) + if !ok { + return "" + } + name, _, ok := strings.Cut(quoted, `"`) + if !ok { + return "" + } + return name +} + +// parsePFStatus reads "pfctl -si" output and reports whether pf runs, with the +// uptime text that pfctl prints beside it. +func parsePFStatus(r io.Reader) (enabled bool, since string) { + if r == nil { + return false, "" + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + status, ok := strings.CutPrefix(strings.TrimSpace(scanner.Text()), "Status:") + if !ok { + continue + } + // pfctl pads the status into a column and prints the debug level next to it. + status, _, _ = strings.Cut(status, "Debug:") + state, uptime, _ := strings.Cut(strings.TrimSpace(status), " for ") + if state != "Enabled" { + return false, "" + } + return true, strings.TrimSpace(uptime) + } + // A read that stopped early may have stopped before the status line. + return false, "" +} diff --git a/cmd/cli/pf_state_darwin.go b/cmd/cli/pf_state_darwin.go new file mode 100644 index 0000000..b89dcfe --- /dev/null +++ b/cmd/cli/pf_state_darwin.go @@ -0,0 +1,119 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "os/exec" + "time" +) + +// pfStateReadTimeout bounds each state read. These reads run on the network change +// path, so a pfctl that waits on a busy kernel lock must not hold it. +const pfStateReadTimeout = 3 * time.Second + +// pfStateReadMaxBytes bounds the output of one state read. A ruleset or an +// interface dump of this size is not the state ctrld reads, and these reads run +// on the network change path. +const pfStateReadMaxBytes = 1 << 20 + +// errPFStateOutputTooLarge marks output that passed the bound. The bytes kept +// are a prefix of the real output, so the caller must not parse them. +var errPFStateOutputTooLarge = errors.New("pf state output passed its bound") + +// boundedBuffer keeps at most limit bytes and remembers that it dropped the +// rest. +type boundedBuffer struct { + bytes.Buffer + limit int + exceeded bool +} + +func (b *boundedBuffer) Write(p []byte) (int, error) { + room := b.limit - b.Len() + if len(p) > room { + b.exceeded = true + p = p[:max(room, 0)] + } + if _, err := b.Buffer.Write(p); err != nil { + return 0, err + } + return len(p), nil +} + +// pfStateRunCommand runs one read-only state command. A var so tests replace it +// instead of shelling out. +var pfStateRunCommand = func(name string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), pfStateReadTimeout) + defer cancel() + out := &boundedBuffer{limit: pfStateReadMaxBytes} + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdout = out + cmd.Stderr = out + err := cmd.Run() + if out.exceeded { + return nil, errPFStateOutputTooLarge + } + return out.Bytes(), err +} + +// pfRuleDump carries pfctl rule output that the caller already read. A nil +// field means that the caller holds no bytes for that flag, so the read runs +// here. +type pfRuleDump struct { + rules []byte // "pfctl -sr" + nat []byte // "pfctl -sn" +} + +// pfAnchorNames returns the anchor names of the running ruleset. A failed read +// gives no names, because a partial list reads as an anchor that went away. +func (p *prog) pfAnchorNames(dump pfRuleDump) []string { + rules, ok := p.pfRuleBytes(dump.rules, "-sr", "list filter anchors") + if !ok { + return nil + } + nat, ok := p.pfRuleBytes(dump.nat, "-sn", "list translation anchors") + if !ok { + return nil + } + return parsePFAnchorNames(bytes.NewReader(rules), bytes.NewReader(nat)) +} + +// pfRuleBytes returns the rule output of one pfctl flag. It starts pfctl only +// when the caller holds no bytes, and a failed read feeds the backoff that +// keeps ctrld off an exhausted host. +func (p *prog) pfRuleBytes(held []byte, flag, operation string) ([]byte, bool) { + if held != nil { + return held, true + } + out, err := pfStateRunCommand("pfctl", flag) + if err != nil { + p.pfBackoffResourceExhaustion(err, out, operation) + return nil, false + } + return out, true +} + +// pfStatus reports whether pf runs and for how long. A failed read gives false +// and "". +func (p *prog) pfStatus() (bool, string) { + out, err := pfStateRunCommand("pfctl", "-si") + if err != nil { + p.pfBackoffResourceExhaustion(err, out, "read pf status") + return false, "" + } + return parsePFStatus(bytes.NewReader(out)) +} + +// tunnelOwner returns the network extension that owns a tunnel interface, or "" +// when the read fails or no extension owns it. +func tunnelOwner(iface string) string { + if iface == "" { + return "" + } + out, err := pfStateRunCommand("ifconfig", "-v", iface) + if err != nil { + return "" + } + return parseIfconfigAgent(bytes.NewReader(out)) +} diff --git a/cmd/cli/pf_state_test.go b/cmd/cli/pf_state_test.go new file mode 100644 index 0000000..b7c4723 --- /dev/null +++ b/cmd/cli/pf_state_test.go @@ -0,0 +1,137 @@ +package cli + +import ( + "slices" + "strings" + "testing" +) + +// pfStatusEnabled is "pfctl -si" output from a Mac with pf enabled. +const pfStatusEnabled = `Status: Enabled for 0 days 02:11:05 Debug: Urgent +` + +// Captured from macOS 26.6 with ctrld installed. "pfctl -sr" prints the scrub and +// filter anchors, "pfctl -sn" prints the translation anchors, and both carry the +// ALTQ warnings because every reader here uses CombinedOutput. +const pfShowRules = altqNoise + `scrub-anchor "com.apple/*" all fragment reassemble +anchor "com.apple/*" all +anchor "com.controld.ctrld" all +pass in quick on lo0 reply-to lo0 inet proto udp from any to 127.0.0.1 port = 5354 +` + +const pfShowNAT = altqNoise + `nat-anchor "com.apple/*" all +rdr-anchor "com.apple/*" all +rdr-anchor "com.controld.ctrld" all +` + +func Test_parsePFAnchorNames(t *testing.T) { + tests := []struct { + name string + rules string + nat string + want []string + }{ + { + name: "a real ruleset with ctrld installed", + rules: pfShowRules, + nat: pfShowNAT, + want: []string{"com.apple/*", "com.controld.ctrld"}, + }, + { + // The names arrive out of order, so the result proves the sort. + name: "a VPN adds its own anchors", + rules: altqNoise + `anchor "windscribe" all +anchor "com.apple/*" all +`, + nat: altqNoise + `dummynet-anchor "com.apple/*" all +rdr-anchor "windscribe" all +`, + want: []string{"com.apple/*", "windscribe"}, + }, + { + name: "anchors in the translation output only", + rules: altqNoise, + nat: altqNoise + "rdr-anchor \"com.controld.ctrld\" all\n", + want: []string{"com.controld.ctrld"}, + }, + { + name: "a ruleset with no anchor", + rules: altqNoise + "block drop in all\n", + nat: altqNoise + "(no rules)\n", + want: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := parsePFAnchorNames(strings.NewReader(tc.rules), strings.NewReader(tc.nat)) + if !slices.Equal(got, tc.want) { + t.Fatalf("parsePFAnchorNames() = %v, want %v", got, tc.want) + } + }) + } +} + +func Test_parsePFStatus(t *testing.T) { + tests := []struct { + name string + output string + wantEnabled bool + wantSince string + }{ + { + name: "enabled with an uptime", + output: altqNoise + `Status: Enabled for 3 days 02:11:05 Debug: Urgent + +Interface Stats for en0 IPv4 IPv6 + Bytes In 123456789 0 +`, + wantEnabled: true, + wantSince: "3 days 02:11:05", + }, + { + name: "disabled", + output: altqNoise + "Status: Disabled Debug: Urgent\n\n", + }, + { + name: "enabled without an uptime", + output: "Status: Enabled Debug: Urgent\n", + wantEnabled: true, + }, + { + name: "no status line", + output: altqNoise, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + enabled, since := parsePFStatus(strings.NewReader(tc.output)) + if enabled != tc.wantEnabled || since != tc.wantSince { + t.Fatalf("parsePFStatus() = %v, %q, want %v, %q", enabled, since, tc.wantEnabled, tc.wantSince) + } + }) + } +} + +// oversizedLine is longer than the largest line a scanner holds, so the read +// of it stops with an error. +var oversizedLine = strings.Repeat("a", 70<<10) + "\n" + +// Test_parsePFAnchorNamesRejectsAnOversizedLine covers a truncated read. The +// names collected before the error read as a ruleset that lost its anchors, +// and that is the state the watchdog acts on. +func Test_parsePFAnchorNamesRejectsAnOversizedLine(t *testing.T) { + if got := parsePFAnchorNames(strings.NewReader(pfShowRules+oversizedLine), strings.NewReader(pfShowNAT)); got != nil { + t.Fatalf("anchor names of a truncated rule list = %v, want none", got) + } + if got := parsePFAnchorNames(strings.NewReader(pfShowRules), strings.NewReader(pfShowNAT+oversizedLine)); got != nil { + t.Fatalf("anchor names of a truncated translation list = %v, want none", got) + } +} + +// Test_parsePFStatusRejectsAnOversizedLine covers a truncated status read. +func Test_parsePFStatusRejectsAnOversizedLine(t *testing.T) { + enabled, since := parsePFStatus(strings.NewReader(oversizedLine + pfStatusEnabled)) + if enabled || since != "" { + t.Fatalf("pf status of a truncated read = %v, %q, want false and an empty uptime", enabled, since) + } +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 03c60e1..d924019 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -145,11 +145,24 @@ type prog struct { queryFromSelfMap sync.Map initInternalLogWriterOnce sync.Once internalLogWriter *logWriter - internalWarnLogWriter *logWriter + internalJournalWriter *logWriter + querySampler errorSampler + lastNetworkState atomic.Pointer[netmon.State] internalLogSent time.Time runningIface string requiredMultiNICsConfig bool + // The network journal state. The value fields here hold a mutex, so no + // code may copy a prog. Every user of a prog holds a pointer to it. + networkNoise noiseCoalescer + wake wakeReporter + repeats repeatLogger //lint:ignore U1000 used on darwin + dnsConfig *dnsConfigPoller + health *queryHealth + snapshotWrites snapshotLimiter + lastNAT64Mu sync.Mutex + lastNAT64Logged string + selfUninstallMu sync.Mutex refusedQueryCount int canSelfUninstall atomic.Bool @@ -170,6 +183,10 @@ type prog struct { networkSourceState *netmon.State networkSourceEpoch *netmon.State networkSourceReadFailed bool + // netmon hands a minor callback the cached major snapshot in Old, so the + // diff keeps the state that the callback before it reported. + networkDeltaState *netmon.State + networkDeltaEpoch *netmon.State // recoveryDebounceTimer coalesces rapid NetworkChange recovery triggers // into a single handleRecovery call. Only handleRecovery is debounced — @@ -194,6 +211,11 @@ type prog struct { interceptDNSTargetSetValue string //lint:ignore U1000 used on darwin interceptDNSTargetLoaded bool //lint:ignore U1000 used on darwin + // interceptTargetMirror holds the same value as interceptDNSTargetSetValue + // for readers that must not wait: the mutex above covers networksetup + // calls that take seconds. + interceptTargetMirror atomic.Value // holds string + // DNS intercept mode state (platform-specific). // On Windows: *wfpState, on macOS: *pfState, nil on other platforms. dnsInterceptState any @@ -217,7 +239,11 @@ type prog struct { // lastTunnelIfaces tracks the tunnel set included in the last successfully loaded // pf anchor. Pending tunnel state is kept separately so failed PF work is retried // instead of being mistaken for an applied update. Protected by mu. - lastTunnelIfaces []string //lint:ignore U1000 used on darwin + lastTunnelIfaces []string //lint:ignore U1000 used on darwin + // lastLoggedTunnelIfaces is the tunnel set of the last logged event. The + // reconcile retries a set until it succeeds, so a diff of the applied set + // repeats one event at every retry. + lastLoggedTunnelIfaces []string //lint:ignore U1000 used on darwin pendingTunnelIfaces []string //lint:ignore U1000 used on darwin hasPendingTunnelIfaces bool //lint:ignore U1000 used on darwin @@ -464,6 +490,10 @@ func (p *prog) runWait() { } p.mu.Unlock() + // The header names the mode, the listeners, and the upstreams, so the + // open files take the values of the config that now runs. + p.refreshLogHeader() + p.Notice().Msg("Reloading config successfully") select { @@ -525,7 +555,7 @@ func (p *prog) postRun() { p.resetDNS(false, false) } loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) - ns, systemNameservers := initializeOsResolverWithSystemNameserversFn(loggerCtx, false) + ns, systemNameservers := initializeOsResolverWithSystemNameserversFn(loggerCtx, false, osResolverReasonStart) p.Debug().Msgf("Initialized os resolver with nameservers: %v", ns) p.setDNS(systemNameservers) if p.allowList != nil { @@ -533,7 +563,6 @@ func (p *prog) postRun() { } p.csSetDnsDone <- struct{}{} close(p.csSetDnsDone) - p.logInterfacesState() } } @@ -824,7 +853,7 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { } } - p.um = newUpstreamMonitor(p.cfg, p.logger.Load()) + p.replaceUpstreamMonitor() if !reload { p.sema = &chanSemaphore{ready: make(chan struct{}, defaultSemaphoreCap)} @@ -850,12 +879,8 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { } if !reload { - go func() { - // Start network monitoring - if err := p.monitorNetworkChanges(ctx); err != nil { - p.Error().Err(err).Msg("Failed to start network monitoring") - } - }() + p.newNetworkJournal() + go p.startNetworkMonitorAndJournal(ctx) } for listenerNum := range p.cfg.Listener { @@ -931,6 +956,96 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { wg.Wait() } +// dnsConfigChangedMessage names the journal event of the resolver table of the +// host. A log tool selects every resolver change of a run by this message. +const dnsConfigChangedMessage = "DNS configuration changed" + +// runSCUtilDNSFn reads the resolver table of the host. A test replaces it, +// because the configuration daemon of the host is not a fixture. +var runSCUtilDNSFn = runSCUtilDNS + +// replaceUpstreamMonitor puts a new upstream monitor in place of the one of +// the run before. The new monitor starts with every upstream up, so the old one +// closes its open outages first. Without this the journal holds a down event +// that no up event ever follows. +func (p *prog) replaceUpstreamMonitor() { + p.mu.Lock() + defer p.mu.Unlock() + if p.um != nil { + p.um.retire() + } + p.um = newUpstreamMonitor(p.cfg, p.logger.Load()) +} + +// upstreamMonitorNow reads the monitor of this run under the lock that a +// replacement takes, because a reload swaps it while the journal loops run. +func (p *prog) upstreamMonitorNow() *upstreamMonitor { + p.mu.Lock() + defer p.mu.Unlock() + return p.um +} + +// newNetworkJournal creates the trackers that follow the host network. The +// query path reads them without a lock, so they exist before a listener starts. +func (p *prog) newNetworkJournal() { + p.health = newQueryHealth() + p.querySampler.onFailure = p.health.countFailure + if runtime.GOOS == "darwin" { + p.dnsConfig = newDNSConfigPoller(runSCUtilDNSFn) + } +} + +// startNetworkMonitorAndJournal starts the network monitor and then opens the +// journal. The trackers do not depend on the monitor, so a monitor that cannot +// start leaves them running. It passes on the wait function of the journal. +func (p *prog) startNetworkMonitorAndJournal(ctx context.Context) func() { + if err := p.monitorNetworkChanges(ctx); err != nil { + p.Error().Err(err).Msg("Failed to start network monitoring") + } + return p.startNetworkJournal() +} + +// startNetworkJournal runs the trackers and opens the journal of this run with +// one snapshot. Support reads the grade of the query path and the resolver +// table of each moment after an incident. It returns a function that waits for +// both loops, so a caller that closed the stop channel knows when they ended. +func (p *prog) startNetworkJournal() func() { + healthDone := p.health.startLoop(p.stopCh, func() (int, bool) { + return p.upstreamMonitorNow().countDownExcept(isInternalDomainUpstream), p.recoveryBypass.Load() + }) + var configDone <-chan struct{} + if p.dnsConfig != nil { + configDone = p.dnsConfig.startLoop(p.stopCh, p.logDNSConfigChanges) + } + p.logNetworkSnapshot("start") + return func() { + <-healthDone + if configDone != nil { + <-configDone + } + } +} + +// logDNSConfigChanges puts each changed resolver of the host in the journal. A +// resolver that vanished comes without a nameserver, so the reader sees the +// drop. The search suffixes name the organization of the endpoint, so the +// journal holds their number only. +func (p *prog) logDNSConfigChanges(entries []dnsResolverEntry) { + for _, entry := range entries { + journal(mainLog.Load().Info()). + Strs("nameservers", entry.Nameservers). + Int("if_index", entry.IfIndex). + Str("interface", entry.Interface). + Bool("scoped", entry.Scoped). + Str("action", entry.Action). + Str("flags", entry.Flags). + Int("search_domain_count", len(entry.SearchDomains)). + Int("order", entry.Order). + Str("reachable", entry.Reachable). + Msg(dnsConfigChangedMessage) + } +} + // setupClientInfoDiscover performs necessary works for running client info discover. func (p *prog) setupClientInfoDiscover() { selfIP := p.defaultRouteIP() @@ -962,6 +1077,8 @@ func (p *prog) Stop(_ service.Service) error { p.Debug().Msg("Finish running onStopped functions") defer func() { p.Info().Msg("Service stopped") + // The files take the last line of this run before they close. + p.closeInternalLogs() }() if err := p.deAllocateIP(); err != nil { p.Error().Err(err).Msg("De-allocate ip failed") @@ -1037,7 +1154,7 @@ var ( startDNSInterceptFn = (*prog).startDNSIntercept ensureInterceptDNSTargetFn = (*prog).ensureInterceptDNSTarget removeInterceptDNSTargetFn = (*prog).removeInterceptDNSTarget - initializeOsResolverWithSystemNameserversFn = ctrld.InitializeOsResolverWithSystemNameservers + initializeOsResolverWithSystemNameserversFn = ctrld.InitializeOsResolverWithSystemNameserversReason setDnsForRunningIfaceFn = (*prog).setDnsForRunningIface resetDNSFn = (*prog).resetDNS // refuseFallbackFatal reports a startup failure the interface-DNS fallback @@ -1475,28 +1592,6 @@ func (p *prog) resetDNSForRunningIface(isStart bool, restoreStatic bool) (runnin return } -func (p *prog) logInterfacesState() { - withEachPhysicalInterfaces("", "", func(i *net.Interface) error { - addrs, err := i.Addrs() - if err != nil { - p.Warn().Str("interface", i.Name).Err(err).Msg("Failed to get addresses") - } - nss, err := currentStaticDNS(i) - if err != nil { - p.Warn().Str("interface", i.Name).Err(err).Msg("Failed to get DNS") - } - if len(nss) == 0 { - nss = currentDNS(i) - } - p.Debug(). - Any("addrs", addrs). - Strs("nameservers", nss). - Int("index", i.Index). - Msgf("interface state: %s", i.Name) - return nil - }) -} - // findWorkingInterface looks for a network interface with a valid IP configuration func (p *prog) findWorkingInterface() string { currentIface := p.runningIface diff --git a/cmd/cli/provision_result.go b/cmd/cli/provision_result.go index 478c83e..c123476 100644 --- a/cmd/cli/provision_result.go +++ b/cmd/cli/provision_result.go @@ -192,10 +192,17 @@ func newProvisionResult(code provisionFailureCode, message string, attempts []pr return r } -// redactSecrets removes every non-empty secret from s. +// redactSecretMinLen is the length below which a value is too short to +// redact. Two or three letters also match the keys of a JSON line and the +// words of a message, so a redaction of them corrupts the output. Corrupt +// output costs support more than a short value that the API rejects at the +// next step. It is the shortest token that the API parses. +const redactSecretMinLen = provisionTokenMinLen + +// redactSecrets removes every secret from s that is long enough to redact. func redactSecrets(s string, secrets ...string) string { for _, secret := range secrets { - if secret == "" { + if len(secret) < redactSecretMinLen { continue } s = strings.ReplaceAll(s, secret, "[redacted]") diff --git a/cmd/cli/provision_result_test.go b/cmd/cli/provision_result_test.go index f7f95d3..e4720ab 100644 --- a/cmd/cli/provision_result_test.go +++ b/cmd/cli/provision_result_test.go @@ -292,3 +292,22 @@ func TestFailProvisionWritesLogsNotifiesAndExits(t *testing.T) { t.Errorf("persisted code = %q", out.Code) } } + +// Test_redactSecretsKeepsAShortClientID puts a two-letter client ID in the +// resolver UID. The provisioning messages hold the same letters in ordinary +// words, and a redaction of the client ID would mangle them. +func Test_redactSecretsKeepsAShortClientID(t *testing.T) { + origCdUID, origCdOrg := cdUID, cdOrg + t.Cleanup(func() { cdUID, cdOrg = origCdUID, origCdOrg }) + cdUID, cdOrg = "uid12345678/os", "" + + const in = "failed to reach the host uid12345678/os" + got := redactSecrets(in, provisionSecrets()...) + + if !strings.Contains(got, "failed to reach the host ") { + t.Fatalf("the redaction mangled the words of the message: %s", got) + } + if strings.Contains(got, "uid12345678") { + t.Fatalf("the redaction kept the resolver uid: %s", got) + } +} diff --git a/cmd/cli/query_error_sampler.go b/cmd/cli/query_error_sampler.go new file mode 100644 index 0000000..70262b4 --- /dev/null +++ b/cmd/cli/query_error_sampler.go @@ -0,0 +1,178 @@ +package cli + +import ( + "sync" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// Classes of per-query errors. Each class keeps its own budget per upstream. +const ( + sampleClassResolveFailed = "resolve_failed" + sampleClassAllEndpointsFailed = "all_endpoints_failed" + sampleClassSendResponseFailed = "send_response_failed" + // sampleClassInternalDomain names a failure of an Internal Domain resolver. + // It stays apart from the other classes, because an endpoint away from the + // organization network cannot reach such a resolver. + sampleClassInternalDomain = "internal_domain" +) + +const ( + // querySampleLimit is the number of lines that stay at error level in one window. + querySampleLimit = 5 + // querySampleWindow is the length of a window. It starts at the first error of the window. + querySampleWindow = 60 * time.Second +) + +// sampleKey identifies one budget. +type sampleKey struct { + class string + upstream string +} + +// sampleWindow counts the events of one budget between its start and its end. +type sampleWindow struct { + start time.Time + count int + suppressed int + timer *time.Timer +} + +// ended reports whether the window closed at or before now. +func (w *sampleWindow) ended(now time.Time) bool { + return !now.Before(w.start.Add(querySampleWindow)) +} + +// sampleSummary holds what one closed window reports. +type sampleSummary struct { + key sampleKey + count int + suppressed int +} + +// errorSampler bounds the noise of a repeated per-query error. It lowers the +// level of a line, never its presence, so the debug stream stays complete. +// The console and the log_path file show the lowered lines at debug level only. +// The zero value is ready to use. +type errorSampler struct { + mu sync.Mutex + windows map[sampleKey]*sampleWindow + now func() time.Time + after func(time.Duration, func()) *time.Timer + // onFailure counts the event for the query health tracker. The daemon sets + // it once at start, before a listener serves a query. + onFailure func(class string) +} + +// event returns the event a call site logs with. The first querySampleLimit +// events of a window come back at error level, the later ones at debug level. +// The caller adds its own fields and the message. +func (s *errorSampler) event(class, upstream string) *ctrld.LogEvent { + key := sampleKey{class: class, upstream: upstream} + + s.mu.Lock() + now := s.clock() + var summaries []sampleSummary + window := s.windows[key] + if window != nil && window.ended(now) { + summaries = appendWindowSummary(summaries, key, window) + s.dropWindow(key, window) + window = nil + } + if window == nil { + window = s.startWindow(key, now) + } + window.count++ + suppressed := window.count > querySampleLimit + if suppressed { + window.suppressed++ + } + s.mu.Unlock() + + // A lowered line keeps its place in the breakdown of the window. It is one + // event, not one failed query, so it does not change the grade. + if s.onFailure != nil { + s.onFailure(class) + } + s.logSummaries(summaries) + + if suppressed { + return mainLog.Load().Debug() + } + return mainLog.Load().Error() +} + +// closeExpired closes every window that ended at or before now. The window +// timer calls it, so an idle window reports without a new event. +func (s *errorSampler) closeExpired(now time.Time) { + s.mu.Lock() + var summaries []sampleSummary + for key, window := range s.windows { + if !window.ended(now) { + continue + } + summaries = appendWindowSummary(summaries, key, window) + s.dropWindow(key, window) + } + s.mu.Unlock() + + s.logSummaries(summaries) +} + +// startWindow opens a window and arms the timer that closes it. The caller holds the lock. +func (s *errorSampler) startWindow(key sampleKey, start time.Time) *sampleWindow { + if s.windows == nil { + s.windows = make(map[sampleKey]*sampleWindow) + } + window := &sampleWindow{start: start} + s.windows[key] = window + window.timer = s.afterFunc()(querySampleWindow, func() { s.closeExpired(s.clock()) }) + return window +} + +// dropWindow forgets a window and stops its timer. The caller holds the lock. +func (s *errorSampler) dropWindow(key sampleKey, window *sampleWindow) { + if window.timer != nil { + window.timer.Stop() + } + delete(s.windows, key) +} + +// logSummaries reports the closed windows. The caller released the lock first, +// because the logger writes back through code that can take it again. +func (s *errorSampler) logSummaries(summaries []sampleSummary) { + for _, summary := range summaries { + mainLog.Load().Error(). + Str("class", summary.key.class). + Str("upstream", journalUpstreamName(summary.key.upstream)). + Int("count", summary.count). + Int("suppressed", summary.suppressed). + Int("window_s", int(querySampleWindow/time.Second)). + Msg("Per-query errors sampled") + } +} + +// clock reads the time of the sampler. A test sets its own clock, so a window +// closes without a wait. +func (s *errorSampler) clock() time.Time { + if s.now != nil { + return s.now() + } + return time.Now() +} + +func (s *errorSampler) afterFunc() func(time.Duration, func()) *time.Timer { + if s.after != nil { + return s.after + } + return time.AfterFunc +} + +// appendWindowSummary keeps the windows that suppressed at least one line. +func appendWindowSummary(summaries []sampleSummary, key sampleKey, window *sampleWindow) []sampleSummary { + if window.suppressed == 0 { + return summaries + } + return append(summaries, sampleSummary{key: key, count: window.count, suppressed: window.suppressed}) +} diff --git a/cmd/cli/query_error_sampler_test.go b/cmd/cli/query_error_sampler_test.go new file mode 100644 index 0000000..4eca36e --- /dev/null +++ b/cmd/cli/query_error_sampler_test.go @@ -0,0 +1,274 @@ +package cli + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +// samplerTestMessage is the message the fake call site writes on every sampled event. +const samplerTestMessage = "Failed to resolve query" + +// samplerSummaryMessage is the message of the window summary. +const samplerSummaryMessage = "Per-query errors sampled" + +// captureDebugMainLog sends mainLog into a JSON buffer at debug level, so the +// test also sees the lines that the code under test drops to debug. It swaps +// a global, so no test of this package runs in parallel. +func captureDebugMainLog(t *testing.T) *syncBuffer { + t.Helper() + return captureJSONMainLog(t) +} + +// jsonLogEvents parses the captured lines as JSON events, in order. An empty +// message takes every line. +func jsonLogEvents(t *testing.T, logs *syncBuffer, message string) []map[string]any { + t.Helper() + var events []map[string]any + for _, line := range strings.Split(logs.String(), "\n") { + if strings.TrimSpace(line) == "" { + continue + } + event := map[string]any{} + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("log line is not JSON: %q: %v", line, err) + } + if message != "" && event["message"] != message { + continue + } + events = append(events, event) + } + return events +} + +// samplerHarness drives an errorSampler on a clock and a timer queue the test owns. +type samplerHarness struct { + sampler *errorSampler + logs *syncBuffer + now time.Time + pending []func() +} + +func newSamplerHarness(t *testing.T) *samplerHarness { + t.Helper() + h := &samplerHarness{ + logs: captureDebugMainLog(t), + now: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + } + h.sampler = &errorSampler{ + now: func() time.Time { return h.now }, + after: func(_ time.Duration, fn func()) *time.Timer { + h.pending = append(h.pending, fn) + return nil + }, + } + return h +} + +// emit plays a call site: it takes an event from the sampler and adds its own fields. +func (h *samplerHarness) emit(class, upstream string) { + h.sampler.event(class, upstream).Str("class", class).Str("upstream", upstream).Msg(samplerTestMessage) +} + +func (h *samplerHarness) advance(d time.Duration) { + h.now = h.now.Add(d) +} + +func (h *samplerHarness) events(t *testing.T) []map[string]any { + t.Helper() + return jsonLogEvents(t, h.logs, "") +} + +func countSampled(events []map[string]any, level, upstream string) int { + count := 0 + for _, event := range events { + if event["message"] != samplerTestMessage { + continue + } + if event["level"] != level || event["upstream"] != upstream { + continue + } + count++ + } + return count +} + +func samplerSummaries(events []map[string]any) []map[string]any { + var summaries []map[string]any + for _, event := range events { + if event["message"] == samplerSummaryMessage { + summaries = append(summaries, event) + } + } + return summaries +} + +func wantField(t *testing.T, event map[string]any, field string, want any) { + t.Helper() + if got := event[field]; got != want { + t.Fatalf("field %q: got %v, want %v", field, got, want) + } +} + +func Test_errorSamplerLimitsWindowAndSummarizes(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+2; i++ { + h.emit(sampleClassResolveFailed, "upstream.0") + } + + events := h.events(t) + if got := countSampled(events, "error", "upstream.0"); got != querySampleLimit { + t.Fatalf("error lines: got %d, want %d", got, querySampleLimit) + } + if got := countSampled(events, "debug", "upstream.0"); got != 2 { + t.Fatalf("debug lines: got %d, want 2", got) + } + if got := len(samplerSummaries(events)); got != 0 { + t.Fatalf("summaries before the window closed: got %d, want 0", got) + } + + h.advance(querySampleWindow) + h.emit(sampleClassResolveFailed, "upstream.0") + + events = h.events(t) + summaries := samplerSummaries(events) + if len(summaries) != 1 { + t.Fatalf("summaries after the window closed: got %d, want 1", len(summaries)) + } + summary := summaries[0] + wantField(t, summary, "level", "error") + wantField(t, summary, "class", sampleClassResolveFailed) + wantField(t, summary, "upstream", "upstream.0") + wantField(t, summary, "count", float64(7)) + wantField(t, summary, "suppressed", float64(2)) + wantField(t, summary, "window_s", float64(60)) + + last := events[len(events)-1] + wantField(t, last, "message", samplerTestMessage) + wantField(t, last, "level", "error") +} + +func Test_errorSamplerKeepsBudgetPerUpstream(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+1; i++ { + h.emit(sampleClassResolveFailed, "upstream.0") + } + for i := 0; i < querySampleLimit; i++ { + h.emit(sampleClassResolveFailed, "upstream.1") + } + + events := h.events(t) + if got := countSampled(events, "error", "upstream.1"); got != querySampleLimit { + t.Fatalf("error lines for the second upstream: got %d, want %d", got, querySampleLimit) + } + if got := countSampled(events, "debug", "upstream.1"); got != 0 { + t.Fatalf("debug lines for the second upstream: got %d, want 0", got) + } + if got := countSampled(events, "debug", "upstream.0"); got != 1 { + t.Fatalf("debug lines for the first upstream: got %d, want 1", got) + } +} + +func Test_errorSamplerKeepsQuietWindowSilent(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < 3; i++ { + h.emit(sampleClassAllEndpointsFailed, "") + } + + h.advance(querySampleWindow) + h.sampler.closeExpired(h.now) + + if got := len(samplerSummaries(h.events(t))); got != 0 { + t.Fatalf("summaries for a window without suppressed lines: got %d, want 0", got) + } +} + +func Test_errorSamplerCloseExpiredSummarizesOnce(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+2; i++ { + h.emit(sampleClassSendResponseFailed, "") + } + sampled := len(h.events(t)) + + h.advance(querySampleWindow) + h.sampler.closeExpired(h.now) + + events := h.events(t) + if len(events) != sampled+1 { + t.Fatalf("lines after closeExpired: got %d, want %d", len(events), sampled+1) + } + summaries := samplerSummaries(events) + if len(summaries) != 1 { + t.Fatalf("summaries after closeExpired: got %d, want 1", len(summaries)) + } + wantField(t, summaries[0], "class", sampleClassSendResponseFailed) + wantField(t, summaries[0], "suppressed", float64(2)) + + h.sampler.closeExpired(h.now.Add(querySampleWindow)) + + if got := len(h.events(t)); got != sampled+1 { + t.Fatalf("lines after the second closeExpired: got %d, want %d", got, sampled+1) + } +} + +func Test_errorSamplerTimerClosesIdleWindow(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+1; i++ { + h.emit(sampleClassResolveFailed, "upstream.0") + } + if len(h.pending) != 1 { + t.Fatalf("window timers: got %d, want 1", len(h.pending)) + } + + h.advance(querySampleWindow) + h.pending[0]() + + summaries := samplerSummaries(h.events(t)) + if len(summaries) != 1 { + t.Fatalf("summaries after the timer fired: got %d, want 1", len(summaries)) + } + wantField(t, summaries[0], "suppressed", float64(1)) + wantField(t, summaries[0], "count", float64(6)) +} + +func Test_errorSamplerZeroValueWorks(t *testing.T) { + buf := captureDebugMainLog(t) + + var sampler errorSampler + // The zero value arms a real timer. Close its window after the assertions, + // so no late summary line reaches another test. + t.Cleanup(func() { sampler.closeExpired(time.Now().Add(querySampleWindow)) }) + sampler.event(sampleClassResolveFailed, "upstream.0").Msg(samplerTestMessage) + + if got := strings.Count(buf.String(), samplerTestMessage); got != 1 { + t.Fatalf("lines from the zero value sampler: got %d, want 1", got) + } + if !strings.Contains(buf.String(), `"level":"error"`) { + t.Fatalf("first line of a window is not at error level: %s", buf.String()) + } +} + +// Test_errorSamplerKeepsAnEmptyUpstreamEmpty drives a window of a class that +// names no upstream. The summary must not report the window as a custom +// upstream. +func Test_errorSamplerKeepsAnEmptyUpstreamEmpty(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+1; i++ { + h.emit(sampleClassAllEndpointsFailed, "") + } + h.advance(querySampleWindow) + h.sampler.closeExpired(h.now) + + summaries := samplerSummaries(h.events(t)) + if len(summaries) != 1 { + t.Fatalf("summaries after the window closed: got %d, want 1", len(summaries)) + } + wantField(t, summaries[0], "upstream", "") +} diff --git a/cmd/cli/query_health.go b/cmd/cli/query_health.go new file mode 100644 index 0000000..7fd84aa --- /dev/null +++ b/cmd/cli/query_health.go @@ -0,0 +1,284 @@ +package cli + +import ( + "maps" + "slices" + "sync" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// Health classes of the query path. The names go into the journal. +const ( + queryHealthHealthy = "healthy" + queryHealthDegraded = "degraded" + queryHealthFailing = "failing" +) + +const ( + // queryHealthWindow is the length of one counting window and the time + // between two heartbeats. + queryHealthWindow = 15 * time.Minute + // queryHealthFailingRatio and queryHealthDegradedRatio grade the failed + // share of a window. + queryHealthFailingRatio = 0.5 + queryHealthDegradedRatio = 0.05 + // queryHealthFailingQueries is the smallest sample that can leave the + // healthy class and the smallest sample that can report a class change. + queryHealthFailingQueries = 10 + // queryHealthClassTicks is the number of ticks in a row that a new class + // must hold before it reports. + queryHealthClassTicks = 2 + // queryHealthMessage names the journal event of the tracker. + queryHealthMessage = "Query health" +) + +// queryHealthTick is the time between two evaluations. A test shortens it. +var queryHealthTick = time.Minute + +// queryHealth grades the query path over a window. Support reads the grade +// after an incident, so the journal needs one class per window instead of a +// line per query. +type queryHealth struct { + mu sync.Mutex + window time.Duration + now func() time.Time + windowStart time.Time + queries uint64 + cacheHits uint64 + // failedQueries counts the client queries that ended without an answer. + // The grade follows this count, and failures holds the sampler classes + // behind it, which raise several events for one failed query. + failedQueries uint64 + failures map[string]uint64 + lastClass string + // pendingClass is the class that the last tick graded, and pendingTicks + // counts how many ticks in a row graded it. A class reports only after it + // held, so one noisy minute opens no event. + pendingClass string + pendingTicks int +} + +// queryHealthEvent holds the class and the counters of one window. +type queryHealthEvent struct { + Class string + WindowSeconds int + Queries uint64 + CacheHits uint64 + FailedQueries uint64 + Failures map[string]uint64 + UpstreamsDown int + BypassActive bool + CacheHitRatio float64 +} + +func newQueryHealth() *queryHealth { + return &queryHealth{ + window: queryHealthWindow, + now: time.Now, + failures: make(map[string]uint64), + } +} + +// A prog outside a run holds no tracker, so every counter of the query path +// and every loop accepts a nil receiver. +func (q *queryHealth) countQuery() { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + q.queries++ +} + +func (q *queryHealth) countCacheHit() { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + q.cacheHits++ +} + +// countFailedQuery counts one client query that ended without an answer. A +// query that failed on several upstreams counts once, because the grade +// measures the service the client got. +func (q *queryHealth) countFailedQuery() { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + q.failedQueries++ +} + +func (q *queryHealth) countFailure(class string) { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + if q.failures == nil { + q.failures = make(map[string]uint64) + } + q.failures[class]++ +} + +// evaluate grades the counters and reports whether the event goes out. The +// report covers the window that just closed, or the running one. The window +// starts at the first evaluate, so the caller owns the clock. +func (q *queryHealth) evaluate(now time.Time, upstreamsDown int, bypassActive bool) (queryHealthEvent, bool) { + if q == nil { + return queryHealthEvent{}, false + } + q.mu.Lock() + defer q.mu.Unlock() + + if q.windowStart.IsZero() { + q.windowStart = now + } + event := queryHealthEvent{ + Class: healthClass(q.queries, q.failedQueries), + WindowSeconds: int(q.window / time.Second), + Queries: q.queries, + CacheHits: q.cacheHits, + FailedQueries: q.failedQueries, + Failures: copyFailures(q.failures), + UpstreamsDown: upstreamsDown, + BypassActive: bypassActive, + CacheHitRatio: shareOf(q.cacheHits, q.queries), + } + // The end of a window always reports, so a quiet daemon stays visible and + // no window is lost after an early report. A class change waits for a + // sample and for a second tick. + windowEnded := !now.Before(q.windowStart.Add(q.window)) + emit := q.lastClass == "" || windowEnded || q.classChangeHolds(event.Class) + if emit { + q.lastClass = event.Class + q.pendingClass, q.pendingTicks = "", 0 + } + if windowEnded { + q.startWindow(now) + } + return event, emit +} + +// classChangeHolds reports whether a new class may open an event. It needs a +// window with enough queries, and the same class on two ticks in a row. The +// caller holds the lock. +func (q *queryHealth) classChangeHolds(class string) bool { + if class == q.lastClass || q.queries < queryHealthFailingQueries { + q.pendingClass, q.pendingTicks = "", 0 + return false + } + if class != q.pendingClass { + q.pendingClass, q.pendingTicks = class, 1 + return false + } + q.pendingTicks++ + return q.pendingTicks >= queryHealthClassTicks +} + +// startWindow drops the counters of the window that closed. Each report covers +// one window, so a quiet hour cannot hide inside the totals of a busy one. The +// caller holds the lock. +func (q *queryHealth) startWindow(now time.Time) { + q.windowStart = now + q.queries = 0 + q.cacheHits = 0 + q.failedQueries = 0 + q.failures = make(map[string]uint64) +} + +func (q *queryHealth) log(event queryHealthEvent) { + if q == nil { + return + } + journal(mainLog.Load().Info()). + Str("class", event.Class). + Int("window_s", event.WindowSeconds). + Uint64("queries", event.Queries). + Uint64("cache_hits", event.CacheHits). + Uint64("failed_queries", event.FailedQueries). + Dict("failures_by_class", failuresDict(event.Failures)). + Int("upstreams_down", event.UpstreamsDown). + Bool("bypass_active", event.BypassActive). + Float64("cache_hit_ratio", event.CacheHitRatio). + Msg(queryHealthMessage) +} + +// startLoop runs the grading loop and returns a channel that closes when the +// loop returns. A caller joins that channel before it reads the log. +func (q *queryHealth) startLoop(stop <-chan struct{}, inputs func() (upstreamsDown int, bypassActive bool)) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) + q.loop(stop, inputs) + }() + return done +} + +// loop grades the counters on every tick until the caller stops it. +func (q *queryHealth) loop(stop <-chan struct{}, inputs func() (upstreamsDown int, bypassActive bool)) { + if q == nil { + return + } + ticker := time.NewTicker(queryHealthTick) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + upstreamsDown, bypassActive := inputs() + event, emit := q.evaluate(q.now(), upstreamsDown, bypassActive) + if !emit { + continue + } + q.log(event) + } + } +} + +// healthClass grades the failed share of a window. A small sample stays +// healthy, because a few failures at a start are not an outage. +func healthClass(queries, failures uint64) string { + if queries < queryHealthFailingQueries { + return queryHealthHealthy + } + failed := shareOf(failures, queries) + if failed >= queryHealthFailingRatio { + return queryHealthFailing + } + if failed >= queryHealthDegradedRatio { + return queryHealthDegraded + } + return queryHealthHealthy +} + +// shareOf reports the part of the total, and 0 for an empty total. +func shareOf(part, total uint64) float64 { + if total == 0 { + return 0 + } + return float64(part) / float64(total) +} + +// copyFailures gives the report its own map, because the counters keep running +// while the caller holds the report. +func copyFailures(failures map[string]uint64) map[string]uint64 { + copied := make(map[string]uint64, len(failures)) + maps.Copy(copied, failures) + return copied +} + +// failuresDict renders the classes in a stable order, so two lines of the same +// window compare by eye. +func failuresDict(failures map[string]uint64) *ctrld.LogEvent { + dict := ctrld.Dict() + for _, class := range slices.Sorted(maps.Keys(failures)) { + dict = dict.Uint64(class, failures[class]) + } + return dict +} diff --git a/cmd/cli/query_health_prog_test.go b/cmd/cli/query_health_prog_test.go new file mode 100644 index 0000000..5e02973 --- /dev/null +++ b/cmd/cli/query_health_prog_test.go @@ -0,0 +1,467 @@ +package cli + +import ( + "context" + "errors" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/stretchr/testify/require" + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/dnscache" + "github.com/Control-D-Inc/ctrld/testhelper" +) + +// wantDNSConfigStrings compares one array field of a logged resolver entry. +func wantDNSConfigStrings(t *testing.T, event map[string]any, field string, want ...string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("field %q: got %T, want an array", field, event[field]) + } + if len(values) != len(want) { + t.Fatalf("field %q: got %v, want %v", field, values, want) + } + for i, value := range values { + if value != want[i] { + t.Fatalf("field %q at %d: got %v, want %v", field, i, value, want[i]) + } + } +} + +// Test_queryHealthCountsFailedProxyQueries drives the real query path against a +// refusing upstream: every sampled error reaches the tracker, so the window of +// the daemon grades as failing. +func Test_queryHealthCountsFailedProxyQueries(t *testing.T) { + captureDebugMainLog(t) + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load()), health: newQueryHealth()} + p.logger.Store(mainLog.Load()) + // A new monitor holds back the failure counts for one second, and its + // timer would mark the upstream down after the test. + p.um.clearRecovered(upstreamOS) + p.um.after = func(time.Duration, func()) {} + p.querySampler.onFailure = p.health.countFailure + // The zero value sampler arms a real timer. Close its window after the + // assertions, so no late summary line reaches another test. + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + origOSUpstream := osUpstreamConfig + t.Cleanup(func() { osUpstreamConfig = origOSUpstream }) + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "dead resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 500, + } + + const failedQueries = 12 + const servedQueries = 20 + ctx := context.Background() + for i := 0; i < failedQueries; i++ { + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeServerFailure { + t.Fatalf("query %d: rcode %s, want SERVFAIL", i+1, dns.RcodeToString[rcode]) + } + } + // serveDNS counts every query it serves, and the eight answered ones never + // reach the sampler. + for i := 0; i < servedQueries; i++ { + p.health.countQuery() + } + + event, emit := p.health.evaluate(time.Now(), 0, false) + if !emit { + t.Fatal("the first evaluate did not emit") + } + if event.Class != queryHealthFailing { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthFailing) + } + if event.Queries != servedQueries { + t.Fatalf("queries: got %d, want %d", event.Queries, servedQueries) + } + // The grade follows one failure per failed query. Each of those queries + // also raised two sampler events, and the breakdown still shows them. + if event.FailedQueries != failedQueries { + t.Fatalf("failed queries: got %d, want %d", event.FailedQueries, failedQueries) + } + if got := event.Failures[sampleClassResolveFailed]; got != failedQueries { + t.Fatalf("%s failures: got %d, want %d", sampleClassResolveFailed, got, failedQueries) + } + if got := event.Failures[sampleClassAllEndpointsFailed]; got != failedQueries { + t.Fatalf("%s failures: got %d, want %d", sampleClassAllEndpointsFailed, got, failedQueries) + } +} + +// Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade drives the same +// number of failed queries against an Internal Domain resolver. An endpoint +// away from the organization network cannot reach that resolver, so the class +// stays healthy, the failures keep their own name, and the down entry of that +// resolver stays out of upstreams_down. +func Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade(t *testing.T) { + captureDebugMainLog(t) + const internalUpstream = upstreamPrefix + internalDomainUpstreamPrefix + "1" + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + cfg.Upstream = map[string]*ctrld.UpstreamConfig{ + internalDomainUpstreamPrefix + "1": { + Name: "dead internal resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 500, + }, + } + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load()), health: newQueryHealth()} + p.logger.Store(mainLog.Load()) + p.um.after = func(time.Duration, func()) {} + p.um.clearRecovered(internalUpstream) + p.querySampler.onFailure = p.health.countFailure + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + const failedQueries = 12 + const servedQueries = 20 + ctx := context.Background() + msg := new(dns.Msg) + msg.SetQuestion("internal.corp.example.", dns.TypeA) + for i := 0; i < failedQueries; i++ { + res := p.proxy(ctx, &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{upstreams: []string{internalUpstream}, srcAddr: "192.168.0.1:1234"}, + }) + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeServerFailure { + t.Fatalf("query %d: the internal resolver answered", i+1) + } + } + for i := 0; i < servedQueries; i++ { + p.health.countQuery() + } + p.um.mu.Lock() + p.um.markDown(internalUpstream, 3, "immediate") + p.um.mu.Unlock() + + event, emit := p.health.evaluate(time.Now(), p.um.countDownExcept(isInternalDomainUpstream), false) + if !emit { + t.Fatal("the first evaluate did not emit") + } + if event.Class != queryHealthHealthy { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthHealthy) + } + if event.FailedQueries != 0 { + t.Fatalf("failed queries: got %d, want 0", event.FailedQueries) + } + if got := event.Failures[sampleClassInternalDomain]; got != failedQueries { + t.Fatalf("%s failures: got %d, want %d", sampleClassInternalDomain, got, failedQueries) + } + if event.UpstreamsDown != 0 { + t.Fatalf("upstreams_down: got %d, want 0", event.UpstreamsDown) + } + if got := p.um.countDown(); got != 1 { + t.Fatalf("countDown: got %d, want 1", got) + } +} + +// Test_queryHealthCountsProxyCacheHits proves that an answer from the cache +// counts a cache hit, so the ratio of the journal follows the query path. +func Test_queryHealthCountsProxyCacheHits(t *testing.T) { + captureDebugMainLog(t) + p := &prog{cfg: testhelper.SampleConfig(t), health: newQueryHealth()} + p.logger.Store(mainLog.Load()) + cache, err := dnscache.NewLRUCache(16) + require.NoError(t, err) + p.cache = cache + + msg := new(dns.Msg) + msg.SetQuestion("cache.test.", dns.TypeA) + answer := new(dns.Msg) + answer.SetRcode(msg, dns.RcodeSuccess) + p.cache.Add(dnscache.NewKey(msg, "upstream.1"), dnscache.NewValue(answer, time.Now().Add(time.Minute))) + + res := p.proxy(context.Background(), &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{upstreams: []string{"upstream.1"}}, + }) + require.NotNil(t, res) + require.True(t, res.cached, "the answer did not come from the cache") + + event, _ := p.health.evaluate(time.Now(), 0, false) + if event.CacheHits != 1 { + t.Fatalf("cache hits: got %d, want 1", event.CacheHits) + } +} + +// Test_logDNSConfigChangesLogsEveryEntry proves that each changed resolver of +// the host reaches the journal with its fields. A resolver that vanished keeps +// an empty nameserver list. +func Test_logDNSConfigChangesLogsEveryEntry(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + p.logger.Store(mainLog.Load()) + + p.logDNSConfigChanges([]dnsResolverEntry{ + { + Order: 200000, + Nameservers: []string{"192.168.50.200", "192.168.50.1"}, + IfIndex: 16, + Interface: "en0", + Flags: "Request A records, Request AAAA records", + SearchDomains: []string{"corp.example"}, + Reachable: "Reachable,Directly Reachable Address", + }, + {Order: 101600, IfIndex: 21, Interface: "utun4", Flags: "Scoped"}, + }) + + events := jsonLogEvents(t, logs, dnsConfigChangedMessage) + if len(events) != 2 { + t.Fatalf("configuration lines: got %d, want 2", len(events)) + } + + wantField(t, events[0], "journal", true) + wantField(t, events[0], "level", "info") + wantField(t, events[0], "if_index", float64(16)) + wantField(t, events[0], "interface", "en0") + wantField(t, events[0], "flags", "Request A records, Request AAAA records") + wantField(t, events[0], "order", float64(200000)) + wantField(t, events[0], "reachable", "Reachable,Directly Reachable Address") + wantDNSConfigStrings(t, events[0], "nameservers", "192.168.50.200", "192.168.50.1") + // The suffixes of an organization stay out of the journal. Only their + // number goes in. + wantField(t, events[0], "search_domain_count", float64(1)) + if _, ok := events[0]["search_domains"]; ok { + t.Fatal("the journal holds the search domains") + } + + wantField(t, events[1], "journal", true) + wantField(t, events[1], "interface", "utun4") + wantDNSConfigStrings(t, events[1], "nameservers") + wantField(t, events[1], "search_domain_count", float64(0)) +} + +// Test_startNetworkJournalLogsTheStartSnapshot proves that the daemon opens the +// journal with one snapshot and runs the trackers that follow the query path +// and the DNS configuration of the host. +func Test_startNetworkJournalLogsTheStartSnapshot(t *testing.T) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + origTick := queryHealthTick + t.Cleanup(func() { queryHealthTick = origTick }) + queryHealthTick = time.Millisecond + + var polls atomic.Int64 + // The poll loop reads the delay on every pass, so it parks on a long delay + // before the test puts the package value back. + parked := make(chan struct{}, 1) + origRunner, origDelay := runSCUtilDNSFn, dnsConfigDelayFn + t.Cleanup(func() { runSCUtilDNSFn, dnsConfigDelayFn = origRunner, origDelay }) + movedResolver := wifiResolverEntry() + movedResolver.Nameservers = []string{"10.0.0.1"} + runSCUtilDNSFn = func() ([]dnsResolverEntry, error) { + if polls.Add(1) == 1 { + return []dnsResolverEntry{wifiResolverEntry()}, nil + } + return []dnsResolverEntry{movedResolver}, nil + } + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { + if polls.Load() < 2 { + return time.Millisecond + } + select { + case parked <- struct{}{}: + default: + } + return time.Hour + } + + cfg := &ctrld.Config{} + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load()), stopCh: make(chan struct{})} + p.logger.Store(mainLog.Load()) + // The monitor arms a real timer for each upstream it resets. Nothing in + // this test waits for one. + p.um.after = func(time.Duration, func()) {} + p.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.9/24")) + p.recoveryBypass.Store(true) + p.um.mu.Lock() + p.um.markDown(upstreamOS, 3, "timer") + p.um.mu.Unlock() + + p.newNetworkJournal() + if p.health == nil || p.querySampler.onFailure == nil { + t.Fatal("the trackers do not exist before the loops start") + } + // A counter of the query path works as soon as the trackers exist. + p.health.countQuery() + waitForJournalLoops := p.startNetworkJournal() + t.Cleanup(func() { + close(p.stopCh) + waitForJournalLoops() + }) + + events := jsonLogEvents(t, logs, networkSnapshotMessage) + if len(events) != 1 { + t.Fatalf("snapshot lines: got %d, want 1", len(events)) + } + wantField(t, events[0], "journal", true) + wantField(t, events[0], "level", "info") + wantField(t, events[0], "trigger", "start") + + waitForLogLine(t, logs, queryHealthMessage) + health := jsonLogEvents(t, logs, queryHealthMessage) + wantField(t, health[0], "upstreams_down", float64(1)) + wantField(t, health[0], "bypass_active", true) + + if runtime.GOOS != "darwin" { + if p.dnsConfig != nil { + t.Fatal("the DNS configuration poller runs outside macOS") + } + return + } + if p.dnsConfig == nil { + t.Fatal("macOS starts no DNS configuration poller") + } + waitForLogLine(t, logs, dnsConfigChangedMessage) + select { + case <-parked: + case <-time.After(5 * time.Second): + t.Fatal("the poll loop did not park") + } + resolvers := jsonLogEvents(t, logs, dnsConfigChangedMessage) + if len(resolvers) != 1 { + t.Fatalf("configuration lines: got %d, want 1", len(resolvers)) + } + wantField(t, resolvers[0], "interface", "en0") + wantDNSConfigStrings(t, resolvers[0], "nameservers", "10.0.0.1") +} + +// Test_startNetworkJournalReadsTheMonitorAcrossAReload replaces the upstream +// monitor while the health loop runs. A reload builds a new monitor, and the +// loop read the field of the prog with no lock, which the race detector +// reports. +func Test_startNetworkJournalReadsTheMonitorAcrossAReload(t *testing.T) { + captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + origTick := queryHealthTick + t.Cleanup(func() { queryHealthTick = origTick }) + queryHealthTick = time.Millisecond + + cfg := &ctrld.Config{} + p := &prog{cfg: cfg, stopCh: make(chan struct{})} + p.logger.Store(mainLog.Load()) + p.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.9/24")) + p.replaceUpstreamMonitor() + p.health = newQueryHealth() + + waitForJournalLoops := p.startNetworkJournal() + for i := 0; i < 30; i++ { + p.replaceUpstreamMonitor() + time.Sleep(time.Millisecond) + } + close(p.stopCh) + waitForJournalLoops() +} + +// Test_startNetworkJournalRunsAfterAFailedMonitorStart proves that a network +// monitor that cannot start leaves the journal running. The error line goes +// out, the start snapshot opens the journal, and the health loop reports. +func Test_startNetworkJournalRunsAfterAFailedMonitorStart(t *testing.T) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + origTick := queryHealthTick + t.Cleanup(func() { queryHealthTick = origTick }) + queryHealthTick = time.Millisecond + + origMonitor := newNetworkChangeMonitorFn + t.Cleanup(func() { newNetworkChangeMonitorFn = origMonitor }) + newNetworkChangeMonitorFn = func(func(string, ...any)) (networkChangeMonitor, error) { + return nil, errors.New("no network monitor here") + } + origRunner, origDelay := runSCUtilDNSFn, dnsConfigDelayFn + t.Cleanup(func() { runSCUtilDNSFn, dnsConfigDelayFn = origRunner, origDelay }) + runSCUtilDNSFn = func() ([]dnsResolverEntry, error) { + return []dnsResolverEntry{wifiResolverEntry()}, nil + } + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return time.Hour } + + cfg := &ctrld.Config{} + p := &prog{cfg: cfg, stopCh: make(chan struct{})} + p.logger.Store(mainLog.Load()) + p.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.9/24")) + p.replaceUpstreamMonitor() + p.upstreamMonitorNow().after = func(time.Duration, func()) {} + + p.newNetworkJournal() + p.health.countQuery() + p.health.mu.Lock() + queries := p.health.queries + p.health.mu.Unlock() + if queries != 1 { + t.Fatalf("queries after one count: got %d, want 1", queries) + } + + waitForJournalLoops := p.startNetworkMonitorAndJournal(context.Background()) + t.Cleanup(func() { + close(p.stopCh) + waitForJournalLoops() + }) + + failures := jsonLogEvents(t, logs, "Failed to start network monitoring") + if len(failures) != 1 { + t.Fatalf("monitor failure lines: got %d, want 1", len(failures)) + } + wantField(t, failures[0], "level", "error") + + snapshots := jsonLogEvents(t, logs, networkSnapshotMessage) + if len(snapshots) != 1 { + t.Fatalf("snapshot lines: got %d, want 1", len(snapshots)) + } + wantField(t, snapshots[0], "trigger", "start") + + waitForLogLine(t, logs, queryHealthMessage) +} + +// Test_queryHealthCountsNoFailureWhenAnUpstreamAnswers proves that a failure on +// one upstream does not grade the query. The client got an answer, so the +// failure stays in the breakdown only. The DNS64 companion lookup fails through +// the same call, so it cannot grade a query either. +func Test_queryHealthCountsNoFailureWhenAnUpstreamAnswers(t *testing.T) { + captureDebugMainLog(t) + fixture := startDNSFixture(t) + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + cfg.Upstream = map[string]*ctrld.UpstreamConfig{ + "0": {Name: "dead resolver", Type: ctrld.ResolverTypeLegacy, Endpoint: deadUpstreamEndpoint, Timeout: 500}, + "1": {Name: "live resolver", Type: ctrld.ResolverTypeLegacy, Endpoint: fixture.addr, Timeout: 2000}, + } + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load()), health: newQueryHealth()} + p.logger.Store(mainLog.Load()) + p.um.after = func(time.Duration, func()) {} + p.um.clearRecovered(upstreamPrefix + "0") + p.querySampler.onFailure = p.health.countFailure + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + msg := new(dns.Msg) + msg.SetQuestion("failover.test.", dns.TypeA) + res := p.proxy(context.Background(), &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{ + upstreams: []string{upstreamPrefix + "0", upstreamPrefix + "1"}, + srcAddr: "192.168.0.1:1234", + }, + }) + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Fatal("the second upstream did not answer") + } + p.health.countQuery() + + event, _ := p.health.evaluate(time.Now(), 0, false) + if event.FailedQueries != 0 { + t.Fatalf("failed queries: got %d, want 0", event.FailedQueries) + } + if got := event.Failures[sampleClassResolveFailed]; got != 1 { + t.Fatalf("%s failures: got %d, want 1", sampleClassResolveFailed, got) + } + if event.Class != queryHealthHealthy { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthHealthy) + } +} diff --git a/cmd/cli/query_health_test.go b/cmd/cli/query_health_test.go new file mode 100644 index 0000000..6a77583 --- /dev/null +++ b/cmd/cli/query_health_test.go @@ -0,0 +1,413 @@ +package cli + +import ( + "fmt" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// queryHealthBase is the start time of every tracker test. The tracker takes +// the time from the caller, so no test waits for a real clock. +var queryHealthBase = time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + +// feedQueryHealth counts queries, and one failed query for each failure of a +// class. The grade follows the failed queries, so a test that wants a class +// gives the failures that produced it. +func feedQueryHealth(q *queryHealth, queries int, failures map[string]int) { + for i := 0; i < queries; i++ { + q.countQuery() + } + for class, count := range failures { + for i := 0; i < count; i++ { + q.countFailure(class) + q.countFailedQuery() + } + } +} + +// waitForLogLine polls the buffer, because the loop logs from its own goroutine. +func waitForLogLine(t *testing.T, logs *syncBuffer, message string) { + t.Helper() + waitForLogLineOrExplain(t, logs, message, nil) +} + +// waitForLogLineOrExplain is waitForLogLine with extra text for the failure. +func waitForLogLineOrExplain(t *testing.T, logs *syncBuffer, message string, explain func() string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if strings.Contains(logs.String(), message) { + return + } + time.Sleep(time.Millisecond) + } + extra := "" + if explain != nil { + extra = " " + explain() + } + t.Fatalf("no %q line before the deadline (captured %q)%s", message, logs.String(), extra) +} + +// loggerSwapRecord is one change of the main logger. +type loggerSwapRecord struct { + at time.Duration + writer string +} + +// watchLoggerSwaps records every change of the main logger while the test +// runs, so a missing line names what took the logger. +func watchLoggerSwaps(t *testing.T) func() string { + t.Helper() + start := time.Now() + stop := make(chan struct{}) + done := make(chan struct{}) + var mu sync.Mutex + var records []loggerSwapRecord + go func() { + defer close(done) + lastLogger := mainLog.Load() + for { + select { + case <-stop: + return + default: + } + logger := mainLog.Load() + if logger != lastLogger { + mu.Lock() + records = append(records, loggerSwapRecord{time.Since(start), loggerWriterType(logger)}) + mu.Unlock() + lastLogger = logger + } + time.Sleep(50 * time.Microsecond) + } + }() + t.Cleanup(func() { close(stop); <-done }) + return func() string { + mu.Lock() + defer mu.Unlock() + parts := make([]string, 0, len(records)) + for _, r := range slices.Clone(records) { + parts = append(parts, fmt.Sprintf("{at=%s writer=%s}", r.at, r.writer)) + } + return fmt.Sprintf("swaps=%d %s current=%s", len(records), strings.Join(parts, " "), loggerWriterType(mainLog.Load())) + } +} + +// loggerWriterType names the core behind a logger, so a swapped logger tells +// which init function built it. +func loggerWriterType(l *ctrld.Logger) string { + if l == nil || l.Logger == nil { + return "nil" + } + return fmt.Sprintf("%T", l.Core()) +} + +func Test_queryHealthClassThresholds(t *testing.T) { + // The class names go into the journal, so the test pins the text. + tests := []struct { + name string + queries int + failures int + want string + }{ + {"no queries", 0, 0, "healthy"}, + {"two percent", 100, 2, "healthy"}, + {"five percent", 100, 5, "degraded"}, + {"six percent", 100, 6, "degraded"}, + {"half of twenty", 20, 12, "failing"}, + {"half of ten", 10, 5, "failing"}, + {"small sample cannot fail", 9, 9, "healthy"}, + {"small sample cannot degrade", 9, 1, "healthy"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, tc.queries, map[string]int{sampleClassResolveFailed: tc.failures}) + + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit { + t.Fatal("the first evaluate did not emit") + } + if event.Class != tc.want { + t.Fatalf("class: got %q, want %q", event.Class, tc.want) + } + }) + } +} + +func Test_queryHealthEmitsHeartbeatWithoutClassChange(t *testing.T) { + q := newQueryHealth() + if _, emit := q.evaluate(queryHealthBase, 0, false); !emit { + t.Fatal("the first evaluate did not emit") + } + + for _, minutes := range []int{1, 7, 14} { + at := queryHealthBase.Add(time.Duration(minutes) * time.Minute) + if _, emit := q.evaluate(at, 0, false); emit { + t.Fatalf("emit at minute %d without a class change", minutes) + } + } + + event, emit := q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit { + t.Fatal("no heartbeat at 15 minutes") + } + if event.Class != "healthy" { + t.Fatalf("heartbeat class: got %q, want %q", event.Class, "healthy") + } + if _, emit := q.evaluate(queryHealthBase.Add(16*time.Minute), 0, false); emit { + t.Fatal("a second emit follows the heartbeat") + } +} + +func Test_queryHealthEmitsOnClassChange(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 100, nil) + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit || event.Class != "healthy" { + t.Fatalf("first report: emit %v, class %q", emit, event.Class) + } + + feedQueryHealth(q, 0, map[string]int{sampleClassResolveFailed: 50}) + if _, emit := q.evaluate(queryHealthBase.Add(time.Minute), 0, false); emit { + t.Fatal("emit on the first tick of the new class") + } + + event, emit = q.evaluate(queryHealthBase.Add(2*time.Minute), 0, false) + if !emit { + t.Fatal("no emit after the new class held two ticks") + } + if event.Class != "failing" { + t.Fatalf("class: got %q, want %q", event.Class, "failing") + } + + if _, emit := q.evaluate(queryHealthBase.Add(3*time.Minute), 0, false); emit { + t.Fatal("emit without a class change") + } +} + +// Test_queryHealthHoldsAClassChangeForTwoTicks proves that a class which lasts +// one tick reports nothing. A minute of noise must not open an event. +func Test_queryHealthHoldsAClassChangeForTwoTicks(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 20, nil) + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit || event.Class != queryHealthHealthy { + t.Fatalf("first report: emit %v, class %q", emit, event.Class) + } + + feedQueryHealth(q, 0, map[string]int{sampleClassResolveFailed: 12}) + if event, emit := q.evaluate(queryHealthBase.Add(time.Minute), 0, false); emit { + t.Fatalf("emit after one tick of class %q", event.Class) + } + + // The next tick grades the same window as degraded, so the failing class + // never held two ticks and never reported. + feedQueryHealth(q, 80, nil) + if event, emit := q.evaluate(queryHealthBase.Add(2*time.Minute), 0, false); emit { + t.Fatalf("emit after the class moved again to %q", event.Class) + } + + event, emit = q.evaluate(queryHealthBase.Add(3*time.Minute), 0, false) + if !emit { + t.Fatal("no emit after the new class held two ticks") + } + if event.Class != queryHealthDegraded { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthDegraded) + } +} + +// Test_queryHealthNeedsASampleBeforeAClassChange proves that the empty window +// which follows a heartbeat reports nothing. The counters start at zero, and +// zero queries say nothing about the query path. +func Test_queryHealthNeedsASampleBeforeAClassChange(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 20, map[string]int{sampleClassResolveFailed: 12}) + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit || event.Class != queryHealthFailing { + t.Fatalf("first report: emit %v, class %q", emit, event.Class) + } + + event, emit = q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit || event.Class != queryHealthFailing { + t.Fatalf("window end: emit %v, class %q", emit, event.Class) + } + + for _, minutes := range []int{16, 17} { + at := queryHealthBase.Add(time.Duration(minutes) * time.Minute) + if event, emit := q.evaluate(at, 0, false); emit { + t.Fatalf("emit at minute %d with %d queries and class %q", minutes, event.Queries, event.Class) + } + } +} + +func Test_queryHealthCacheHitRatio(t *testing.T) { + q := newQueryHealth() + event, _ := q.evaluate(queryHealthBase, 0, false) + if event.CacheHitRatio != 0 { + t.Fatalf("ratio without queries: got %v, want 0", event.CacheHitRatio) + } + + feedQueryHealth(q, 20, nil) + for i := 0; i < 5; i++ { + q.countCacheHit() + } + + event, _ = q.evaluate(queryHealthBase.Add(time.Minute), 0, false) + if event.Queries != 20 || event.CacheHits != 5 { + t.Fatalf("counters: got %d queries and %d cache hits, want 20 and 5", event.Queries, event.CacheHits) + } + if event.CacheHitRatio != 0.25 { + t.Fatalf("ratio: got %v, want 0.25", event.CacheHitRatio) + } +} + +func Test_queryHealthResetsCountersAtWindowEnd(t *testing.T) { + q := newQueryHealth() + if _, emit := q.evaluate(queryHealthBase, 0, false); !emit { + t.Fatal("the first evaluate did not emit") + } + feedQueryHealth(q, 20, map[string]int{sampleClassResolveFailed: 12}) + q.countCacheHit() + + closed, emit := q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit { + t.Fatal("no emit at the window end") + } + if closed.Queries != 20 || closed.CacheHits != 1 || closed.Failures[sampleClassResolveFailed] != 12 { + t.Fatalf("the closed window lost its counters: %+v", closed) + } + + q.countFailure(sampleClassResolveFailed) + next, _ := q.evaluate(queryHealthBase.Add(16*time.Minute), 0, false) + if next.Queries != 0 || next.CacheHits != 0 { + t.Fatalf("the counters survived the window end: %+v", next) + } + if next.Failures[sampleClassResolveFailed] != 1 { + t.Fatalf("failures of the new window: got %d, want 1", next.Failures[sampleClassResolveFailed]) + } +} + +func Test_queryHealthReportKeepsItsOwnFailures(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 10, map[string]int{sampleClassResolveFailed: 1}) + + report, _ := q.evaluate(queryHealthBase, 0, false) + q.countFailure(sampleClassResolveFailed) + + if got := report.Failures[sampleClassResolveFailed]; got != 1 { + t.Fatalf("the report shares the live failure map: got %d, want 1", got) + } +} + +func Test_queryHealthLogsJournalEvent(t *testing.T) { + swaps := watchLoggerSwaps(t) + logs := captureDebugMainLog(t) + q := newQueryHealth() + feedQueryHealth(q, 20, map[string]int{ + sampleClassResolveFailed: 8, + sampleClassAllEndpointsFailed: 4, + }) + for i := 0; i < 5; i++ { + q.countCacheHit() + } + + event, emit := q.evaluate(queryHealthBase, 2, true) + if !emit { + t.Fatal("the first evaluate did not emit") + } + q.log(event) + + events := jsonLogEvents(t, logs, queryHealthMessage) + if len(events) != 1 { + t.Fatalf("health lines: got %d, want 1 (captured %q) %s", len(events), logs.String(), swaps()) + } + logged := events[0] + wantField(t, logged, "journal", true) + wantField(t, logged, "level", "info") + wantField(t, logged, "class", "failing") + wantField(t, logged, "window_s", float64(900)) + wantField(t, logged, "queries", float64(20)) + wantField(t, logged, "cache_hits", float64(5)) + wantField(t, logged, "upstreams_down", float64(2)) + wantField(t, logged, "bypass_active", true) + wantField(t, logged, "cache_hit_ratio", 0.25) + + failures, ok := logged["failures_by_class"].(map[string]any) + if !ok { + t.Fatalf("failures_by_class: got %T, want an object", logged["failures_by_class"]) + } + if len(failures) != 2 { + t.Fatalf("failure classes: got %d, want 2", len(failures)) + } + wantField(t, failures, sampleClassResolveFailed, float64(8)) + wantField(t, failures, sampleClassAllEndpointsFailed, float64(4)) +} + +func Test_queryHealthLoopEmitsUntilStop(t *testing.T) { + swaps := watchLoggerSwaps(t) + logs := captureDebugMainLog(t) + tick := queryHealthTick + defer func() { queryHealthTick = tick }() + queryHealthTick = time.Millisecond + + q := newQueryHealth() + stop := make(chan struct{}) + done := make(chan struct{}) + var once sync.Once + stopLoop := func() { + once.Do(func() { + close(stop) + <-done + }) + } + defer stopLoop() + + go func() { + defer close(done) + q.loop(stop, func() (int, bool) { return 1, true }) + }() + + waitForLogLineOrExplain(t, logs, queryHealthMessage, swaps) + stopLoop() + + events := jsonLogEvents(t, logs, queryHealthMessage) + if len(events) == 0 { + t.Fatal("the loop logged no health event") + } + wantField(t, events[0], "journal", true) + wantField(t, events[0], "upstreams_down", float64(1)) + wantField(t, events[0], "bypass_active", true) +} + +// Test_queryHealthReportsEveryWindowAfterAnEarlyEmit proves that a class +// change in the middle of a window does not make the rest of the window +// vanish. The heartbeat follows the window, not the last report. +func Test_queryHealthReportsEveryWindowAfterAnEarlyEmit(t *testing.T) { + q := newQueryHealth() + if _, emit := q.evaluate(queryHealthBase, 0, false); !emit { + t.Fatal("the first evaluate did not emit") + } + feedQueryHealth(q, 20, map[string]int{sampleClassResolveFailed: 20}) + if _, emit := q.evaluate(queryHealthBase.Add(5*time.Minute), 0, false); emit { + t.Fatal("emit on the first tick of the new class") + } + event, emit := q.evaluate(queryHealthBase.Add(6*time.Minute), 0, false) + if !emit || event.Class != queryHealthFailing { + t.Fatalf("class change at 6 minutes: emit %v, class %q", emit, event.Class) + } + feedQueryHealth(q, 7, nil) + event, emit = q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit { + t.Fatal("the end of the window did not emit") + } + if event.Queries != 27 { + t.Fatalf("queries in the window report: got %d, want 27", event.Queries) + } +} diff --git a/cmd/cli/recovery_events_test.go b/cmd/cli/recovery_events_test.go new file mode 100644 index 0000000..edc7796 --- /dev/null +++ b/cmd/cli/recovery_events_test.go @@ -0,0 +1,352 @@ +package cli + +import ( + "context" + "reflect" + "strings" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// The two events that bound one recovery pass. +const ( + recoveryBeginMessage = "Recovery begin" + recoveryEndMessage = "Recovery end" +) + +// recoveredUpstreamName is the upstream that the stubbed probe reports. +const recoveredUpstreamName = upstreamPrefix + "0" + +// recoveryEventHarness runs one whole recovery pass. It stubs every seam of +// the pass, because a real pass probes an upstream, reads the host resolvers, +// and rewrites the PF anchor. +type recoveryEventHarness struct { + prog *prog + logs *syncBuffer + dhcpServers []string + target string + recovered string +} + +func newRecoveryEventHarness(t *testing.T) *recoveryEventHarness { + t.Helper() + h := &recoveryEventHarness{ + logs: captureDebugMainLog(t), + prog: &prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}, + dhcpServers: []string{"192.168.10.1"}, + target: "127.0.0.53", + recovered: recoveredUpstreamName, + } + h.prog.logger.Store(mainLog.Load()) + stubHeaderSnapshotSources(t) + h.prog.um = newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + h.prog.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.10/24")) + // A stabilizing host defers the DHCP exemption, so the pass rewrites no + // PF anchor. + h.prog.pfStabilizing.Store(true) + + intercept := dnsIntercept + discovery := initializeOsResolverWithSystemNameserversFn + ensure := ensureInterceptDNSTargetFn + wait := waitForUpstreamRecoveryFn + t.Cleanup(func() { + dnsIntercept = intercept + initializeOsResolverWithSystemNameserversFn = discovery + ensureInterceptDNSTargetFn = ensure + waitForUpstreamRecoveryFn = wait + }) + dnsIntercept = true + initializeOsResolverWithSystemNameserversFn = func(context.Context, bool, string) ([]string, []string) { + return effectiveNameserversOf(h.dhcpServers), h.dhcpServers + } + ensureInterceptDNSTargetFn = func(p *prog, _ []string) { + p.interceptDNSTargetMu.Lock() + defer p.interceptDNSTargetMu.Unlock() + p.interceptDNSTargetSetValue = h.target + p.publishInterceptTarget(h.target) + } + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + return h.recovered, nil + } + return h +} + +// effectiveNameserversOf models the OS resolver list that the root package +// builds: every discovered server with a port, and a synthetic public resolver +// when the network gave no public one. +func effectiveNameserversOf(discovered []string) []string { + effective := make([]string, 0, len(discovered)+1) + for _, server := range discovered { + effective = append(effective, server+":53") + } + return append(effective, "76.76.2.0:53") +} + +// oneRecoveryEvent returns the single event with the given message. +func oneRecoveryEvent(t *testing.T, logs *syncBuffer, message string) map[string]any { + t.Helper() + events := jsonLogEvents(t, logs, message) + if len(events) != 1 { + t.Fatalf("%q events: got %d, want 1", message, len(events)) + } + return events[0] +} + +// snapshotTriggers lists the trigger of every snapshot event, in order. +func snapshotTriggers(t *testing.T, logs *syncBuffer) []string { + t.Helper() + var triggers []string + for _, event := range jsonLogEvents(t, logs, networkSnapshotMessage) { + trigger, _ := event["trigger"].(string) + triggers = append(triggers, trigger) + } + return triggers +} + +func Test_recoveryCompletedJournalsBeginAndEnd(t *testing.T) { + h := newRecoveryEventHarness(t) + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + begin := oneRecoveryEvent(t, h.logs, recoveryBeginMessage) + wantField(t, begin, "journal", true) + wantField(t, begin, "level", "info") + + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + wantField(t, end, "journal", true) + wantField(t, end, "outcome", "completed") + wantField(t, end, "recovered_upstream", recoveredUpstreamName) + wantField(t, end, "bypass_active", false) + wantField(t, end, "intercept_target_action", "set") + wantField(t, end, "dhcp_server_count", float64(1)) + if got, want := end["dhcp_servers"], []any{"192.168.10.1"}; !reflect.DeepEqual(got, want) { + t.Fatalf("dhcp_servers = %v, want %v", got, want) + } + + // The network of this recovery does not change, so the end snapshot equals + // the begin snapshot and the writer drops it. + triggers := snapshotTriggers(t, h.logs) + if want := []string{"recovery_begin"}; !reflect.DeepEqual(triggers, want) { + t.Fatalf("snapshot triggers = %v, want %v", triggers, want) + } +} + +// Test_recoveryEndReportsTheDiscoveredDHCPNameservers drives the real OS +// resolver initialization. The effective list carries a synthetic public +// resolver, and a reader who counts it reads a DHCP server that the network +// never gave. +func Test_recoveryEndReportsTheDiscoveredDHCPNameservers(t *testing.T) { + h := newRecoveryEventHarness(t) + previousRead := ctrld.NameserversFn + ctrld.NameserversFn = func(context.Context) []string { return []string{"192.168.1.1"} } + t.Cleanup(func() { ctrld.NameserversFn = previousRead }) + initializeOsResolverWithSystemNameserversFn = ctrld.InitializeOsResolverWithSystemNameserversReason + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + wantField(t, end, "dhcp_server_count", float64(1)) + if got, want := end["dhcp_servers"], []any{"192.168.1.1"}; !reflect.DeepEqual(got, want) { + t.Fatalf("dhcp_servers = %v, want %v", got, want) + } +} + +// Test_recoveryJournalsTheReinitializedOSResolver covers the resolver line of +// a recovered network. The line names the nameservers that the recovery +// installed, and an outage report needs them. +func Test_recoveryJournalsTheReinitializedOSResolver(t *testing.T) { + h := newRecoveryEventHarness(t) + previousRead := ctrld.NameserversFn + ctrld.NameserversFn = func(context.Context) []string { return []string{"192.168.1.1"} } + t.Cleanup(func() { ctrld.NameserversFn = previousRead }) + + h.prog.handleRecoveryForTransition(RecoveryReasonNetworkChange, 0) + + event := oneRecoveryEvent(t, h.logs, "Reinitialized OS resolver with nameservers: [192.168.1.1:53 76.76.2.0:53]") + wantField(t, event, "journal", true) + wantField(t, event, "level", "info") +} + +// Test_recoveryEndSpeedsUpTheDNSConfigurationPoll covers the poll cadence +// after a recovery. The resolver table of the host settles late, and a poll +// every five minutes reports that change long after the outage. +func Test_recoveryEndSpeedsUpTheDNSConfigurationPoll(t *testing.T) { + h := newRecoveryEventHarness(t) + now := time.Date(2026, 9, 17, 9, 0, 0, 0, time.UTC) + previousNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return now } + t.Cleanup(func() { networkEventsNowFn = previousNow }) + h.prog.dnsConfig = newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil }) + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + if got := h.prog.dnsConfig.nextDelay(now); got != dnsConfigFastInterval { + t.Fatalf("DNS configuration poll delay after a recovery = %s, want %s", got, dnsConfigFastInterval) + } +} + +func Test_recoveryJournalsTheRecoveredUpstream(t *testing.T) { + h := newRecoveryEventHarness(t) + h.prog.um.mu.Lock() + h.prog.um.markDown(recoveredUpstreamName, 7, "timer") + h.prog.um.mu.Unlock() + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + event := oneRecoveryEvent(t, h.logs, "Upstream \""+recoveredUpstreamName+"\" recovered; re-applying DNS settings") + wantField(t, event, "journal", true) + wantField(t, event, "upstream", recoveredUpstreamName) + if _, ok := event["down_for_ms"]; !ok { + t.Fatalf("event has no down_for_ms field: %v", event) + } +} + +// Test_recoveryBoundsTheRecoveredUpstreamName covers an upstream key that +// holds a token. The key is operator text, and the journal leaves the host +// for a support case, so no line may carry it. +func Test_recoveryBoundsTheRecoveredUpstreamName(t *testing.T) { + const secret = "SECRET0123" + h := newRecoveryEventHarness(t) + h.recovered = upstreamPrefix + "doh-" + secret + h.prog.um.mu.Lock() + h.prog.um.markDown(h.recovered, 1, "timer") + h.prog.um.mu.Unlock() + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + const boundedName = upstreamPrefix + "custom" + event := oneRecoveryEvent(t, h.logs, "Upstream \""+boundedName+"\" recovered; re-applying DNS settings") + wantField(t, event, "upstream", boundedName) + wantField(t, oneRecoveryEvent(t, h.logs, recoveryEndMessage), "recovered_upstream", boundedName) + if strings.Contains(h.logs.String(), secret) { + t.Fatalf("a log line holds the upstream key: %s", h.logs.String()) + } +} + +func Test_recoveryJournalsTheDHCPBypassLines(t *testing.T) { + h := newRecoveryEventHarness(t) + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + for _, message := range []string{ + "DNS intercept recovery: enabling DHCP bypass (filters stay active)", + "DNS intercept recovery: found DHCP nameservers: [192.168.10.1]", + "DNS intercept recovery complete: disabling DHCP bypass, resuming normal flow", + } { + wantField(t, oneRecoveryEvent(t, h.logs, message), "journal", true) + } +} + +func Test_recoveryJournalsTheMissingDHCPNameservers(t *testing.T) { + h := newRecoveryEventHarness(t) + h.dhcpServers = nil + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + event := oneRecoveryEvent(t, h.logs, "DNS intercept recovery: no DHCP nameservers found") + wantField(t, event, "journal", true) + wantField(t, oneRecoveryEvent(t, h.logs, recoveryEndMessage), "dhcp_server_count", float64(0)) +} + +// wantOneRecoveryEndPerGeneration asserts the two lines that bound one pass: +// one end event and one snapshot beside it. +func wantOneRecoveryEndPerGeneration(t *testing.T, h *recoveryEventHarness) map[string]any { + t.Helper() + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + triggers := snapshotTriggers(t, h.logs) + // Nothing changes between the begin and the end of these passes, so the + // snapshot limiter drops the end snapshot as a repeat of the begin one. + if want := []string{"recovery_begin"}; !reflect.DeepEqual(triggers, want) { + t.Fatalf("snapshot triggers = %v, want %v", triggers, want) + } + return end +} + +// Test_recoveryCanceledEndReportsTheCanceledPass covers the canceled outcome. +// A canceled pass releases the bypass, and support reads the end event to tell +// a canceled pass from a pass that never ended. +func Test_recoveryCanceledEndReportsTheCanceledPass(t *testing.T) { + h := newRecoveryEventHarness(t) + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + return "", context.Canceled + } + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + end := wantOneRecoveryEndPerGeneration(t, h) + wantField(t, end, "outcome", "canceled") + wantField(t, end, "bypass_active", false) + wantField(t, end, "recovered_upstream", "") +} + +// Test_recoverySupersededEndReportsNoChange covers the superseded outcome. The +// successor owns the target and the bypass flag, so the end event of the +// superseded pass must not report the state of its successor. +func Test_recoverySupersededEndReportsNoChange(t *testing.T) { + h := newRecoveryEventHarness(t) + waitForUpstreamRecoveryFn = func(p *prog, _ context.Context, _ map[string]*ctrld.UpstreamConfig, _ *recoveryDiagnostic) (string, error) { + // A network change took ownership while the probe ran. + p.recoveryGen.Add(1) + return recoveredUpstreamName, nil + } + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + end := wantOneRecoveryEndPerGeneration(t, h) + wantField(t, end, "outcome", "superseded") + wantField(t, end, "bypass_active", false) + wantField(t, end, "intercept_target_action", "unchanged") +} + +func Test_recoveryCanceledWithNoSuccessorIsJournaled(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + p.logger.Store(mainLog.Load()) + setupInterceptRecovery(t, p) + + p.recoveryCanceledCleanup(p.recoveryGen.Add(1)) + + event := oneRecoveryEvent(t, logs, "Recovery canceled with no successor; cleared recovery state and DHCP bypass") + wantField(t, event, "journal", true) +} + +func Test_interceptTargetAction(t *testing.T) { + for _, tc := range []struct { + name string + before, after string + want string + }{ + {"no target at all", "", "", "unchanged"}, + {"target kept", "127.0.0.53", "127.0.0.53", "unchanged"}, + {"target written", "", "127.0.0.53", "set"}, + {"target moved", "127.0.0.53", "127.0.0.54", "set"}, + {"target dropped", "127.0.0.53", "", "removed"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := interceptTargetAction(tc.before, tc.after); got != tc.want { + t.Fatalf("interceptTargetAction(%q, %q) = %q, want %q", tc.before, tc.after, got, tc.want) + } + }) + } +} + +func Test_noteNAT64ResultReportsChangesOnly(t *testing.T) { + p := &prog{} + p.logger.Store(mainLog.Load()) + const prefix = "64:ff9b::/96" + + for _, tc := range []struct { + result string + want bool + }{ + {prefix, true}, + {prefix, false}, + {"", true}, + } { + if got := p.noteNAT64Result(tc.result); got != tc.want { + t.Fatalf("noteNAT64Result(%q) = %v, want %v", tc.result, got, tc.want) + } + } +} diff --git a/cmd/cli/recovery_state.go b/cmd/cli/recovery_state.go index e062925..2d3b66e 100644 --- a/cmd/cli/recovery_state.go +++ b/cmd/cli/recovery_state.go @@ -37,7 +37,7 @@ func (p *prog) recoveryOwnsState(gen uint64) bool { func (p *prog) systemNameserversForInterceptRetry() []string { loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) - _, system := initializeOsResolverWithSystemNameserversFn(loggerCtx, true) + _, system := initializeOsResolverWithSystemNameserversFn(loggerCtx, true, recoveryResolverReason) if system == nil { return []string{} } @@ -76,5 +76,5 @@ func (p *prog) recoveryCanceledCleanup(gen uint64) { // Superseded: the newer recovery owns the shared state. return } - p.Info().Msg("Recovery canceled with no successor; cleared recovery state and DHCP bypass") + journal(p.Info()).Msg("Recovery canceled with no successor; cleared recovery state and DHCP bypass") } diff --git a/cmd/cli/recovery_state_test.go b/cmd/cli/recovery_state_test.go index 9407e89..59b720e 100644 --- a/cmd/cli/recovery_state_test.go +++ b/cmd/cli/recovery_state_test.go @@ -147,7 +147,7 @@ func TestRecoveryCleanupClearsBypassAfterInterceptStateDisappears(t *testing.T) func TestSystemNameserversForInterceptRetryNormalizesEmptyDiscovery(t *testing.T) { original := initializeOsResolverWithSystemNameserversFn called := false - initializeOsResolverWithSystemNameserversFn = func(_ context.Context, guard bool) ([]string, []string) { + initializeOsResolverWithSystemNameserversFn = func(_ context.Context, guard bool, _ string) ([]string, []string) { called = true if !guard { t.Error("intercept retry discovery did not guard the existing resolver") diff --git a/cmd/cli/route_darwin.go b/cmd/cli/route_darwin.go new file mode 100644 index 0000000..15d81db --- /dev/null +++ b/cmd/cli/route_darwin.go @@ -0,0 +1,158 @@ +package cli + +import ( + "net" + "net/netip" + "syscall" + + "golang.org/x/net/route" +) + +// routeInterfaceNameFn names the interface of one route table index. A test +// replaces it, because the interfaces of the host are not a fixture. +var routeInterfaceNameFn = interfaceNameByIndex + +// defaultRoutes reads the default route of each family. It reads the kernel +// route table, because a snapshot follows every network change and a +// subprocess per change costs too much on a busy host. A table that the kernel +// does not give leaves both routes empty. +func defaultRoutes() (v4, v6 defaultRoute) { + rib, err := route.FetchRIB(syscall.AF_UNSPEC, route.RIBTypeRoute, 0) + if err != nil { + return defaultRoute{}, defaultRoute{} + } + messages, err := route.ParseRIB(route.RIBTypeRoute, rib) + if err != nil { + return defaultRoute{}, defaultRoute{} + } + return defaultRoutesFromMessages(messages) +} + +// defaultRoutesFromMessages picks the first default route of each family. The +// next hop and the interface come from one message, so a reader sees the pair +// that the kernel holds. The kernel keeps one unscoped default per family, so +// the first match is the only one. +func defaultRoutesFromMessages(messages []route.Message) (v4, v6 defaultRoute) { + foundV4, foundV6 := false, false + for _, message := range messages { + match, ok := matchDefaultRoute(message) + if !ok { + continue + } + found := defaultRoute{Interface: routeInterfaceNameFn(match.index)} + if match.gateway.IsValid() { + found.Gateway = match.gateway.String() + } + if match.v4 && !foundV4 { + v4, foundV4 = found, true + } + if !match.v4 && !foundV6 { + v6, foundV6 = found, true + } + if foundV4 && foundV6 { + return v4, v6 + } + } + return v4, v6 +} + +// defaultRouteMatch is one default route of the table. A tunnel default has +// no next hop, so gateway can be invalid while the interface is set. +type defaultRouteMatch struct { + v4 bool + gateway netip.Addr + index int +} + +// defaultRouteGateway reports the next hop and the interface index of one +// default route, for the tests of the selector. See matchDefaultRoute. +func defaultRouteGateway(message route.Message) (netip.Addr, int, bool) { + match, ok := matchDefaultRoute(message) + return match.gateway, match.index, ok +} + +// matchDefaultRoute reports whether one message is the default route of the +// host for its family. A route to another destination or to a part of the +// address space gives nothing: a VPN splits the default into two /1 routes, +// and the zero address alone does not make a route the default. A route bound +// to one interface serves that interface only, a host route is no default, and +// a reject or blackhole route delivers nothing. With the gateway flag, the +// next hop must be a host of the same family. Without it, the link itself +// delivers, which is how a tunnel owns the default; the match then has no +// gateway and names the interface. +func matchDefaultRoute(message route.Message) (defaultRouteMatch, bool) { + routeMessage, ok := message.(*route.RouteMessage) + if !ok || len(routeMessage.Addrs) <= syscall.RTAX_NETMASK { + return defaultRouteMatch{}, false + } + if routeMessage.Flags&(syscall.RTF_IFSCOPE|syscall.RTF_HOST|syscall.RTF_REJECT|syscall.RTF_BLACKHOLE) != 0 { + return defaultRouteMatch{}, false + } + destination, ok := routeMessageAddr(routeMessage.Addrs[syscall.RTAX_DST]) + if !ok || !destination.IsUnspecified() || !zeroNetmask(routeMessage.Addrs[syscall.RTAX_NETMASK], destination.Is4()) { + return defaultRouteMatch{}, false + } + match := defaultRouteMatch{v4: destination.Is4(), index: routeMessage.Index} + if routeMessage.Flags&syscall.RTF_GATEWAY == 0 { + if _, link := routeMessage.Addrs[syscall.RTAX_GATEWAY].(*route.LinkAddr); !link { + return defaultRouteMatch{}, false + } + return match, true + } + gateway, ok := routeMessageAddr(routeMessage.Addrs[syscall.RTAX_GATEWAY]) + if !ok || gateway.Is4() != destination.Is4() { + return defaultRouteMatch{}, false + } + match.gateway = gateway + return match, true +} + +// zeroNetmask reports whether the netmask of a route covers the whole address +// space of its family, which only the default route does. +func zeroNetmask(addr route.Addr, v4 bool) bool { + switch mask := addr.(type) { + case *route.Inet4Addr: + return v4 && mask.IP == [4]byte{} + case *route.Inet6Addr: + return !v4 && mask.IP == [16]byte{} + } + return false +} + +func routeMessageAddr(addr route.Addr) (netip.Addr, bool) { + switch address := addr.(type) { + case *route.Inet4Addr: + return netip.AddrFrom4(address.IP), true + case *route.Inet6Addr: + return withInterfaceZone(netip.AddrFrom16(address.IP), address.ZoneID), true + } + return netip.Addr{}, false +} + +// withInterfaceZone names the interface of a link-local next hop. Every link +// repeats the same range, so the address alone names no host. A tunnel gives +// the bare address fe80:: and the interface is the only part that tells the +// links apart. +func withInterfaceZone(address netip.Addr, zoneID int) netip.Addr { + if zoneID == 0 || !address.IsLinkLocalUnicast() { + return address + } + name := routeInterfaceNameFn(zoneID) + if name == "" { + return address + } + return address.WithZone(name) +} + +// interfaceNameByIndex names one interface of the route table. An index that +// no interface holds gives an empty name. +func interfaceNameByIndex(index int) string { + if index == 0 { + return "" + } + iface, err := net.InterfaceByIndex(index) + if err != nil { + return "" + } + return iface.Name +} diff --git a/cmd/cli/route_darwin_test.go b/cmd/cli/route_darwin_test.go new file mode 100644 index 0000000..8384cf1 --- /dev/null +++ b/cmd/cli/route_darwin_test.go @@ -0,0 +1,291 @@ +package cli + +import ( + "net/netip" + "syscall" + "testing" + + "golang.org/x/net/route" +) + +// stubRouteInterfaceNames names the interface of each index, because the +// interfaces of the host are not a fixture. +func stubRouteInterfaceNames(t *testing.T, names map[int]string) { + t.Helper() + orig := routeInterfaceNameFn + t.Cleanup(func() { routeInterfaceNameFn = orig }) + routeInterfaceNameFn = func(index int) string { return names[index] } +} + +// routeAddrs puts one destination, one netmask, and one gateway at the places +// of the route message that the kernel uses. A nil netmask leaves the slot +// empty, the shape of a host route. +func routeAddrs(destination, netmask, gateway route.Addr) []route.Addr { + addrs := make([]route.Addr, syscall.RTAX_NETMASK+1) + addrs[syscall.RTAX_DST] = destination + addrs[syscall.RTAX_GATEWAY] = gateway + addrs[syscall.RTAX_NETMASK] = netmask + return addrs +} + +func inet4(a, b, c, d byte) *route.Inet4Addr { + return &route.Inet4Addr{IP: [4]byte{a, b, c, d}} +} + +func inet6(address string, zoneID int) *route.Inet6Addr { + addr := netip.MustParseAddr(address) + return &route.Inet6Addr{IP: addr.As16(), ZoneID: zoneID} +} + +func Test_defaultRouteGateway(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{8: "utun4"}) + + for _, tt := range []struct { + name string + message route.Message + wantGateway string + wantIndex int + wantOK bool + }{ + { + name: "v4_gateway", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + wantGateway: "192.168.1.1", + wantIndex: 5, + wantOK: true, + }, + { + name: "v6_link_local_gateway_with_a_zone", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet6("::", 0), inet6("::", 0), inet6("fe80::1", 8)), + }, + wantGateway: "fe80::1%utun4", + wantIndex: 8, + wantOK: true, + }, + { + name: "destination_is_not_the_default_route", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(192, 168, 1, 0), inet4(255, 255, 255, 0), inet4(192, 168, 1, 1)), + }, + }, + { + name: "gateway_names_a_link", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), &route.LinkAddr{Index: 5, Name: "en0"}), + }, + }, + { + name: "gateway_family_differs_from_the_destination", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet6("fe80::1", 8)), + }, + }, + { + name: "address_list_holds_no_gateway", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: []route.Addr{inet4(0, 0, 0, 0)}, + }, + }, + { + name: "message_is_not_a_route", + message: &route.InterfaceMessage{Index: 5}, + }, + { + name: "route_has_no_gateway_flag", + message: &route.RouteMessage{ + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + }, + { + name: "route_is_bound_to_one_interface", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY | syscall.RTF_IFSCOPE, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + gateway, index, ok := defaultRouteGateway(tt.message) + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v", ok, tt.wantOK) + } + if !ok { + return + } + if gateway.String() != tt.wantGateway { + t.Errorf("gateway = %q, want %q", gateway.String(), tt.wantGateway) + } + if index != tt.wantIndex { + t.Errorf("index = %d, want %d", index, tt.wantIndex) + } + }) + } +} + +// Test_defaultRoutesFromMessages covers the table of a host with a v4 default +// route and a v6 default route on another interface. The v6 route must not +// take the interface of the v4 route. +func Test_defaultRoutesFromMessages(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{4: "en0", 8: "utun4"}) + + v4, v6 := defaultRoutesFromMessages([]route.Message{ + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY | syscall.RTF_IFSCOPE, + Index: 9, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(10, 0, 0, 1)), + }, + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 4, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet6("::", 0), inet6("::", 0), inet6("fe80::1", 8)), + }, + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(172, 16, 0, 1)), + }, + }) + + if want := (defaultRoute{Gateway: "192.168.1.1", Interface: "en0"}); v4 != want { + t.Errorf("v4 route = %+v, want %+v", v4, want) + } + if want := (defaultRoute{Gateway: "fe80::1%utun4", Interface: "utun4"}); v6 != want { + t.Errorf("v6 route = %+v, want %+v", v6, want) + } +} + +// Test_defaultRoutesFromMessagesWithoutADefaultRoute covers a table that holds +// no default route, for example a host with no link. +func Test_defaultRoutesFromMessagesWithoutADefaultRoute(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{4: "en0"}) + + v4, v6 := defaultRoutesFromMessages([]route.Message{ + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 4, + Addrs: routeAddrs(inet4(192, 168, 1, 0), inet4(255, 255, 255, 0), inet4(192, 168, 1, 1)), + }, + }) + + if v4 != (defaultRoute{}) || v6 != (defaultRoute{}) { + t.Errorf("routes = %+v / %+v, want two empty routes", v4, v6) + } +} + +// Test_defaultRouteGatewayNeedsAZeroNetmask covers routes whose destination is +// the zero address but whose prefix is not /0. OpenVPN splits the default into +// two /1 routes, and a host route to 0.0.0.0 carries no mask at all. +func Test_defaultRouteGatewayNeedsAZeroNetmask(t *testing.T) { + for _, tt := range []struct { + name string + message route.Message + }{ + { + name: "half of the v4 space", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(128, 0, 0, 0), inet4(10, 8, 0, 1)), + }, + }, + { + name: "host route to the zero address", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY | syscall.RTF_HOST, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), nil, inet4(10, 8, 0, 1)), + }, + }, + { + name: "one eighth of the v6 space", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet6("::", 0), inet6("e000::", 0), inet6("fe80::1", 8)), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + if gateway, _, ok := defaultRouteGateway(tt.message); ok { + t.Fatalf("a %s route counted as the default route with gateway %s", tt.name, gateway) + } + }) + } + v4, _ := defaultRoutesFromMessages([]route.Message{&route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(128, 0, 0, 0), inet4(10, 8, 0, 1)), + }}) + if v4.Gateway != "" || v4.Interface != "" { + t.Fatalf("a table with one /1 route reported the default route %+v, want none", v4) + } +} + +// Test_defaultRoutesTakeALinkDefault covers a VPN that owns the default +// route. Its default has no next hop: the gateway is the link itself and the +// gateway flag is off. The snapshot must name that interface, with an empty +// gateway, instead of the interface of the network monitor. +func Test_defaultRoutesTakeALinkDefault(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{28: "utun4", 16: "en0"}) + linkDefault := &route.RouteMessage{ + Flags: syscall.RTF_UP | syscall.RTF_STATIC, + Index: 28, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), &route.LinkAddr{Index: 28, Name: "utun4"}), + } + scopedEn0 := &route.RouteMessage{ + Flags: syscall.RTF_UP | syscall.RTF_GATEWAY | syscall.RTF_IFSCOPE, + Index: 16, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 50, 1)), + } + + v4, _ := defaultRoutesFromMessages([]route.Message{linkDefault, scopedEn0}) + if v4.Interface != "utun4" || v4.Gateway != "" { + t.Fatalf("default route = %+v, want utun4 with no gateway", v4) + } +} + +// Test_defaultRouteGatewaySkipsRejectAndBlackholeRoutes covers a kill switch +// that installs a default route to nowhere. Such a route carries the gateway +// flag, but it delivers nothing, so it is not the default route of the host. +func Test_defaultRouteGatewaySkipsRejectAndBlackholeRoutes(t *testing.T) { + for _, tt := range []struct { + name string + flags int + }{ + {"reject", syscall.RTF_UP | syscall.RTF_GATEWAY | syscall.RTF_REJECT}, + {"blackhole", syscall.RTF_UP | syscall.RTF_GATEWAY | syscall.RTF_BLACKHOLE}, + } { + t.Run(tt.name, func(t *testing.T) { + message := &route.RouteMessage{ + Flags: tt.flags, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(10, 8, 0, 1)), + } + if gateway, _, ok := defaultRouteGateway(message); ok { + t.Fatalf("a %s route counted as the default route with gateway %s", tt.name, gateway) + } + }) + } +} diff --git a/cmd/cli/route_others.go b/cmd/cli/route_others.go new file mode 100644 index 0000000..65f1330 --- /dev/null +++ b/cmd/cli/route_others.go @@ -0,0 +1,10 @@ +//go:build !darwin + +package cli + +// defaultRoutes gives no route. Only macOS reads the route table in this +// release, so the snapshot of another system holds no gateway and no default +// route of the v6 family. +func defaultRoutes() (v4, v6 defaultRoute) { + return defaultRoute{}, defaultRoute{} +} diff --git a/cmd/cli/service_others.go b/cmd/cli/service_others.go index ce630d6..e524106 100644 --- a/cmd/cli/service_others.go +++ b/cmd/cli/service_others.go @@ -4,6 +4,7 @@ package cli import ( "os" + "syscall" ) // hasElevatedPrivilege checks if the current process has elevated privileges @@ -11,9 +12,10 @@ func hasElevatedPrivilege() (bool, error) { return os.Geteuid() == 0, nil } -// openLogFile opens a log file with the specified flags +// openLogFile opens a log file. It refuses a symlink, because ctrld often +// writes its logs as root into a directory that other users can write. func openLogFile(path string, flags int) (*os.File, error) { - return os.OpenFile(path, flags, os.FileMode(0o600)) + return os.OpenFile(path, flags|syscall.O_NOFOLLOW, os.FileMode(0o600)) } // ConfigureWindowsServiceFailureActions is a no-op on non-Windows platforms diff --git a/cmd/cli/tunnel_owner.go b/cmd/cli/tunnel_owner.go new file mode 100644 index 0000000..b57b085 --- /dev/null +++ b/cmd/cli/tunnel_owner.go @@ -0,0 +1,35 @@ +package cli + +import ( + "bufio" + "io" + "strings" +) + +// agentDescriptionKey starts the quoted product name on an ifconfig agent line. +const agentDescriptionKey = `desc:"` + +// parseIfconfigAgent returns the owner of a tunnel from "ifconfig -v utunN" +// output. The owner is the description of the first agent line, and a tunnel that +// no network extension owns has no agent line, so the result is "". +func parseIfconfigAgent(r io.Reader) string { + if r == nil { + return "" + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if !strings.HasPrefix(line, "agent ") { + continue + } + _, quoted, ok := strings.Cut(line, agentDescriptionKey) + if !ok { + continue + } + if description, _, ok := strings.Cut(quoted, `"`); ok && description != "" { + return description + } + } + // An owner that the read did not reach is not an owner that does not exist. + return "" +} diff --git a/cmd/cli/tunnel_owner_test.go b/cmd/cli/tunnel_owner_test.go new file mode 100644 index 0000000..76ba623 --- /dev/null +++ b/cmd/cli/tunnel_owner_test.go @@ -0,0 +1,83 @@ +package cli + +import ( + "strings" + "testing" +) + +// Captured from "ifconfig -v utunN" on macOS. A tunnel that a network extension +// owns carries an agent line with the product name in desc. +const ifconfigWindscribeTunnel = `utun4: flags=8051 mtu 1420 + index 22 + eflags=1002080 + options=6460 + inet 100.96.0.2 --> 100.96.0.2 netmask 0xffffffff + nd6 options=201 + agent domain:NetworkExtension type:NetworkExtension flags:0xf desc:"Windscribe VPN" + agent domain:Multipath type:Path Source flags:0x3 desc:"Multipath path manager" + netif: 6d1e1b2a-4c55-4f0b-9f0a-1d2e3f4a5b6c + scheduler: FQ_CODEL + link quality: 100 (good) +` + +const ifconfigTailscaleTunnel = `utun6: flags=8051 mtu 1280 + index 24 + agent domain:NetworkExtension type:NetworkExtension flags:0xf desc:"Tailscale Tunnel" + nd6 options=201 +` + +const ifconfigSystemTunnel = `utun0: flags=8051 mtu 1500 + index 16 + eflags=1002080 + inet6 fe80::e1a2:3b4c:5d6e:7f80%utun0 prefixlen 64 scopeid 0x10 + nd6 options=201 + scheduler: FQ_CODEL +` + +func Test_parseIfconfigAgent(t *testing.T) { + tests := []struct { + name string + output string + want string + }{ + { + name: "a VPN network extension owns the tunnel", + output: ifconfigWindscribeTunnel, + want: "Windscribe VPN", + }, + { + name: "a mesh client owns the tunnel", + output: ifconfigTailscaleTunnel, + want: "Tailscale Tunnel", + }, + { + name: "no agent line", + output: ifconfigSystemTunnel, + }, + { + name: "no output", + output: "", + }, + { + name: "an agent line without a description", + output: "\tagent domain:Multipath type:Path Source flags:0x3\n\tagent domain:NetworkExtension type:NetworkExtension flags:0xf desc:\"Windscribe VPN\"\n", + want: "Windscribe VPN", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := parseIfconfigAgent(strings.NewReader(tc.output)); got != tc.want { + t.Fatalf("parseIfconfigAgent() = %q, want %q", got, tc.want) + } + }) + } +} + +// Test_parseIfconfigAgentRejectsAnOversizedLine covers a truncated read. An +// owner that the reader did not reach is not an owner that does not exist. +func Test_parseIfconfigAgentRejectsAnOversizedLine(t *testing.T) { + output := strings.Repeat("a", 70<<10) + "\n" + ifconfigWindscribeTunnel + if got := parseIfconfigAgent(strings.NewReader(output)); got != "" { + t.Fatalf("tunnel owner of a truncated read = %q, want an empty owner", got) + } +} diff --git a/cmd/cli/upstream_monitor.go b/cmd/cli/upstream_monitor.go index fe211b5..f660ed2 100644 --- a/cmd/cli/upstream_monitor.go +++ b/cmd/cli/upstream_monitor.go @@ -1,6 +1,7 @@ package cli import ( + "strings" "sync" "sync/atomic" "time" @@ -18,6 +19,12 @@ const ( // the backoff so an unroutable endpoint is still re-probed periodically and // recovers once the route returns. checkUpstreamUnreachableBackoffMax = 60 * time.Second + // upstreamDownDelay is the time that the failures of an upstream must last + // before the monitor marks it down. + upstreamDownDelay = 10 * time.Second + // upstreamRecoveredDelay is the time that a reset holds back the failure + // counts of an upstream. + upstreamRecoveredDelay = 1 * time.Second ) // unreachableRecoveryBackoff returns the retry interval for the given streak of @@ -34,19 +41,45 @@ func unreachableRecoveryBackoff(streak int) time.Duration { return d } +// upstreamDownState remembers when an upstream went down and how many +// failures put it there, so the up event can report the outage. +type upstreamDownState struct { + since time.Time + failures uint64 +} + // upstreamMonitor performs monitoring upstreams health. type upstreamMonitor struct { cfg *ctrld.Config logger atomic.Pointer[ctrld.Logger] - mu sync.RWMutex + mu sync.RWMutex + // down holds one entry for each upstream that is down now. + down map[string]upstreamDownState checking map[string]bool - down map[string]bool failureReq map[string]uint64 recovered map[string]bool // failureTimerActive tracks if a timer is already running for a given upstream. failureTimerActive map[string]bool + // retired is set when a reload replaced this monitor. A timer that was + // armed before the reload must not open an outage that nothing closes. + retired bool + + // now is a seam, so a test can set the reported down time. + now func() time.Time + // after arms a one-shot timer, so a test fires the check itself instead of + // waiting for it. nil means time.AfterFunc. + after func(time.Duration, func()) +} + +// afterFunc returns the timer seam. A real timer drops its handle, because +// nothing stops these checks. +func (um *upstreamMonitor) afterFunc() func(time.Duration, func()) { + if um.after != nil { + return um.after + } + return func(d time.Duration, fn func()) { time.AfterFunc(d, fn) } } // newUpstreamMonitor creates a new upstream monitor instance @@ -54,10 +87,11 @@ func newUpstreamMonitor(cfg *ctrld.Config, logger *ctrld.Logger) *upstreamMonito um := &upstreamMonitor{ cfg: cfg, checking: make(map[string]bool), - down: make(map[string]bool), + down: make(map[string]upstreamDownState), failureReq: make(map[string]uint64), recovered: make(map[string]bool), failureTimerActive: make(map[string]bool), + now: time.Now, } um.logger.Store(logger) for n := range cfg.Upstream { @@ -75,6 +109,9 @@ func (um *upstreamMonitor) increaseFailureCount(upstream string) { um.mu.Lock() defer um.mu.Unlock() + if um.retired { + return + } if um.recovered[upstream] { um.logger.Load().Debug().Msgf("Upstream %q is recovered, skipping failure count increase", upstream) return @@ -86,53 +123,191 @@ func (um *upstreamMonitor) increaseFailureCount(upstream string) { // Log the updated failure count. um.logger.Load().Debug().Msgf("Upstream %q failure count updated to %d", upstream, failedCount) - // If this is the first failure and no timer is running, start a 10-second timer. + // If this is the first failure and no timer is running, start the timer. if failedCount == 1 && !um.failureTimerActive[upstream] { um.failureTimerActive[upstream] = true - go func(upstream string) { - time.Sleep(10 * time.Second) - um.mu.Lock() - defer um.mu.Unlock() - // If no success occurred during the 10-second window (i.e. counter remains > 0) - // and the upstream is not in a recovered state, mark it as down. - if um.failureReq[upstream] > 0 && !um.recovered[upstream] { - um.down[upstream] = true - um.logger.Load().Warn().Msgf("Upstream %q marked as down after 10 seconds (failure count: %d)", upstream, um.failureReq[upstream]) - } - // Reset the timer flag so that a new timer can be spawned if needed. - um.failureTimerActive[upstream] = false - }(upstream) + um.afterFunc()(upstreamDownDelay, func() { um.markDownAfterDelay(upstream) }) } // If the failure count quickly reaches the threshold, mark the upstream as down immediately. if failedCount >= maxFailureRequest { - um.down[upstream] = true - um.logger.Load().Warn().Msgf("Upstream %q marked as down immediately (failure count: %d)", upstream, failedCount) + um.markDown(upstream, failedCount, "immediate") } } +// markDownAfterDelay marks an upstream down when its failures did not clear +// while the timer ran. A success during that time leaves the count at zero. +func (um *upstreamMonitor) markDownAfterDelay(upstream string) { + um.mu.Lock() + defer um.mu.Unlock() + if um.retired { + return + } + if um.failureReq[upstream] > 0 && !um.recovered[upstream] { + um.markDown(upstream, um.failureReq[upstream], "timer") + } + // The next failure of this upstream can arm a new timer. + um.failureTimerActive[upstream] = false +} + +// markDown marks an upstream as down and logs the transition once. The caller holds the lock. +func (um *upstreamMonitor) markDown(upstream string, failures uint64, reason string) { + if um.retired { + return + } + if _, down := um.down[upstream]; down { + return + } + um.down[upstream] = upstreamDownState{since: um.now(), failures: failures} + journal(um.logger.Load().Warn()). + Str("upstream", journalUpstreamName(upstream)). + Str("state", "down"). + Uint64("failure_count", failures). + Str("reason", reason). + Msg("Upstream state changed") +} + +// markUp marks an upstream as up and logs the transition once. The caller holds the lock. +func (um *upstreamMonitor) markUp(upstream, reason string) { + state, down := um.down[upstream] + if !down { + return + } + delete(um.down, upstream) + journal(um.logger.Load().Info()). + Str("upstream", journalUpstreamName(upstream)). + Str("state", "up"). + Str("reason", reason). + Int64("down_for_ms", um.now().Sub(state.since).Milliseconds()). + Uint64("failure_count", state.failures). + Msg("Upstream state changed") +} + +// journalUpstreamName bounds the upstream name that a journal event carries. +// ctrld generates upstream.os, upstream., and upstream.internal_; every +// other key is operator text that can hold a token, so it reads as +// upstream.custom. An empty name holds no operator text and stays empty, +// because a line that names no upstream must not read as a custom one. +func journalUpstreamName(upstream string) string { + if upstream == "" || upstream == upstreamOS || generatedUpstreamKey(strings.TrimPrefix(upstream, upstreamPrefix)) { + return upstream + } + return upstreamPrefix + "custom" +} + +// journalUpstreamNames bounds every name of a list, for a line that names +// several upstreams. +func journalUpstreamNames(upstreams []string) []string { + names := make([]string, 0, len(upstreams)) + for _, upstream := range upstreams { + names = append(names, journalUpstreamName(upstream)) + } + return names +} + +// generatedUpstreamKey reports whether key is a number, or internal_ and a +// number, the two shapes that ctrld writes itself. +func generatedUpstreamKey(key string) bool { + digits := strings.TrimPrefix(key, internalDomainUpstreamPrefix) + if digits == "" { + return false + } + for _, r := range digits { + if r < '0' || r > '9' { + return false + } + } + return true +} + +// retire closes the outage of every upstream that is still down and stops +// every later state change. A reload builds a new monitor, so the journal ends +// the outages of the old one, and the old timers must not open new ones. +func (um *upstreamMonitor) retire() { + um.mu.Lock() + defer um.mu.Unlock() + for upstream := range um.down { + um.markUp(upstream, "reload") + } + um.retired = true + clear(um.failureReq) + clear(um.failureTimerActive) +} + +// noteSuccess records an answer from an upstream. It does not set the recovered +// flag, because that flag stops failure counts for one second and an answer +// must not hide the failures that follow it. +func (um *upstreamMonitor) noteSuccess(upstream string) { + um.mu.Lock() + defer um.mu.Unlock() + + um.failureReq[upstream] = 0 + um.markUp(upstream, "recovered") +} + +// countDown reports how many upstreams are down at this time. +func (um *upstreamMonitor) countDown() int { + um.mu.RLock() + defer um.mu.RUnlock() + return len(um.down) +} + +// countDownExcept reports how many upstreams are down at this time, without +// the ones that skip accepts. The grade of the query path leaves out the +// resolvers that only an endpoint on the organization network can reach. +func (um *upstreamMonitor) countDownExcept(skip func(string) bool) int { + if skip == nil { + return um.countDown() + } + um.mu.RLock() + defer um.mu.RUnlock() + count := 0 + for upstream := range um.down { + if skip(upstream) { + continue + } + count++ + } + return count +} + +// downFor reports how long an upstream has been down. An upstream that is up +// gives zero, so a caller logs the field without a branch. +func (um *upstreamMonitor) downFor(upstream string) time.Duration { + um.mu.RLock() + defer um.mu.RUnlock() + state, down := um.down[upstream] + if !down { + return 0 + } + return um.now().Sub(state.since) +} + // isDown reports whether the given upstream is being marked as down. func (um *upstreamMonitor) isDown(upstream string) bool { um.mu.Lock() defer um.mu.Unlock() - return um.down[upstream] + _, down := um.down[upstream] + return down } // reset marks an upstream as up and set failed queries counter to zero. func (um *upstreamMonitor) reset(upstream string) { um.mu.Lock() um.failureReq[upstream] = 0 - um.down[upstream] = false + um.markUp(upstream, "recovered") um.recovered[upstream] = true um.mu.Unlock() - go func() { - // debounce the recovery to avoid incrementing failure counts already in flight - time.Sleep(1 * time.Second) - um.mu.Lock() - um.recovered[upstream] = false - um.mu.Unlock() - }() + // The flag holds back the failure counts that are already in flight. + um.afterFunc()(upstreamRecoveredDelay, func() { um.clearRecovered(upstream) }) +} + +// clearRecovered lets the failures of an upstream count again. +func (um *upstreamMonitor) clearRecovered(upstream string) { + um.mu.Lock() + defer um.mu.Unlock() + um.recovered[upstream] = false } // countHealthy returns the number of upstreams in the provided map that are considered healthy. @@ -140,7 +315,7 @@ func (um *upstreamMonitor) countHealthy(upstreams []string) int { var count int um.mu.RLock() for _, upstream := range upstreams { - if !um.down[upstream] { + if _, down := um.down[upstream]; !down { count++ } } diff --git a/cmd/cli/upstream_monitor_test.go b/cmd/cli/upstream_monitor_test.go new file mode 100644 index 0000000..b09e44b --- /dev/null +++ b/cmd/cli/upstream_monitor_test.go @@ -0,0 +1,286 @@ +package cli + +import ( + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// upstreamStateMessage is the message of every upstream transition event. +const upstreamStateMessage = "Upstream state changed" + +// downMonitor returns a monitor on a clock the test owns, with the upstream down. +func downMonitor(t *testing.T, upstream string) (*upstreamMonitor, *syncBuffer, *time.Time) { + t.Helper() + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + now := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + um.now = func() time.Time { return now } + for i := 0; i < maxFailureRequest; i++ { + um.increaseFailureCount(upstream) + } + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 1 { + t.Fatalf("state events after the failures: got %d, want 1", got) + } + return um, logs, &now +} + +func Test_upstreamMonitorLogsOneDownEvent(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + + for i := 0; i < 633; i++ { + um.increaseFailureCount(upstream) + } + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 1 { + t.Fatalf("state events: got %d, want 1", len(events)) + } + wantField(t, events[0], "level", "warn") + wantField(t, events[0], "journal", true) + wantField(t, events[0], "upstream", upstream) + wantField(t, events[0], "state", "down") + wantField(t, events[0], "reason", "immediate") + wantField(t, events[0], "failure_count", float64(maxFailureRequest)) +} + +func Test_upstreamMonitorTimerLogsDownEvent(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + var armed []func() + um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + + um.increaseFailureCount(upstream) + if len(armed) != 1 { + t.Fatalf("armed timers: got %d, want 1", len(armed)) + } + armed[0]() + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 1 { + t.Fatalf("state events: got %d, want 1", len(events)) + } + wantField(t, events[0], "state", "down") + wantField(t, events[0], "reason", "timer") + wantField(t, events[0], "failure_count", float64(1)) +} + +func Test_upstreamMonitorResetLogsOneUpEvent(t *testing.T) { + const upstream = upstreamPrefix + "0" + um, logs, now := downMonitor(t, upstream) + + *now = now.Add(1500 * time.Millisecond) + um.reset(upstream) + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want 2", len(events)) + } + up := events[1] + wantField(t, up, "level", "info") + wantField(t, up, "journal", true) + wantField(t, up, "upstream", upstream) + wantField(t, up, "state", "up") + wantField(t, up, "reason", "recovered") + wantField(t, up, "down_for_ms", float64(1500)) + wantField(t, up, "failure_count", float64(maxFailureRequest)) +} + +func Test_upstreamMonitorRetireLogsTheOpenOutage(t *testing.T) { + const upstream = upstreamPrefix + "0" + um, logs, now := downMonitor(t, upstream) + + *now = now.Add(3 * time.Second) + um.retire() + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want 2", len(events)) + } + up := events[1] + wantField(t, up, "upstream", upstream) + wantField(t, up, "state", "up") + wantField(t, up, "reason", "reload") + wantField(t, up, "down_for_ms", float64(3000)) + + um.retire() + + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 2 { + t.Fatalf("state events after the second retire: got %d, want 2", got) + } +} + +func Test_upstreamMonitorNoteSuccessLogsOneUpEvent(t *testing.T) { + const upstream = upstreamPrefix + "0" + um, logs, now := downMonitor(t, upstream) + + *now = now.Add(2 * time.Second) + um.noteSuccess(upstream) + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want 2", len(events)) + } + up := events[1] + wantField(t, up, "state", "up") + wantField(t, up, "reason", "recovered") + wantField(t, up, "down_for_ms", float64(2000)) + wantField(t, up, "failure_count", float64(maxFailureRequest)) + if got := um.failureReq[upstream]; got != 0 { + t.Fatalf("failure count after the answer: got %d, want 0", got) + } +} + +func Test_upstreamMonitorNoteSuccessOnUpUpstreamIsQuiet(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + + um.noteSuccess(upstream) + um.noteSuccess(upstream) + + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 0 { + t.Fatalf("state events: got %d, want 0", got) + } +} + +// Test_upstreamMonitorRetiredTimerStaysSilent fires the down timer that a +// failure armed before the reload retired the monitor. A retired monitor +// owns no outage, so the timer must add no event. +func Test_upstreamMonitorRetiredTimerStaysSilent(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + var armed []func() + um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + + um.increaseFailureCount(upstream) + um.retire() + for _, fire := range armed { + fire() + } + + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 0 { + t.Fatalf("state events from the retired monitor: got %d, want 0: %s", got, logs.String()) + } + if um.isDown(upstream) { + t.Fatal("the retired monitor marked the upstream down") + } +} + +// Test_upstreamMonitorRetiredAfterImmediateDownStaysSilent retires a monitor +// that already reported an outage. The armed timer and a late failure must +// add no event after the closing up event. +func Test_upstreamMonitorRetiredAfterImmediateDownStaysSilent(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + var armed []func() + um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + + for i := 0; i < maxFailureRequest; i++ { + um.increaseFailureCount(upstream) + } + um.retire() + before := jsonLogEvents(t, logs, upstreamStateMessage) + if len(before) != 2 { + t.Fatalf("state events after the retire: got %d, want down and up", len(before)) + } + for _, fire := range armed { + fire() + } + um.increaseFailureCount(upstream) + + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 2 { + t.Fatalf("state events after the retired timer fired: got %d, want 2: %s", got, logs.String()) + } + if um.isDown(upstream) { + t.Fatal("the retired monitor marked the upstream down again") + } +} + +// Test_upstreamMonitorKeepsACustomNameOutOfTheJournal marks an upstream with +// an operator-chosen key down. The key can hold a token, so the event names +// the upstream by a bounded name. +func Test_upstreamMonitorKeepsACustomNameOutOfTheJournal(t *testing.T) { + const upstream = upstreamPrefix + "dns.controld.com/org-v1-SECRET0123" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + + for i := 0; i < maxFailureRequest; i++ { + um.increaseFailureCount(upstream) + } + um.noteSuccess(upstream) + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want down and up", len(events)) + } + for _, event := range events { + wantField(t, event, "upstream", upstreamPrefix+"custom") + } + for _, generated := range []string{upstreamOS, upstreamPrefix + "0", upstreamPrefix + "internal_3"} { + if got := journalUpstreamName(generated); got != generated { + t.Fatalf("journalUpstreamName(%q) = %q, want the name itself", generated, got) + } + } +} + +func Test_upstreamMonitorCountDown(t *testing.T) { + captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + if got := um.countDown(); got != 0 { + t.Fatalf("countDown on a new monitor = %d, want 0", got) + } + um.mu.Lock() + um.markDown(upstreamPrefix+"0", 3, "immediate") + um.markDown(upstreamPrefix+"1", 3, "immediate") + um.mu.Unlock() + if got := um.countDown(); got != 2 { + t.Fatalf("countDown with two down upstreams = %d, want 2", got) + } +} + +func Test_upstreamMonitorDownFor(t *testing.T) { + const upstream = upstreamPrefix + "0" + um, _, now := downMonitor(t, upstream) + + *now = now.Add(2500 * time.Millisecond) + + if got := um.downFor(upstream); got != 2500*time.Millisecond { + t.Fatalf("downFor a down upstream = %s, want 2.5s", got) + } + if got := um.downFor(upstreamPrefix + "9"); got != 0 { + t.Fatalf("downFor an unknown upstream = %s, want 0", got) + } + + um.reset(upstream) + + if got := um.downFor(upstream); got != 0 { + t.Fatalf("downFor an upstream that is up = %s, want 0", got) + } +} + +func Test_upstreamMonitorCountDownExcept(t *testing.T) { + captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + um.mu.Lock() + um.markDown(upstreamPrefix+"0", 3, "immediate") + um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"1", 3, "immediate") + um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"2", 3, "immediate") + um.mu.Unlock() + + if got := um.countDown(); got != 3 { + t.Fatalf("countDown = %d, want 3", got) + } + if got := um.countDownExcept(isInternalDomainUpstream); got != 1 { + t.Fatalf("countDownExcept(isInternalDomainUpstream) = %d, want 1", got) + } + if got := um.countDownExcept(nil); got != 3 { + t.Fatalf("countDownExcept(nil) = %d, want 3", got) + } +} diff --git a/cmd/cli/upstream_probe_journal_test.go b/cmd/cli/upstream_probe_journal_test.go new file mode 100644 index 0000000..d829d48 --- /dev/null +++ b/cmd/cli/upstream_probe_journal_test.go @@ -0,0 +1,151 @@ +package cli + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// probeSecret is the operator text that a custom upstream key, name, and +// endpoint hold in these tests. +const probeSecret = "org-v1-SECRET0123" + +// customProbeUpstream builds an upstream whose key, name, and endpoint all +// hold operator text. +func customProbeUpstream() (string, *ctrld.UpstreamConfig) { + return upstreamPrefix + probeSecret, &ctrld.UpstreamConfig{ + Name: probeSecret, + Endpoint: "https://dns.example.com/" + probeSecret, + } +} + +// retainedProbeLines returns the captured lines that the journal keeps. +func retainedProbeLines(t *testing.T, logs *syncBuffer) []map[string]any { + t.Helper() + var retained []map[string]any + for _, event := range jsonLogEvents(t, logs, "") { + switch event["level"] { + case "error", "warn": + retained = append(retained, event) + } + } + return retained +} + +// wantNoProbeSecret fails when a retained line carries the operator text. +func wantNoProbeSecret(t *testing.T, logs *syncBuffer) { + t.Helper() + for _, event := range retainedProbeLines(t, logs) { + for field, value := range event { + text, ok := value.(string) + if ok && strings.Contains(text, probeSecret) { + t.Fatalf("retained field %q holds the operator text: %s", field, text) + } + } + } +} + +// Test_upstreamFailureLogKeepsACustomKeyOutOfTheJournal drives one recovery +// check failure against an operator-chosen upstream key. The journal keeps +// error lines, so the key, the name, and the endpoint must stay off the line. +func Test_upstreamFailureLogKeepsACustomKeyOutOfTheJournal(t *testing.T) { + logs := captureDebugMainLog(t) + upstream, uc := customProbeUpstream() + + (&upstreamFailureLog{}).report(probeTestProg(), upstream, uc, errors.New("probe failed"), time.Second) + + retained := retainedProbeLines(t, logs) + if len(retained) != 1 { + t.Fatalf("retained lines: got %d, want 1", len(retained)) + } + wantField(t, retained[0], "upstream", upstreamPrefix+"custom") + wantNoProbeSecret(t, logs) + if !strings.Contains(logs.String(), probeSecret) { + t.Fatal("the operator text must stay at debug level for diagnosis") + } +} + +// Test_checkUpstreamOnceKeepsACustomKeyOutOfTheJournal drives the resolver +// creation failure of a recovery check. It logs at error level, so the journal +// keeps it. +func Test_checkUpstreamOnceKeepsACustomKeyOutOfTheJournal(t *testing.T) { + logs := captureDebugMainLog(t) + upstream, uc := customProbeUpstream() + // An empty type has no resolver, so the check fails before it sends. + uc.Type = "" + p := &prog{} + p.logger.Store(mainLog.Load()) + + if err := p.checkUpstreamOnce(upstream, uc, &upstreamFailureLog{}); err == nil { + t.Fatal("expected the resolver creation to fail") + } + + retained := retainedProbeLines(t, logs) + if len(retained) != 1 { + t.Fatalf("retained lines: got %d, want 1", len(retained)) + } + wantField(t, retained[0], "upstream", upstreamPrefix+"custom") + wantNoProbeSecret(t, logs) +} + +// Test_checkDnsLoopKeepsACustomKeyOutOfTheJournal drives the loop checker +// against an operator-chosen upstream key. The loop checker writes at warn +// level, so the journal keeps its line too. +func Test_checkDnsLoopKeepsACustomKeyOutOfTheJournal(t *testing.T) { + logs := captureDebugMainLog(t) + uc := &ctrld.UpstreamConfig{ + Name: probeSecret, + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 200, + } + uc.Init(context.Background()) + cfg := &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{probeSecret: uc}} + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load()), loop: make(map[string]bool)} + p.logger.Store(mainLog.Load()) + + p.checkDnsLoop() + + retained := retainedProbeLines(t, logs) + if len(retained) == 0 { + t.Fatal("the loop check failure must stay visible above debug") + } + for _, event := range retained { + wantField(t, event, "upstream", upstreamPrefix+"custom") + } + wantNoProbeSecret(t, logs) +} + +// Test_logUpstreamPingFailuresKeepsACustomKeyOutOfTheJournal drives the report +// that follows a failed self-check. It writes at error level, so the journal +// keeps it and the operator text must stay off the line. +func Test_logUpstreamPingFailuresKeepsACustomKeyOutOfTheJournal(t *testing.T) { + logs := captureDebugMainLog(t) + _, uc := customProbeUpstream() + origPing := upstreamPingFn + t.Cleanup(func() { upstreamPingFn = origPing }) + upstreamPingFn = func(*ctrld.UpstreamConfig, context.Context) error { return errors.New("ping failed") } + + logUpstreamPingFailures(context.Background(), map[string]*ctrld.UpstreamConfig{probeSecret: uc}) + + retained := retainedProbeLines(t, logs) + if len(retained) != 1 { + t.Fatalf("retained lines: got %d, want 1", len(retained)) + } + wantField(t, retained[0], "upstream", upstreamPrefix+"custom") + wantNoProbeSecret(t, logs) + if !strings.Contains(logs.String(), probeSecret) { + t.Fatal("the operator text must stay at debug level for diagnosis") + } +} + +// probeTestProg returns a prog that logs through the captured main logger. +func probeTestProg() *prog { + p := &prog{} + p.logger.Store(mainLog.Load()) + return p +} diff --git a/cmd/cli/vpn_dns.go b/cmd/cli/vpn_dns.go index b475328..818d063 100644 --- a/cmd/cli/vpn_dns.go +++ b/cmd/cli/vpn_dns.go @@ -2,6 +2,7 @@ package cli import ( "context" + "fmt" "net" "runtime" "strings" @@ -65,10 +66,16 @@ type vpnDNSManager struct { refreshRunning bool refreshPending bool discoveryMu sync.Mutex + // repeats holds the last refresh summary. The manager has no prog, and a + // network change storm runs one refresh per delta. + repeats repeatLogger // Called when VPN DNS server list changes, to update intercept exemptions. onServersChanged vpnDNSExemptFunc } +// vpnDNSRefreshKey names the repeat counter of the refresh summary. +const vpnDNSRefreshKey = "vpn_dns_refresh" + // newVPNDNSManager creates a new manager. Only call when dnsIntercept is active. // exemptFunc is called whenever VPN DNS servers are discovered/changed, to update // the OS-level intercept rules to permit ctrld's outbound queries to those IPs. @@ -230,8 +237,11 @@ func (m *vpnDNSManager) refreshOnce(ctx context.Context, guardAgainstNoNameserve } m.domainlessServers = domainlessServers - ctrld.Log(ctx, logger.Debug(), "VPN DNS refresh completed: %d configs, %d routes, %d domainless servers, %d unique exemptions", + summary := fmt.Sprintf("%d configs, %d routes, %d domainless servers, %d unique exemptions", len(m.configs), len(m.routes), len(m.domainlessServers), len(exemptions)) + if changed, repeats := m.repeats.changed(vpnDNSRefreshKey, summary); changed { + ctrld.Log(ctx, logger.Debug().Uint64("repeats", repeats), "VPN DNS refresh completed: %s", summary) + } // Update intercept rules only when desired exemptions differ from the last // successfully applied set. Failed PF/WFP callbacks remain retryable on the diff --git a/cmd/cli/vpn_dns_test.go b/cmd/cli/vpn_dns_test.go index eba943c..f773c92 100644 --- a/cmd/cli/vpn_dns_test.go +++ b/cmd/cli/vpn_dns_test.go @@ -3,6 +3,7 @@ package cli import ( "context" "errors" + "strings" "sync" "sync/atomic" "testing" @@ -296,3 +297,55 @@ func TestVPNDNSFullAndRouteOnlyDiscoveryAreSerialized(t *testing.T) { t.Fatalf("serialized exemption updates = %v, want old then new", exemptionUpdates) } } + +// eventsWithPrefix returns the captured events whose message starts with the +// prefix. Several lines of this module carry a value in the message, so no +// constant matches the whole of it. +func eventsWithPrefix(t *testing.T, logs *syncBuffer, prefix string) []map[string]any { + t.Helper() + var matched []map[string]any + for _, event := range jsonLogEvents(t, logs, "") { + message, _ := event["message"].(string) + if strings.HasPrefix(message, prefix) { + matched = append(matched, event) + } + } + return matched +} + +// TestVPNDNSRefreshSummaryLogsOnChangeOnly covers the refresh summary. A network +// change storm runs the refresh every few seconds, and the same summary in every +// one of those lines buries the refresh that changed something. +func TestVPNDNSRefreshSummaryLogsOnChangeOnly(t *testing.T) { + logs := captureDebugMainLog(t) + m := newVPNDNSManager(&mainLog, nil) + configs := []ctrld.VPNDNSConfig{{ + InterfaceName: "utun4", + Servers: []string{"10.0.0.1"}, + Domains: []string{"corp.internal"}, + }} + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { return configs } + + // Refresh logs through the logger of its context. + ctx := ctrld.LoggerCtx(context.Background(), mainLog.Load()) + m.Refresh(ctx, true) + m.Refresh(ctx, true) + + summaries := eventsWithPrefix(t, logs, "VPN DNS refresh completed") + if len(summaries) != 1 { + t.Fatalf("got %d refresh summaries for two identical results, want 1", len(summaries)) + } + wantField(t, summaries[0], "repeats", float64(0)) + + configs = append(configs, ctrld.VPNDNSConfig{ + InterfaceName: "utun6", + Servers: []string{"10.0.0.2"}, + }) + m.Refresh(ctx, true) + + summaries = eventsWithPrefix(t, logs, "VPN DNS refresh completed") + if len(summaries) != 2 { + t.Fatalf("got %d refresh summaries after the result changed, want 2", len(summaries)) + } + wantField(t, summaries[1], "repeats", float64(1)) +} diff --git a/config.go b/config.go index a44fdc9..81b1f04 100644 --- a/config.go +++ b/config.go @@ -224,6 +224,8 @@ func (c *Config) FirstUpstream() *UpstreamConfig { type ServiceConfig struct { LogLevel string `mapstructure:"log_level" toml:"log_level,omitempty"` LogPath string `mapstructure:"log_path" toml:"log_path,omitempty"` + LogMaxSizeMB int `mapstructure:"log_max_size_mb" toml:"log_max_size_mb,omitempty" validate:"omitempty,gte=1,lte=1024"` + LogMaxBackups *int `mapstructure:"log_max_backups" toml:"log_max_backups,omitempty" validate:"omitempty,gte=0,lte=64"` CacheEnable bool `mapstructure:"cache_enable" toml:"cache_enable,omitempty"` CacheSize int `mapstructure:"cache_size" toml:"cache_size,omitempty"` CacheTTLOverride int `mapstructure:"cache_ttl_override" toml:"cache_ttl_override,omitempty"` diff --git a/config_test.go b/config_test.go index cd392d5..bb2c7b2 100644 --- a/config_test.go +++ b/config_test.go @@ -169,6 +169,72 @@ discover_ptr = false require.False(t, *cfg.Service.DiscoverPtr) } +func TestLogMaxKeys(t *testing.T) { + tests := []struct { + name string + serviceKeys string + wantSize int + wantBackups *int + }{ + {"size and backups set", "log_max_size_mb = 20\nlog_max_backups = 1", 20, intPtr(1)}, + {"size zero, backups absent", "log_max_size_mb = 0", 0, nil}, + {"backups zero", "log_max_backups = 0", 0, intPtr(0)}, + {"the largest values the keys take", "log_max_size_mb = 1024\nlog_max_backups = 64", 1024, intPtr(64)}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := configWithServiceKeys(t, tc.serviceKeys) + validate := validator.New() + require.NoError(t, ctrld.ValidateConfig(validate, cfg)) + require.Equal(t, tc.wantSize, cfg.Service.LogMaxSizeMB) + if tc.wantBackups == nil { + require.Nil(t, cfg.Service.LogMaxBackups) + return + } + require.NotNil(t, cfg.Service.LogMaxBackups) + require.Equal(t, *tc.wantBackups, *cfg.Service.LogMaxBackups) + }) + } +} + +func TestLogMaxKeysOutOfRange(t *testing.T) { + tests := []struct { + name string + serviceKeys string + wantField string + }{ + {"negative size", "log_max_size_mb = -1", "LogMaxSizeMB"}, + {"negative backups", "log_max_backups = -1", "LogMaxBackups"}, + {"size above the limit", "log_max_size_mb = 1025", "LogMaxSizeMB"}, + {"backups above the limit", "log_max_backups = 65", "LogMaxBackups"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := configWithServiceKeys(t, tc.serviceKeys) + validate := validator.New() + err := ctrld.ValidateConfig(validate, cfg) + require.ErrorContains(t, err, tc.wantField) + }) + } +} + +func configWithServiceKeys(t *testing.T, serviceKeys string) *ctrld.Config { + t.Helper() + v := viper.NewWithOptions(viper.KeyDelimiter("::")) + ctrld.InitConfig(v, "test_log_max_keys") + v.SetConfigType("toml") + require.NoError(t, v.ReadConfig(strings.NewReader("[service]\n"+serviceKeys+"\n"))) + var cfg ctrld.Config + require.NoError(t, v.Unmarshal(&cfg)) + return &cfg +} + +func intPtr(n int) *int { + return &n +} + func defaultConfig(t *testing.T) *ctrld.Config { v := viper.New() ctrld.InitConfig(v, "test_load_default_config") diff --git a/docs/config.md b/docs/config.md index c196a73..864ea3d 100644 --- a/docs/config.md +++ b/docs/config.md @@ -117,10 +117,28 @@ Logging level you wish to enable. ### log_path Relative or absolute path of the log file. +Point `log_path` at a file that no other program rotates. ctrld renames `.1` to `.N` and uploads these files. A lower `log_max_backups` leaves the older numbered `log_path` files in place, but ctrld removes the extra numbered files of its internal logs. + - Type: string - Required: no - Default: "" +### log_max_size_mb +Maximum size of the debug log file in MB. When the file reaches this size, ctrld rotates it. This limit also applies to the `log_path` file. + +- Type: integer +- Required: no +- Valid values: 1 to 1024 +- Default: 10 + +### log_max_backups +Number of rotated debug log files that ctrld keeps. If the value is 0, ctrld keeps no rotated file. This number also applies to the `log_path` file. + +- Type: integer +- Required: no +- Valid values: 0 to 64 +- Default: 4 + ### cache_enable When `cache_enable = true`, all resolved DNS query responses will be cached for duration of the upstream record TTLs. diff --git a/docs/dns-intercept-mode.md b/docs/dns-intercept-mode.md index 4f860df..78ea9c3 100644 --- a/docs/dns-intercept-mode.md +++ b/docs/dns-intercept-mode.md @@ -235,6 +235,10 @@ DNS intercept: pf anchor "com.controld.ctrld" active with 3 rules DNS intercept: pf redirect active — all outbound DNS (port 53) redirected to 127.0.0.1:53 ``` +DNS intercept mode on macOS also runs the wake detector. That detector reports a `Host woke` journal event with the source `detector` and the length of the sleep. +Outside DNS intercept mode the netmon delta is the only wake source, and it reports no length of its own. +The events of the journal are listed in [Network-recovery diagnostics](network-recovery-diagnostics.md#journal-events). + ## Troubleshooting ### Windows @@ -306,12 +310,14 @@ VPN apps commonly add rules like `pass out quick on ipsec0 inet all` that match ### 3. Dynamic Tunnel Interface Detection -The network change monitor (`validInterfacesMap()`) only tracks physical hardware ports (en0, bridge0, etc.) — it doesn't see tunnel interfaces (utun*, ipsec*, etc.) created by VPN software. When a VPN connects and creates a new interface (e.g., utun420 for WireGuard), ctrld detects this through a separate tunnel interface change check and rebuilds the pf anchor to include explicit intercept rules for the new interface. This runs on every network change event, even if no physical interface changed. +The network change monitor (`validInterfacesMap()`) only tracks physical hardware ports (en0, bridge0, etc.) — it doesn't see tunnel interfaces (utun*, ipsec*, etc.) created by VPN software. When a VPN connects and creates a new interface (e.g., utun420 for WireGuard), ctrld detects this through a separate tunnel interface change check and rebuilds the pf anchor to include explicit intercept rules for the new interface. This runs on every network change event that is not noise, even if no physical interface changed. A delta that touches AirDrop or virtual adapters alone is noise, and ctrld skips the handler for it, so no tunnel discovery runs. ### 4. pf Watchdog + Network Change Hooks A background watchdog (30s interval) plus immediate checks on network change events detect when another program replaces the entire pf ruleset (e.g., Windscribe's `pfctl -f /etc/pf.conf`). When detected, ctrld rebuilds its anchor with up-to-date tunnel interface rules and re-injects the anchor reference at the top of the ruleset. A 2-second delayed re-check catches race conditions where the other program clears rules slightly after the network event. +A noise delta starts no immediate check and no delayed re-check. The watchdog keeps its own 30 s cadence, so it still restores a missing anchor during a delta storm. + ### 4a. Active Interception Probe (pf Translation State Corruption) Programs like Parallels Desktop reload `/etc/pf.conf` when creating/destroying virtual network interfaces (bridge100, vmenet0). This can corrupt pf's internal translation engine — rdr rules survive in text form but stop evaluating, causing DNS interception to silently fail while the watchdog reports "intact." @@ -639,6 +645,8 @@ Network change detected (netmon callback) └─ (same as 2s — catches slower VPN teardowns) ``` +A noise delta skips this whole chain. ctrld runs no pf read, no tunnel discovery, no OS resolver read, and no VPN DNS refresh for it. The noise class holds the AirDrop interfaces (`awdl*`, `llw*`) and the virtual adapters. The pf watchdog keeps its own 30 s cadence, so it still tests the rules during a delta storm. + ### VPN Connect Sequence 1. VPN creates tunnel interface (e.g., utun420) diff --git a/docs/internal-domains.md b/docs/internal-domains.md index 44caad6..1630b8d 100644 --- a/docs/internal-domains.md +++ b/docs/internal-domains.md @@ -153,7 +153,10 @@ removed domain leaves no rule and no `internal_*` upstream behind. Generated `internal_*` upstreams carry the organization's private resolver addresses, so anything naming one stays at debug, and the level above it carries only a classification (`timeout`, `refused`, `unreachable`, `source_unavailable`, -`error`). That covers every consumer, not just query handling: +`error`). The per-query error sampler applies to the classification line as +well. After 5 lines in one minute for the same upstream and class, the next +classification lines go to debug, and one `Per-query errors sampled` line gives +the count. That covers every consumer, not just query handling: - the bootstrap-IP line in `setupUpstream`; - the transport error from a failed query, which names the endpoint; diff --git a/docs/network-recovery-diagnostics.md b/docs/network-recovery-diagnostics.md index 9f90c2d..240668e 100644 --- a/docs/network-recovery-diagnostics.md +++ b/docs/network-recovery-diagnostics.md @@ -1,14 +1,148 @@ # Network-recovery diagnostics -Use continuous `ctrld log tail` capture during a reproduction. Internal logs rotate by size. -A later `log send` can contain recent debug events and older warnings. -Match timestamps and IDs. The last retained error does not necessarily describe the current network. -Issue #604 tracks upload-time device and version headers separately. +The retained stream is the journal, the file `ctrld-journal.log` in the ctrld home directory. +It keeps every warning and every error, plus every event marked `journal=true`. +The journal is on disk, so it keeps these events across a restart and a self-upgrade. +Its budget is 2 MB with 2 backup files. +Every `ctrld log send` upload holds the whole journal, so a reproduction does not need a continuous `ctrld log tail` capture. +Match timestamps and IDs. The last retained error does not describe the current network in every case. +Each upload starts with a header line that names the build, the intercept mode, and the network at send time. + +## Journal events + +These events describe the host network. Each one carries `journal=true`, so it stays on disk after a rotation and after a restart. +netmon reports every network change as a delta, which holds the state before the change and the state after it. +The table gives the trigger, the level, and the fields of each event. + +| Event | Trigger | Level | Fields | +|---|---|---|---| +| `Network snapshot` | Start, an accepted transition, a recovery begin, and a recovery end. Two limits hold some of them back | info | `trigger` (`start`, `transition`, `recovery_begin`, `recovery_end`), `network` | +| `Network interface changed` | One event per changed interface of a delta that is not noise. A mixed delta gives an event for the interfaces outside the noise class only | info | `transition_id`, `interface`, `action`, `class`, `hardware_port`, `service`, `ips_before`, `ips_after`, `flags`, `mtu`, `is_default_route` | +| `Network transition` | The end of every network callback that is not noise | info, debug for the outcomes `ignored` and `snapshot_superseded` | `transition_id`, `outcome`, `is_major_change`, `changed`, `interface`, `changed_interfaces`, `time_jumped`, `default_route_before`, `default_route_after`, `have_v4_before`, `have_v4_after`, `have_v6_before`, `have_v6_after`, `source_ipv4_before`, `source_ipv4_after`, `source_ipv6_before`, `source_ipv6_after`, `source_snapshot_available` | +| `Network delta noise` | Each noise delta, and one journal line per 10 minutes of a storm | debug, info for the journal line | `interfaces`, `count`, `first_at`, `last_at` | +| `Host woke` | One event per wake: a time jump in a netmon delta, or the macOS wake detector, which runs in DNS intercept mode only | info | `gap_known`, `gap_ms` (only when `gap_known` is true), `source` (`netmon`, `detector`), `default_route`, `interfaces_up` | +| `OS resolver set changed` | A nameserver read that differs from the read before | info | `source` (`scutil`, `dhcp`, `resolv.conf`, `unknown`), `before`, `after`, `default_route`, `reason` (`start`, `recovery`, `delayed_recheck`, `wake_probe`, `vpn_settle`, `unspecified`) | +| `Reinitialized OS resolver with nameservers: …` | ctrld read the nameservers of the system again and built the OS resolver again | info | none, the nameservers are part of the message | +| `DNS configuration changed` | A `scutil --dns` poll that finds a changed resolver, one event per resolver (macOS) | info | `nameservers`, `if_index`, `interface`, `scoped`, `action` (`added`, `changed`, `removed`), `flags`, `search_domain_count`, `order`, `reachable` | +| `Recovery begin` | The start of every recovery | info | `transition_id`, `recovery_generation`, `recovery_reason`, `intercept` | +| `Recovery end` | The end of every recovery | info, warn for a canceled recovery and for a recovery with a failure | `transition_id`, `recovery_generation`, `recovery_reason`, `outcome`, `had_failure`, `duration_ms`, `recovered_upstream`, `dhcp_servers`, `dhcp_server_count`, `intercept_target_action`, `bypass_active` | +| `Upstream … recovered …` | An upstream that was down answers again | info | `down_for_ms` | +| `Recovery canceled with no successor …` | A recovery stops and no other recovery follows it | info | none | +| `DNS intercept recovery: enabling DHCP bypass …` | ctrld sends the queries of a recovery to the OS resolver | info | none | +| `DNS intercept recovery complete: disabling DHCP bypass …` | The recovery ends and the normal query path returns | info | none | +| `DNS intercept recovery: found DHCP nameservers: …` | ctrld read the DHCP nameservers of the service | info | `dhcp_servers` | +| `DNS intercept recovery: no DHCP nameservers found` | The service gave no DHCP nameserver | warn | none | +| `intercept DNS target: service … set …` | ctrld wrote the loopback target into a network service | warn | `service`, `target`, `reason` | +| `intercept DNS target: removed …` | ctrld took the loopback target out of the service | info | `service`, `target`, `reason` | +| `intercept DNS target: … DNS changed externally …` | Another program changed the DNS of the service, so the target of ctrld is gone (macOS) | info | `service`, `target`, `reason` (`external_change`) | +| `dns64: discovered NAT64 prefix …` | A NAT64 prefix that differs from the result before | info | none | +| `dns64: no NAT64 prefix present …` | A no-prefix result that differs from the result before | info | none | +| `DNS intercept: interface appeared/disappeared …` | An interface appeared or disappeared, and the probe monitor starts | info | `interface`, `class`, `action` | +| `PF anchor list changed` | A missing anchor, a restored anchor, a stabilization start, and a stabilization end (macOS) | warn for the reason `missing`, info for the rest | `reason`, `anchors`, `anchors_known`, `pf_enabled`, `pf_since` | +| `Tunnel interface changed` | The set of tunnel interfaces changes (macOS). A retry of the same change writes no second event | info | `added`, `removed`, `owner` | +| `DNS intercept: VPN connecting …` | A new tunnel interface starts the stabilization mode (macOS) | info | none | +| `DNS intercept: pf stable for …` | The pf ruleset stops changing and stabilization ends (macOS) | info | none | +| `Query health` | A change of the health class that holds for two evaluations, and a heartbeat every 15 minutes | info | `class`, `window_s`, `queries`, `failed_queries`, `cache_hits`, `failures_by_class`, `upstreams_down`, `bypass_active`, `cache_hit_ratio` | + +No event in this catalog holds a query name, an upstream URL, or a token. +The journal does hold network addresses. It holds the LAN and VPN resolver addresses, the gateways, the addresses of the host, and the DHCP servers of a service. +It holds no search suffix. `DNS configuration changed` counts the suffixes in `search_domain_count` and logs the count alone. +The field `owner` of `Tunnel interface changed` names the VPN product that owns the tunnel. On Windows the field `hardware_port` names the VPN product of an adapter in the same way. +The Wi-Fi network name is never logged. On macOS 14 and later, only a process with the Location Services permission can read it, and a daemon cannot request that permission. + +### The network object + +The `network` object describes the host network at one moment. +The `Network snapshot` event and the header line of every log file render the same object, with these fields: + +- Routes: `default_route_v4`, `default_route_v6`, `gateway_v4`, `gateway_v6` +- Families: `have_v4`, `have_v6` +- Interfaces: `interfaces`, an array of `name`, `class`, `up`, `ips`, `hardware_port`, `service`, and the count `interfaces_omitted` +- Resolvers: `resolvers`, the host and the port of each OS nameserver +- Source addresses: `source_ipv4`, `source_ipv6` +- Intercept state: `intercept_target`, `dns_less_target`, `bypass_active`, `recovery_running`, `pf_stabilizing` +- Link: `link_type`, `tethered`, `clat_present`, `nat64_prefix` + +The field `default_route_interface` is gone. Read `default_route_v4` and `default_route_v6` in its place. +The two routes and the two gateways come from the route table of macOS, one route per address family. A tunnel that owns the default route has no next hop, so its route names the tunnel and its gateway stays empty. +Windows and Linux read no route table in this release. They fill `default_route_v4` from the network monitor and leave `default_route_v6`, `gateway_v4`, and `gateway_v6` empty. + +`link_type` is one of `wifi`, `ethernet`, `usb_tether`, `tunnel`, and `unknown`. +macOS reads it from the name of the hardware port. Linux reads `wifi`, `ethernet`, `usb_tether`, and `tunnel` from the kernel. +Windows reads the description of the adapter in place of a port name. The words Ethernet, LAN, and Thunderbolt win over the tunnel class on every platform, so a VPN adapter with a description such as `SSL VPN Virtual Ethernet Adapter` reads as `ethernet`. +Windows gives `tunnel` to the other VPN adapters. The other links are `unknown`. +The field `service` names a macOS network service. It stays empty on Windows and on Linux. +`tethered` is true on every platform when the interface of the default route holds an address of the range 172.20.10.0/28. +It is also true when the default gateway is in that range, and when the hardware port of the default route is `iPhone USB`. + +The list `interfaces` holds the interfaces that are up or hold an address, and it stops at 32 entries. +`interfaces_omitted` counts the interfaces that the list leaves out, the ones over the limit included. +A container host holds hundreds of interfaces, and a list of all of them fills the journal. + +ctrld does not write a snapshot for every trigger. A snapshot equal to the snapshot written last reaches no journal. +Every other snapshot waits 60 s after the snapshot written last. A new value of `default_route_v4`, `default_route_v6`, or `resolvers` passes that wait. +The trigger `start` always writes. + +### Noise class + +A noise delta changes interfaces of the noise class only, and it does not move the default route. +The noise class holds `awdl*` and `llw*` on macOS. +On Linux it holds the names `docker*`, `veth*`, `virbr*`, and `br-*`. A device of the tun or tap type is a tunnel, not a noise interface. +On Windows it holds the virtual adapters without a connector. A Windows adapter whose description names TAP, Wintun, WireGuard, or VPN is a tunnel, not a noise interface. +On Linux and on Windows, an interface that holds a global address on either side of the delta is never noise. +A delta is also not noise when it flips `have_v4` or `have_v6`, or when it carries a time jump. +A noise delta runs no reconcile work. +ctrld starts no `scutil` process, reads no PF rule, discovers no tunnel, refreshes no VPN DNS, and schedules no delayed recheck for it. +A delta that touches a hardware port and a noise interface in the same callback is not noise. +The PF watchdog keeps its own 30 s timer, so it still tests the PF rules during a delta storm. + +Each noise delta logs one debug line. The first noise delta of a storm also writes a journal line with `count` 1. +While the storm lasts, one journal line of the same name arrives every 10 minutes, with the count of that window. +AirDrop in the Finder is the common source of such a storm on macOS. + +### DNS configuration poll (macOS) + +ctrld polls `scutil --dns` on macOS only. The first poll runs at start. +The poll runs every 15 s for 10 minutes after a network change, a wake, or the end of a recovery. +Each of these three events starts the fast cadence at once, even while the poll waits. +At rest the poll runs every 5 minutes. Each changed resolver of a poll gives one `DNS configuration changed` event. + +### Change-only lines + +These lines log at debug level on change only. Each one carries `repeats`, the number of equal results before it: + +- `Got system nameservers` +- `Final available nameservers` +- `pf anchor intact` +- `discovered active tunnel interfaces` +- `VPN DNS refresh completed` + +### PF anchor list (macOS) + +A `PF anchor list changed` event reaches the journal when its reason, its anchor list, or `pf_enabled` differs from the event before it. +The field `pf_since` is no part of that test, because the uptime of pf grows between two reads of one state. +A failed read of the anchor names gives `anchors_known` false and no anchor list. + +### Host woke + +ctrld writes one `Host woke` event per wake. Two sources that report the same wake inside 30 s give one event. +The macOS wake detector runs in DNS intercept mode only. Outside that mode, and on Windows and Linux, the netmon delta is the only source. +The netmon source knows no gap, so its report waits 8 s for the detector. A detector report inside that hold gives the event, with `gap_known` true and the length of the sleep in `gap_ms`. Without a detector report, the netmon report goes out with `gap_known` false and no `gap_ms`. Every later report of the same wake is dropped. + +### Query health + +ctrld grades the query path over a window of 15 minutes and reads the window every minute. +Each failed client query counts one failure, whatever number of upstreams that query tried. +`failures_by_class` keeps one count per failure class. The class `internal_domain` holds the failures of an Internal Domain resolver. +The grade leaves the class `internal_domain` out, and `upstreams_down` counts no Internal Domain resolver. +A laptop away from the company network cannot reach these resolvers. Without this rule the grade stays degraded for the whole day. +A change of the class reports after it holds for two reads of the window. ## Transitions and recovery -`Network transition` debug events include a `transition_id`, affected interface, default route, and stored source addresses before and after the callback. -The outcome identifies an accepted, ignored, superseded, or unusable change. +`Network transition` events include a `transition_id`, affected interface, default route, and stored source addresses before and after the callback. +The outcome identifies an accepted, ignored, superseded, snapshot_superseded, or unusable change. +A callback whose state read a newer snapshot gives the outcome `snapshot_superseded` at debug level with `transition_id` 0, and it gives no interface event. Newer accepted changes supersede pending recovery. Ignored changes do not cancel accepted recovery. Source commits validate addresses against the current network state. Netmon does not order minor callbacks within one major cache epoch. @@ -16,11 +150,12 @@ For minor callbacks and late major callbacks in that epoch, ctrld reads current This read does not change host DNS. A failed read defers source writes, but reconciliation continues. `Removed stale resolver source` is a warning only when ctrld removes an invalid address. -The warning gives the reason and survives debug rotation. +The warning gives the reason. Every warning enters the journal, so it stays on disk after a debug rotation and after a restart. `Recovery begin` and `Recovery end` connect the transition to a `recovery_generation`, reason, outcome, and elapsed time. Only the first upstream failure in each recovery produces a bounded summary. -If a recovery encounters a failure, its final outcome also enters the warning stream. -Normal successful recovery stays at debug level. +Every recovery outcome enters the journal. +A canceled recovery and a recovery with a failure log at warn level. +For the outcome `superseded`, `intercept_target_action` is `unchanged` and `bypass_active` is false. The recovery that follows owns both of them. The outcome `failed` means that the preparation of the recovery reported an error, and the event is at warn level. These summaries exclude upstream URLs and queried names. Each PF probe records the recovery generation at its start. @@ -45,9 +180,9 @@ Note: On the 1.x (`v1.0`) line, the previous message was `Recovery canceled; DNS A changed unsuccessful condition produces a warning. The condition consists of the stage, error code, outcome, and target. Equal results stay at debug level. The next changed condition includes the preceding repeat count. -The first restored receipt also produces a warning-stream event. This event prevents an older failure from appearing current. +The first restored receipt also produces a warning, which enters the journal. This event prevents an older failure from appearing current. Normal receipts and shutdown cancellation stay at debug level. -The warning buffer is finite. These events do not guarantee indefinite retention. +The journal budget is finite and the journal rotates by size. It does not keep these events without a limit. The outcomes have these meanings: diff --git a/docs/pf-dns-intercept.md b/docs/pf-dns-intercept.md index 876aa3a..3e53e75 100644 --- a/docs/pf-dns-intercept.md +++ b/docs/pf-dns-intercept.md @@ -249,6 +249,13 @@ The monitor probes with exponential backoff (0, 0.5, 1, 2, 4s) to win the race against async pf reloads. Only one monitor runs at a time (singleton). The watchdog also runs the probe every 30s as a safety net. +A network delta that touches AirDrop or virtual adapters alone is noise. The +handler returns at once for it, so no probe monitor starts and no pf read runs. +A delta that moves the default route, that flips the v4 or the v6 family, or +that carries a time jump is not noise, whatever its interfaces are. +The watchdog keeps its own 30 s cadence, so it still tests the rules during a +delta storm. + The full pf reload is VPN-safe: it reassembles from `pfctl -sr` + `pfctl -sn` (the current running state), preserving all existing anchors and rules. diff --git a/docs/runtime-internal-logging.md b/docs/runtime-internal-logging.md index 982632c..acee7a6 100644 --- a/docs/runtime-internal-logging.md +++ b/docs/runtime-internal-logging.md @@ -1,46 +1,107 @@ -# Runtime Internal Logging - -When no logging is configured (i.e., `log_path` is not set), ctrld automatically enables an internal logging system. This system stores logs in memory to provide troubleshooting information when problems occur. +# Runtime internal logging ## Purpose -The runtime internal logging system is designed primarily for **ctrld developers**, not end users. It captures detailed diagnostic information that can be useful for troubleshooting issues when they arise, especially in production environments where explicit logging may not be configured. +ctrld writes its own log when the configuration sets no log file. The log gives Control D support and ctrld developers the evidence they need after a problem. Two of the three streams are files on disk. They keep their content across a restart and a self-upgrade. -## When It's Enabled +## When it is on -Internal logging is automatically enabled when: +Internal logging is on when these three conditions are true at the same time: -- ctrld is running in Control D mode (i.e., `--cd` flag is provided) -- No log file is configured (i.e., `log_path` is empty or not set) +- ctrld runs in Control D mode, with the `--cd` flag +- The configuration sets no `log_path` +- Silent mode is off -If a log file is explicitly configured via `log_path`, internal logging will **not** be enabled, as the configured log file serves the logging purpose. +When `log_path` is set, ctrld writes that file and starts no internal stream. The journal needs internal logging, so `log_path` mode has no journal. -## How It Works +## Streams and files -The internal logging system: +ctrld writes three streams. The debug file and the journal file live in the ctrld home directory, next to each other. -- Stores logs in **in-memory buffers** (not written to disk) -- Captures logs at **debug level** for normal operations and **warn level** for warnings -- Maintains separate buffers for normal logs and warning logs -- Automatically manages buffer size to prevent unbounded memory growth -- Preserves initialization logs even when buffers overflow +| Stream | Level filter | File | Budget | Survives a restart | +|---|---|---|---|---| +| Console | Notice and above. The `-v` flag adds the lower levels | none | none | No | +| Debug | Every level | `ctrld.log` | 10 MB, 4 backup files | Yes | +| Journal | Warning and above, notice included, plus every event with `journal=true` | `ctrld-journal.log` | 2 MB, 2 backup files | Yes | -## Configuration +The `-v` flag changes the console level and the level of the `log_path` file. Without the flag, the console shows notice and above. With `-v` it shows info and above. With `-vv` it shows every line. The debug file and the journal keep their filters in each case. -**Important**: The `log_level` configuration option does **not** affect the internal logging system. Internal logging always operates at debug level for normal logs and warn level for warnings, regardless of the `log_level` setting in the configuration file. +A backup file carries a number suffix. The debug backup files are `ctrld.log.1` to `ctrld.log.4`, and `ctrld.log.1` holds the newest rotated content. The journal backup files are `ctrld-journal.log.1` and `ctrld-journal.log.2`. -The `log_level` setting only affects: -- Console output (when running interactively) -- File-based logging (when `log_path` is configured) +Rotation is by size. A write that crosses the size limit rotates the file first. ctrld deletes the file beyond the backup count. -## Accessing Internal Logs +Each stream also keeps a memory buffer: 5 MB for the debug stream and 1 MB for the journal. These buffers serve `ctrld log tail` and the fallback path when a file cannot open. The files hold the history. -Internal logs can be accessed through the control server API endpoints. This functionality is intended for developers and support personnel who need to diagnose issues. +A restart deletes the memory buffers and keeps the files. The new process appends a header line, which marks the restart point inside each file. + +## Header and rotation events + +Every debug file and every journal file starts with one JSON line, the header. ctrld writes the header when it opens a file at start and after every rotation. On an existing file at start, ctrld appends the header. The journal holds JSON lines. The other lines of the debug file have the text format of the console. + +The message of the line is `Log header`. The line carries these fields: + +- Build: `version` and `commit` +- Host: `os`, `arch`, `pid`, and `start_time` +- Mode: `intercept_mode`, `listeners`, `upstream_count`, and `upstream_types` +- Identity: `resolver_uid` +- Files: `log_files` +- Network: `network`, the network snapshot of the host + +The header holds no provisioning token, no upstream endpoint URL, and no query name. `upstream_types` holds the type words, for example `doh` or `os`. `listeners` holds the listen addresses. + +The `network` object holds the routes, the gateways, the interfaces, the resolvers, the link type, and the intercept state. It has no `default_route_interface` field. The runbook [Network-recovery diagnostics](network-recovery-diagnostics.md) lists every field of the object. + +After each rotation ctrld logs one `Log rotated` event at info level, with the fields `file`, `bytes_written`, `first_event_at`, `last_event_at`, and `backups`. This event is a journal event, so it lands in the debug stream and in the journal. + +## Network state events + +The journal holds the events that describe the host network. `Network snapshot` renders the `network` object at start, at each accepted transition, and at each begin and end of a recovery. +A snapshot equal to the snapshot written last reaches no journal, and one snapshot per 60 s is the limit. A new default route and a new resolver set pass that limit. `Network interface changed`, `Network transition`, `Host woke`, `OS resolver set changed`, and `DNS configuration changed` name each change of the network. `Recovery begin`, `Recovery end`, `PF anchor list changed`, and `Tunnel interface changed` name the repair work. + +AirDrop and virtual adapters produce a storm of network callbacks. ctrld puts these interfaces in a noise class. A delta that touches noise interfaces alone skips the handler. The first delta of a storm writes one `Network delta noise` journal line, and one more line covers each 10 minutes of the storm. + +`Query health` grades the query path over a window of 15 minutes. It logs at each change between the classes `healthy`, `degraded`, and `failing`, and once every 15 minutes as a heartbeat. A class change reports after it holds for two reads of the window. + +The runbook [Network-recovery diagnostics](network-recovery-diagnostics.md) lists every event with its trigger, its level, and its fields. + +## Sampling of per-query errors + +A broken network produces one error line per query. ctrld bounds this flood for each error class and each upstream. The first 5 lines in a 60 s window stay at error level. Later lines in the same window drop to debug level, so the debug stream stays complete. The console and the `log_path` file show these lines only at debug level. The debug file keeps them. + +When a window closes, ctrld logs one `Per-query errors sampled` summary at error level. The summary carries `class`, `upstream`, `count`, `suppressed`, and `window_s`. ctrld logs the summary only when the window suppressed at least one line. + +An upstream logs one `Upstream state changed` event per transition. The down event is at warn level and carries `state` `down`, `failure_count`, and `reason`. The up event is at info level and carries `state` `up`, `down_for_ms`, and `failure_count`. Both are journal events. + +## Configuration keys + +The `log_level` key does not change the internal streams. The debug file keeps every level, and the journal keeps the warnings, the notices, the errors, and the journal events. A notice shares the level value of a warning. The `log_level` key changes the `log_path` file only. The `-v` flag changes the console level and the level of the `log_path` file, as the section "Streams and files" explains. + +Two keys in the `[service]` section size the debug stream: + +- The key `log_max_size_mb` sets the size limit of `ctrld.log` in MB. The default is 10. +- The key `log_max_backups` sets the number of backup files. The default is 4. The value 0 keeps no backup file. + +Both keys also apply to the `log_path` file, which rotates the same way. The journal budget is fixed. + +## Uploads + +`ctrld log send` uploads the log to Control D. The body holds these parts, in this order: + +- One header line rendered at send time, with the field `trigger` set to `send` +- The newest 10 MB of the debug files, oldest file first, cut at a line boundary +- The marker `=== LOG_END ===` +- Every journal file, oldest first + +The `network` object of the send-time header comes from a fresh read of the interfaces and of the route table. The resolvers come from the list that ctrld stored at its last resolver initialization, not from a fresh read. `ctrld log send --full` sends every debug file in place of the newest 10 MB. The reported size equals the number of bytes uploaded. + +`ctrld log view` prints the same composition on the terminal and takes the same `--full` flag. ctrld accepts one upload per minute. The body is plain text without gzip. + +In `log_path` mode the body holds the send-time header, then the `log_path` file and its backup files, oldest first. This body has no marker and no journal. + +`ctrld log tail` follows the live debug stream and prints each new line, journal events included. ## Notes -- Internal logging is **not** a replacement for proper log file configuration in production environments -- For production deployments, it is recommended to configure `log_path` to enable persistent file-based logging -- Internal logs are stored in memory and will be lost if the process terminates unexpectedly -- The internal logging system is automatically disabled when explicit logging is configured - +- Internal logging is not a replacement for a configured `log_path` file in production. +- The journal is small on purpose. It keeps the events that describe a problem, not the events of each query. +- Support reads the upload without access to the host. The header line names the build, the mode, and the network, so support does not have to ask for them. diff --git a/internal/controld/config.go b/internal/controld/config.go index 731e95e..d3fd4c0 100644 --- a/internal/controld/config.go +++ b/internal/controld/config.go @@ -10,6 +10,7 @@ import ( "io" "net" "net/http" + "os" "runtime" "slices" "strings" @@ -330,12 +331,21 @@ func SendLogs(ctx context.Context, lr *LogsRequest, cdDev bool) error { apiUrl = logURLDev } + body, size, getBody, cleanup, err := spoolLogBody(lr.Data) + if err != nil { + ctrld.Log(ctx, logger.Error(), "Failed to spool the log upload: %v", err) + return err + } + defer cleanup() + ctrld.Log(ctx, logger.Debug(), "Creating HTTP request for log upload") - req, err := http.NewRequestWithContext(ctx, "POST", apiUrl, lr.Data) + req, err := http.NewRequestWithContext(ctx, "POST", apiUrl, body) if err != nil { ctrld.Log(ctx, logger.Error(), "Failed to create HTTP request: %v", err) return fmt.Errorf("http.NewRequest: %w", err) } + req.ContentLength = size + req.GetBody = getBody q := req.URL.Query() q.Set("uid", lr.UID) req.URL.RawQuery = q.Encode() @@ -590,6 +600,15 @@ func doWithFallback(ctx context.Context, client *http.Client, req *http.Request, ipReq := req.Clone(req.Context()) ipReq.Host = apiIp ipReq.URL.Host = apiIp + // The first attempt consumed the body. A request that can give its body + // again sends the whole body a second time. + if req.GetBody != nil { + body, bodyErr := req.GetBody() + if bodyErr != nil { + return nil, fmt.Errorf("request failed: %w; fallback to direct ip %s failed: %w", err, apiIp, bodyErr) + } + ipReq.Body = body + } resp, fallbackErr := client.Do(ipReq) if fallbackErr != nil { return nil, fmt.Errorf("request failed: %w; fallback to direct ip %s failed: %w", err, apiIp, fallbackErr) @@ -597,6 +616,34 @@ func doWithFallback(ctx context.Context, client *http.Client, req *http.Request, return resp, nil } +// spoolLogBody copies an upload to a temporary file, so the request can go +// out a second time to the direct IP of the API. A log reader yields its +// bytes once, and the first attempt consumes them. The caller runs cleanup +// after the response arrived. +func spoolLogBody(data io.Reader) (body *os.File, size int64, getBody func() (io.ReadCloser, error), cleanup func(), err error) { + f, err := os.CreateTemp("", "ctrld-log-upload-*") + if err != nil { + return nil, 0, nil, nil, fmt.Errorf("creating the upload spool: %w", err) + } + cleanup = func() { + _ = f.Close() + _ = os.Remove(f.Name()) + } + size, err = io.Copy(f, data) + if err != nil { + cleanup() + return nil, 0, nil, nil, fmt.Errorf("writing the upload spool: %w", err) + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + cleanup() + return nil, 0, nil, nil, fmt.Errorf("rewinding the upload spool: %w", err) + } + // The transport closes the body of a failed attempt, so the second body + // is a new handle on the same file. + getBody = func() (io.ReadCloser, error) { return os.Open(f.Name()) } + return f, size, getBody, cleanup, nil +} + // apiServerIP returns the direct IP to connect to API server. func apiServerIP(cdDev bool) string { if cdDev { diff --git a/internal/controld/log_upload_test.go b/internal/controld/log_upload_test.go new file mode 100644 index 0000000..c15ce6f --- /dev/null +++ b/internal/controld/log_upload_test.go @@ -0,0 +1,49 @@ +package controld + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// TestDoWithFallbackResendsTheSpooledBody drives the two-attempt upload: the +// first attempt fails on a closed port and the fallback to the direct IP must +// carry the whole body, which the first attempt consumed. +func TestDoWithFallbackResendsTheSpooledBody(t *testing.T) { + payload := strings.Repeat("log line\n", 4096) + var got string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + got = string(b) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + body, size, getBody, cleanup, err := spoolLogBody(strings.NewReader(payload)) + if err != nil { + t.Fatalf("spoolLogBody: %v", err) + } + defer cleanup() + if size != int64(len(payload)) { + t.Fatalf("size = %d, want %d", size, len(payload)) + } + // Port 1 refuses the connection, so the first attempt fails at once. + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, "http://127.0.0.1:1/logs", body) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + req.ContentLength = size + req.GetBody = getBody + + resp, err := doWithFallback(context.Background(), srv.Client(), req, strings.TrimPrefix(srv.URL, "http://")) + if err != nil { + t.Fatalf("doWithFallback: %v", err) + } + resp.Body.Close() + if got != payload { + t.Fatalf("the fallback carried %d bytes, want %d", len(got), len(payload)) + } +} diff --git a/log.go b/log.go index ea79862..e02a0c0 100644 --- a/log.go +++ b/log.go @@ -259,3 +259,88 @@ func (l *Logger) Write(p []byte) (n int, err error) { func (l *Logger) Printf(format string, v ...any) { l.Info().Msgf(format, v...) } + +// JournalField marks an event for the retained journal stream. +const JournalField = "journal" + +// Journal marks an event for the retained journal stream. The journal core +// keeps every line that carries the field, whatever the level of the line is. +func Journal(e *LogEvent) *LogEvent { + return e.Bool(JournalField, true) +} + +// Time adds a time field to the event +func (e *LogEvent) Time(key string, val time.Time) *LogEvent { + e.fields = append(e.fields, zap.Time(key, val)) + return e +} + +// Float64 adds a float64 field to the event +func (e *LogEvent) Float64(key string, val float64) *LogEvent { + e.fields = append(e.fields, zap.Float64(key, val)) + return e +} + +// Dict starts a nested object. Dict on an event adds it under a key. +func Dict() *LogEvent { + return &LogEvent{} +} + +// Dict adds the fields of dict as a nested object under key +func (e *LogEvent) Dict(key string, dict *LogEvent) *LogEvent { + e.fields = append(e.fields, zap.Object(key, logFields(dict.fields))) + return e +} + +// Array adds an array field to the event +func (e *LogEvent) Array(key string, arr *LogArray) *LogEvent { + e.fields = append(e.fields, zap.Array(key, arr)) + return e +} + +// logFields renders a field list as one nested object +type logFields []zap.Field + +// MarshalLogObject implements zapcore.ObjectMarshaler +func (f logFields) MarshalLogObject(enc zapcore.ObjectEncoder) error { + for _, field := range f { + field.AddTo(enc) + } + return nil +} + +// LogArray holds the items of an array field. Arr starts one. +type LogArray struct { + items []func(zapcore.ArrayEncoder) error +} + +// Arr starts an array field. Array on an event adds it under a key. +func Arr() *LogArray { + return &LogArray{} +} + +// Dict appends a nested object to the array +func (a *LogArray) Dict(dict *LogEvent) *LogArray { + fields := logFields(dict.fields) + a.items = append(a.items, func(enc zapcore.ArrayEncoder) error { return enc.AppendObject(fields) }) + return a +} + +// Str appends a string to the array +func (a *LogArray) Str(val string) *LogArray { + a.items = append(a.items, func(enc zapcore.ArrayEncoder) error { + enc.AppendString(val) + return nil + }) + return a +} + +// MarshalLogArray implements zapcore.ArrayMarshaler +func (a *LogArray) MarshalLogArray(enc zapcore.ArrayEncoder) error { + for _, item := range a.items { + if err := item(enc); err != nil { + return err + } + } + return nil +} diff --git a/net.go b/net.go index 30799bf..27dd75b 100644 --- a/net.go +++ b/net.go @@ -6,8 +6,6 @@ import ( "sync/atomic" "time" - "tailscale.com/net/netmon" - ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) @@ -26,26 +24,16 @@ func HasIPv6(ctx context.Context) bool { val := ctrldnet.IPv6Available(ctx) ipv6Available.Store(val) logger.Debug().Msgf("ipv6 availability: %v", val) - mon, err := netmon.New(func(format string, args ...any) {}) - if err != nil { - logger.Debug().Err(err).Msg("Failed to monitor ipv6 state") - return - } - mon.RegisterChangeCallback(func(delta *netmon.ChangeDelta) { - old := ipv6Available.Load() - cur := delta.Monitor.InterfaceState().HaveV6 - if old != cur { - logger.Warn().Msgf("ipv6 availability changed, old: %v, new: %v", old, cur) - } else { - logger.Debug().Msg("ipv6 availability does not Changed") - } - ipv6Available.Store(cur) - }) - mon.Start() }) return ipv6Available.Load() } +// SetIPv6Available stores the IPv6 state that the network monitor reports, so +// one monitor serves the whole process. +func SetIPv6Available(v bool) { + ipv6Available.Store(v) +} + // DisableIPv6 marks IPv6 as unavailable if enabled. func DisableIPv6(ctx context.Context) { if ipv6Available.CompareAndSwap(true, false) { diff --git a/net_ipv6_test.go b/net_ipv6_test.go new file mode 100644 index 0000000..b37d01f --- /dev/null +++ b/net_ipv6_test.go @@ -0,0 +1,38 @@ +package ctrld + +import ( + "context" + "sync" + "testing" +) + +// resetIPv6Probe clears the one-time IPv6 probe. Test only: the process runs +// the probe once, so a test cannot see it without this reset. +func resetIPv6Probe(t *testing.T) { + t.Helper() + reset := func() { + hasIPv6Once = sync.Once{} + ipv6Available.Store(false) + } + reset() + t.Cleanup(reset) +} + +func TestHasIPv6KeepsTheProbeThenTakesTheStoredFlag(t *testing.T) { + resetIPv6Probe(t) + + probed := HasIPv6(context.Background()) + if got := HasIPv6(context.Background()); got != probed { + t.Fatalf("HasIPv6 = %v on the second call, want the probe value %v", got, probed) + } + + SetIPv6Available(!probed) + if got := HasIPv6(context.Background()); got == probed { + t.Fatalf("HasIPv6 = %v after SetIPv6Available(%v), want the stored value", got, !probed) + } + + SetIPv6Available(false) + if HasIPv6(context.Background()) { + t.Fatal("HasIPv6 = true after SetIPv6Available(false), want false") + } +} diff --git a/resolver.go b/resolver.go index 69a7357..3019c13 100644 --- a/resolver.go +++ b/resolver.go @@ -51,12 +51,64 @@ var controldPublicDnsWithPort = net.JoinHostPort(controldPublicDns, "53") var osResolverExchangeSem = make(chan struct{}, maxConcurrentOSResolverExchanges) var ( - resolverMutex sync.Mutex - or *osResolver + resolverMutex sync.Mutex + or *osResolver + // osResolverPtr holds the same resolver as or, for readers alone. An + // initialization holds resolverMutex across a scutil read of several + // seconds, and a log header render must not wait for it. + osResolverPtr atomic.Pointer[osResolver] defaultLocalIPv4 atomic.Value // holds net.IP (IPv4) defaultLocalIPv6 atomic.Value // holds net.IP (IPv6) ) +// storeOsResolver keeps the resolver of the process and the copy that readers +// take. The caller holds resolverMutex. +func storeOsResolver(resolver *osResolver) { + or = resolver + osResolverPtr.Store(resolver) +} + +// NameserversFn reads the nameservers of the system. A test of this package +// or of a package that drives the OS resolver replaces it to stay off the host. +var NameserversFn = nameservers + +// osResolverReasonUnspecified marks a resolver read that no caller explained. +const osResolverReasonUnspecified = "unspecified" + +// nameserverReads remembers the last nameserver list of one read. The system +// reports the same list many times, and only a change is news. +type nameserverReads struct { + seen bool + last []string + repeats int +} + +// record stores a new read. It reports whether the list changed, how many +// reads repeated the list before it, and the list that this read replaces. +func (n *nameserverReads) record(list []string) (changed bool, repeats int, before []string) { + if n.seen && slices.Equal(n.last, list) { + n.repeats++ + return false, n.repeats, n.last + } + before, repeats = n.last, n.repeats + n.seen = true + n.last = slices.Clone(list) + n.repeats = 0 + return true, repeats, before +} + +// osResolverLogState keeps the last nameserver reads and the reason of the +// initialization that runs. The read sits deep in the call chain, so the +// reason cannot travel as a parameter. +type osResolverLogState struct { + mu sync.Mutex + system nameserverReads + final nameserverReads + reason string +} + +var osResolverLog = osResolverLogState{reason: osResolverReasonUnspecified} + // LanQueryCtxKey is the context.Context key to indicate that the request is for LAN network. type LanQueryCtxKey struct{} @@ -101,8 +153,8 @@ func availableNameservers(ctx context.Context) []string { logger.Debug().Msgf("Added local IP to OS resolverexclusion map: %s", ipStr) } - systemNameservers := nameservers(ctx) - logger.Debug().Msgf("Got system nameservers: %v", systemNameservers) + systemNameservers := NameserversFn(ctx) + logSystemNameservers(ctx, systemNameservers) for _, ns := range systemNameservers { if _, ok := machineIPsMap[ns]; ok { @@ -113,19 +165,88 @@ func availableNameservers(ctx context.Context) []string { logger.Debug().Msgf("Added non-local nameserver: %s", ns) } - logger.Debug().Msgf("Final available nameservers: %v", nss) - + logFinalNameservers(ctx, nss) return nss } +// logSystemNameservers logs the system read at debug level on change only. +func logSystemNameservers(ctx context.Context, list []string) { + osResolverLog.mu.Lock() + changed, repeats, _ := osResolverLog.system.record(list) + osResolverLog.mu.Unlock() + if !changed { + return + } + Log(ctx, LoggerFromCtx(ctx).Debug().Int("repeats", repeats), + "Got system nameservers: %v", list) +} + +// logFinalNameservers logs the effective read at debug level on change only, +// and sends each change to the journal, where an outage report needs it. +func logFinalNameservers(ctx context.Context, list []string) { + osResolverLog.mu.Lock() + changed, repeats, before := osResolverLog.final.record(list) + reason := osResolverLog.reason + osResolverLog.mu.Unlock() + if !changed { + return + } + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug().Int("repeats", repeats), + "Final available nameservers: %v", list) + Journal(logger.Info()). + Str("source", osResolverSource()). + Strs("before", before). + Strs("after", list). + Str("default_route", defaultRouteInterfaceName()). + Str("reason", reason). + Msg("OS resolver set changed") +} + +// osResolverSource names the place where the platform holds the nameservers. +func osResolverSource() string { + switch runtime.GOOS { + case "darwin": + return "scutil" + case "windows": + return "dhcp" + case "linux", "dragonfly", "freebsd", "netbsd", "openbsd": + return "resolv.conf" + } + return "unknown" +} + +// defaultRouteInterfaceName returns the interface of the default route, and an +// empty string when the route table does not answer. +func defaultRouteInterfaceName() string { + name, err := netmon.DefaultRouteInterface() + if err != nil { + return "" + } + return name +} + +// setOsResolverReason keeps the reason of the initialization that runs. +func setOsResolverReason(reason string) { + osResolverLog.mu.Lock() + defer osResolverLog.mu.Unlock() + osResolverLog.reason = reason +} + // InitializeOsResolver initializes OS resolver using the current system DNS settings. // It returns the nameservers that is going to be used by the OS resolver. // // It's the caller's responsibility to ensure the system DNS is in a clean state before // calling this function. func InitializeOsResolver(ctx context.Context, guardAgainstNoNameservers bool) []string { - ns, _ := InitializeOsResolverWithSystemNameservers(ctx, guardAgainstNoNameservers) - return ns + return InitializeOsResolverWithReason(ctx, guardAgainstNoNameservers, osResolverReasonUnspecified) +} + +// InitializeOsResolverWithReason is InitializeOsResolver with the reason that +// the journal reports when the resolver set changes. +func InitializeOsResolverWithReason(ctx context.Context, guardAgainstNoNameservers bool, reason string) []string { + effective, _ := InitializeOsResolverWithSystemNameserversReason(ctx, guardAgainstNoNameservers, reason) + return effective } // InitializeOsResolverWithSystemNameservers initializes the OS resolver and @@ -133,9 +254,19 @@ func InitializeOsResolver(ctx context.Context, guardAgainstNoNameservers bool) [ // discovered from the system. The latter deliberately excludes synthetic // fallbacks added by initializeOsResolver. func InitializeOsResolverWithSystemNameservers(ctx context.Context, guardAgainstNoNameservers bool) (effective, system []string) { + return InitializeOsResolverWithSystemNameserversReason(ctx, guardAgainstNoNameservers, osResolverReasonUnspecified) +} + +// InitializeOsResolverWithSystemNameserversReason is +// InitializeOsResolverWithSystemNameservers with the reason that the journal +// reports when the resolver set changes. +func InitializeOsResolverWithSystemNameserversReason(ctx context.Context, guardAgainstNoNameservers bool, reason string) (effective, system []string) { resolverMutex.Lock() defer resolverMutex.Unlock() + setOsResolverReason(reason) + defer setOsResolverReason(osResolverReasonUnspecified) + system = availableNameservers(ctx) if system == nil { // A non-nil empty slice means discovery completed and found no DNS. @@ -146,7 +277,7 @@ func InitializeOsResolverWithSystemNameservers(ctx context.Context, guardAgainst if skip { return effective, system } - or = newResolverWithNameserver(effective) + storeOsResolver(newResolverWithNameserver(effective)) return effective, system } @@ -159,10 +290,10 @@ func osResolverNameserverSets(system []string, guardAgainstNoNameservers bool) ( // OsResolverNameservers returns the current OS resolver nameservers (host:port format). // Returns nil if the OS resolver has not been initialized. +// The read takes no lock, so a log header render never waits for a resolver +// initialization. func OsResolverNameservers() []string { - resolverMutex.Lock() - r := or - resolverMutex.Unlock() + r := osResolverPtr.Load() if r == nil { return nil } @@ -293,7 +424,7 @@ func NewResolver(ctx context.Context, uc *UpstreamConfig) (Resolver, error) { if or == nil { logger := LoggerFromCtx(ctx) logger.Debug().Msgf("Initialize new OS resolver") - or = newResolverWithNameserver(defaultNameservers(ctx)) + storeOsResolver(newResolverWithNameserver(defaultNameservers(ctx))) } resolverMutex.Unlock() return or, nil @@ -463,7 +594,10 @@ func (o *osResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error Log(ctx, logger.Debug(), "Resolving query: %s - %s", domain, dns.TypeToString[qtype]) msg, err := o.resolve(ctx, msg) if err != nil { - Log(ctx, logger.Error().Err(err), "OS resolver query failed: %s - %s", domain, dns.TypeToString[qtype]) + // One line for each failed query. The caller reports the failure at + // error level through its sampler, so this line stays at debug and + // out of the retained journal. + Log(ctx, logger.Debug().Err(err), "OS resolver query failed: %s - %s", domain, dns.TypeToString[qtype]) return nil, err } // If we got an answer, storing it to the hot cache for hotCacheTTL @@ -735,7 +869,7 @@ func initDefaultOsResolver(ctx context.Context) []string { defer resolverMutex.Unlock() if or == nil { logger.Debug().Msgf("Initialize new OS resolver with default nameservers") - or = newResolverWithNameserver(defaultNameservers(ctx)) + storeOsResolver(newResolverWithNameserver(defaultNameservers(ctx))) } nss := *or.lanServers.Load() nss = append(nss, *or.publicServers.Load()...) diff --git a/resolver_test.go b/resolver_test.go index 0cf7330..0ca5e7d 100644 --- a/resolver_test.go +++ b/resolver_test.go @@ -4,9 +4,13 @@ import ( "context" "crypto/rand" "encoding/hex" + "encoding/json" "errors" "net" "os" + "runtime" + "slices" + "strings" "sync" "sync/atomic" "syscall" @@ -14,6 +18,8 @@ import ( "time" "github.com/miekg/dns" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) func Test_osResolver_Resolve(t *testing.T) { @@ -918,3 +924,256 @@ func generateEdns0ServerCookie(clientCookie string) string { } return clientCookie + hex.EncodeToString(cookie) } + +// syncLogBuffer collects the log lines that any goroutine writes. +type syncLogBuffer struct { + mu sync.Mutex + sb strings.Builder +} + +func (b *syncLogBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.Write(p) +} + +func (b *syncLogBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.String() +} + +// captureProxyLog returns a context whose logger writes one JSON object for +// each line into the returned buffer, at debug level. The resolver code logs +// through the logger of its context. +func captureProxyLog(t *testing.T) (context.Context, *syncLogBuffer) { + t.Helper() + buf := &syncLogBuffer{} + encoderConfig := zap.NewProductionEncoderConfig() + encoderConfig.TimeKey = "time" + encoderConfig.LevelKey = "level" + encoderConfig.EncodeLevel = zapcore.LowercaseLevelEncoder + encoderConfig.MessageKey = "message" + core := zapcore.NewCore(zapcore.NewJSONEncoder(encoderConfig), zapcore.AddSync(buf), zapcore.DebugLevel) + return LoggerCtx(context.Background(), &Logger{Logger: zap.New(core)}), buf +} + +// logEventsWithPrefix returns the log events whose message starts with prefix. +func logEventsWithPrefix(t *testing.T, logs, prefix string) []map[string]any { + t.Helper() + var events []map[string]any + for _, line := range strings.Split(logs, "\n") { + if strings.TrimSpace(line) == "" { + continue + } + event := map[string]any{} + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("log line is not JSON: %q: %v", line, err) + } + message, _ := event["message"].(string) + if !strings.HasPrefix(message, prefix) { + continue + } + events = append(events, event) + } + return events +} + +// wantLogField fails the test when an event misses a field or holds another value. +func wantLogField(t *testing.T, event map[string]any, field string, want any) { + t.Helper() + got, ok := event[field] + if !ok { + t.Fatalf("log event has no field %q: %v", field, event) + } + if got != want { + t.Fatalf("log event field %q = %v, want %v", field, got, want) + } +} + +// wantLogStrings fails the test when a list field does not hold want. +func wantLogStrings(t *testing.T, event map[string]any, field string, want []string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("log event field %q is not a list: %v", field, event[field]) + } + got := make([]string, 0, len(values)) + for _, value := range values { + text, ok := value.(string) + if !ok { + t.Fatalf("log event field %q holds a value that is not a string: %v", field, value) + } + got = append(got, text) + } + if !slices.Equal(got, want) { + t.Fatalf("log event field %q = %v, want %v", field, got, want) + } +} + +// stubNameservers answers each system nameserver read from lists, in order. +// The last list answers every read after it. +func stubNameservers(t *testing.T, lists ...[]string) { + t.Helper() + previous := NameserversFn + reads := 0 + NameserversFn = func(context.Context) []string { + list := lists[min(reads, len(lists)-1)] + reads++ + return slices.Clone(list) + } + t.Cleanup(func() { NameserversFn = previous }) +} + +// resetOsResolverLog clears the change-only state, so one test does not see +// the nameserver reads of another. +func resetOsResolverLog(t *testing.T) { + t.Helper() + reset := func() { + osResolverLog.mu.Lock() + defer osResolverLog.mu.Unlock() + osResolverLog.system = nameserverReads{} + osResolverLog.final = nameserverReads{} + osResolverLog.reason = osResolverReasonUnspecified + } + reset() + t.Cleanup(reset) +} + +// keepOsResolver puts back the OS resolver that the process had before the test. +func keepOsResolver(t *testing.T) { + t.Helper() + resolverMutex.Lock() + previous := or + resolverMutex.Unlock() + t.Cleanup(func() { + resolverMutex.Lock() + storeOsResolver(previous) + resolverMutex.Unlock() + }) +} + +// TestOsResolverNameserversReadsWithoutTheResolverLock covers a log header +// render during a resolver initialization. That initialization holds +// resolverMutex across a scutil read of several seconds, and the header must +// not wait for it. +func TestOsResolverNameserversReadsWithoutTheResolverLock(t *testing.T) { + keepOsResolver(t) + resolverMutex.Lock() + storeOsResolver(newResolverWithNameserver([]string{"192.0.2.1:53"})) + + read := make(chan []string, 1) + go func() { read <- OsResolverNameservers() }() + + select { + case nameservers := <-read: + resolverMutex.Unlock() + if want := []string{"192.0.2.1:53"}; !slices.Equal(nameservers, want) { + t.Fatalf("nameservers = %v, want %v", nameservers, want) + } + case <-time.After(5 * time.Second): + resolverMutex.Unlock() + t.Fatal("OsResolverNameservers waited for resolverMutex") + } +} + +func TestJournalMarksTheEventAndKeepsItsLevel(t *testing.T) { + ctx, logs := captureProxyLog(t) + + Journal(LoggerFromCtx(ctx).Info()).Msg("Journal test event") + + events := logEventsWithPrefix(t, logs.String(), "Journal test event") + if len(events) != 1 { + t.Fatalf("got %d events, want 1: %s", len(events), logs.String()) + } + wantLogField(t, events[0], JournalField, true) + wantLogField(t, events[0], "level", "info") +} + +func TestOsResolverNameserverReadsLogOnChange(t *testing.T) { + ctx, logs := captureProxyLog(t) + resetOsResolverLog(t) + keepOsResolver(t) + first := []string{"192.0.2.1", "192.0.2.2"} + second := []string{"192.0.2.3"} + stubNameservers(t, first, first, second) + + for range 3 { + InitializeOsResolverWithReason(ctx, false, "transition") + } + + systemReads := logEventsWithPrefix(t, logs.String(), "Got system nameservers") + if len(systemReads) != 2 { + t.Fatalf("got %d system nameserver lines, want 2: %s", len(systemReads), logs.String()) + } + wantLogField(t, systemReads[0], "repeats", float64(0)) + wantLogField(t, systemReads[1], "repeats", float64(1)) + wantLogField(t, systemReads[1], "message", "Got system nameservers: [192.0.2.3]") + + finalReads := logEventsWithPrefix(t, logs.String(), "Final available nameservers") + if len(finalReads) != 2 { + t.Fatalf("got %d final nameserver lines, want 2: %s", len(finalReads), logs.String()) + } + wantLogField(t, finalReads[1], "repeats", float64(1)) + + changes := logEventsWithPrefix(t, logs.String(), "OS resolver set changed") + if len(changes) != 2 { + t.Fatalf("got %d resolver change events, want one for each changed list: %s", len(changes), logs.String()) + } + last := changes[1] + wantLogStrings(t, last, "before", first) + wantLogStrings(t, last, "after", second) + wantLogField(t, last, JournalField, true) + wantLogField(t, last, "level", "info") + wantLogField(t, last, "reason", "transition") + if _, ok := last["default_route"]; !ok { + t.Fatalf("resolver change event has no default_route field: %v", last) + } + wantSource := map[string]string{"darwin": "scutil", "windows": "dhcp", "linux": "resolv.conf"}[runtime.GOOS] + if wantSource != "" { + wantLogField(t, last, "source", wantSource) + } +} + +func TestInitializeOsResolverReportsTheUnspecifiedReason(t *testing.T) { + ctx, logs := captureProxyLog(t) + resetOsResolverLog(t) + keepOsResolver(t) + stubNameservers(t, []string{"192.0.2.10"}) + + InitializeOsResolver(ctx, false) + + changes := logEventsWithPrefix(t, logs.String(), "OS resolver set changed") + if len(changes) != 1 { + t.Fatalf("got %d resolver change events, want 1: %s", len(changes), logs.String()) + } + wantLogField(t, changes[0], "reason", "unspecified") + wantLogStrings(t, changes[0], "after", []string{"192.0.2.10"}) +} + +// TestOsResolverQueryFailureLogsAtDebug guards the level of the per-query +// failure line. The journal keeps every error line, and a broken network +// fails every query, so this line must not be an error. +func TestOsResolverQueryFailureLogsAtDebug(t *testing.T) { + ctx, logs := captureProxyLog(t) + resolver := newResolverWithNameserver([]string{"127.0.0.1:1"}) + resolver.exchangeDNS = func(_ context.Context, _ *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + return nil, 0, errors.New("dial udp: connection refused") + } + msg := new(dns.Msg) + msg.SetQuestion("private.example.", dns.TypeA) + + if _, err := resolver.Resolve(ctx, msg); err == nil { + t.Fatal("expected the query to fail") + } + + events := logEventsWithPrefix(t, logs.String(), "OS resolver query failed") + if len(events) == 0 { + t.Fatalf("no failure line: %s", logs.String()) + } + for _, event := range events { + if event["level"] != "debug" { + t.Fatalf("failure line at level %v, want debug: %v", event["level"], event) + } + } +}