diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index e8e238c..4c08083 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -1733,6 +1733,15 @@ func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig) erro mainLog.Load().Debug().Err(err).Msgf("Upstream %s check failed after %v (WFP loopback protect active)", upstream, duration) return errOsHealthcheckSuppressed } + // A no-route/network-unreachable failure means the endpoint's address + // family is available locally but unroutable (e.g. an IPv6 DoH endpoint + // while IPv6 is up but has no route). These repeat until the route + // returns and are handled by bounded backoff in the recovery loop, so + // keep them at debug to avoid sustained error-log spam. + if ctrldnet.IsUnreachable(err) { + mainLog.Load().Debug().Err(err).Msgf("Upstream %s check failed after %v (network unreachable)", upstream, duration) + return err + } mainLog.Load().Error().Err(err).Msgf("Upstream %s check failed after %v", upstream, duration) return err } @@ -1951,6 +1960,7 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string defer wg.Done() mainLog.Load().Debug().Msgf("Starting recovery check loop for upstream: %s", name) attempts := 0 + unreachableStreak := 0 for { select { case <-recoveryCtx.Done(): @@ -1971,8 +1981,22 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string } return } - mainLog.Load().Debug().Msgf("Upstream %s check failed, sleeping before retry", name) - if !sleepWithContext(recoveryCtx, checkUpstreamBackoffSleep) { + // Back off the retry cadence for an unroutable endpoint so a + // host with IPv6 up but no route to the IPv6 DoH endpoint does + // not re-bootstrap/re-check every checkUpstreamBackoffSleep and + // spam the log. The backoff is bounded (checkUpstreamUnreachableBackoffMax) + // so the endpoint is still re-probed and recovers when the route + // returns; any other failure resets to the base cadence. + sleep := checkUpstreamBackoffSleep + if ctrldnet.IsUnreachable(err) { + unreachableStreak++ + sleep = unreachableRecoveryBackoff(unreachableStreak) + mainLog.Load().Debug().Msgf("Upstream %s unreachable (streak %d), backing off %s before retry", name, unreachableStreak, sleep) + } else { + unreachableStreak = 0 + mainLog.Load().Debug().Msgf("Upstream %s check failed, sleeping before retry", name) + } + if !sleepWithContext(recoveryCtx, sleep) { return } diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 1bd1680..5da2490 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -34,6 +34,7 @@ import ( "github.com/Control-D-Inc/ctrld/internal/clientinfo" "github.com/Control-D-Inc/ctrld/internal/controld" "github.com/Control-D-Inc/ctrld/internal/dnscache" + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" "github.com/Control-D-Inc/ctrld/internal/router" "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" ) @@ -1343,13 +1344,14 @@ func errAddrInUse(err error) bool { var _ = errAddrInUse +// The unreachable winsock errnos (ENETUNREACH/EHOSTUNREACH) are matched via +// ctrldnet.IsUnreachable, which owns their definitions. +// // https://learn.microsoft.com/en-us/windows/win32/winsock/windows-sockets-error-codes-2 var ( windowsECONNREFUSED = syscall.Errno(10061) - windowsENETUNREACH = syscall.Errno(10051) windowsEINVAL = syscall.Errno(10022) windowsEADDRINUSE = syscall.Errno(10048) - windowsEHOSTUNREACH = syscall.Errno(10065) ) func errUrlNetworkError(err error) bool { @@ -1366,15 +1368,14 @@ func errNetworkError(err error) bool { if opErr.Temporary() { return true } + if ctrldnet.IsUnreachable(err) { + return true + } switch { case errors.Is(opErr.Err, syscall.ECONNREFUSED), errors.Is(opErr.Err, syscall.EINVAL), - errors.Is(opErr.Err, syscall.ENETUNREACH), - errors.Is(opErr.Err, syscall.EHOSTUNREACH), - errors.Is(opErr.Err, windowsENETUNREACH), errors.Is(opErr.Err, windowsEINVAL), - errors.Is(opErr.Err, windowsECONNREFUSED), - errors.Is(opErr.Err, windowsEHOSTUNREACH): + errors.Is(opErr.Err, windowsECONNREFUSED): return true } } diff --git a/cmd/cli/prog_test.go b/cmd/cli/prog_test.go index c8c17a8..562a8e0 100644 --- a/cmd/cli/prog_test.go +++ b/cmd/cli/prog_test.go @@ -33,6 +33,15 @@ func TestSleepWithContext(t *testing.T) { assert.Less(t, time.Since(start), 100*time.Millisecond) } +func TestUnreachableRecoveryBackoff(t *testing.T) { + // Streak starts at the base cadence and doubles each attempt, capped at the max. + assert.Equal(t, checkUpstreamBackoffSleep, unreachableRecoveryBackoff(0)) + assert.Equal(t, checkUpstreamBackoffSleep, unreachableRecoveryBackoff(1)) + assert.Equal(t, 2*checkUpstreamBackoffSleep, unreachableRecoveryBackoff(2)) + assert.Equal(t, 4*checkUpstreamBackoffSleep, unreachableRecoveryBackoff(3)) + assert.Equal(t, checkUpstreamUnreachableBackoffMax, unreachableRecoveryBackoff(100)) +} + func Test_prog_dnsWatchdogEnabled(t *testing.T) { p := &prog{cfg: &ctrld.Config{}} diff --git a/cmd/cli/upstream_monitor.go b/cmd/cli/upstream_monitor.go index 6e19e38..3eeb24b 100644 --- a/cmd/cli/upstream_monitor.go +++ b/cmd/cli/upstream_monitor.go @@ -12,8 +12,27 @@ const ( maxFailureRequest = 50 // checkUpstreamBackoffSleep is the time interval between each upstream checks. checkUpstreamBackoffSleep = 2 * time.Second + // checkUpstreamUnreachableBackoffMax caps the recovery retry interval for an + // endpoint that keeps failing with a network-unreachable error. It bounds + // the backoff so an unroutable endpoint is still re-probed periodically and + // recovers once the route returns. + checkUpstreamUnreachableBackoffMax = 60 * time.Second ) +// unreachableRecoveryBackoff returns the retry interval for the given streak of +// consecutive network-unreachable failures. It starts at checkUpstreamBackoffSleep +// and doubles each attempt, capped at checkUpstreamUnreachableBackoffMax. +func unreachableRecoveryBackoff(streak int) time.Duration { + d := checkUpstreamBackoffSleep + for i := 1; i < streak; i++ { + d *= 2 + if d >= checkUpstreamUnreachableBackoffMax { + return checkUpstreamUnreachableBackoffMax + } + } + return d +} + // upstreamMonitor performs monitoring upstreams health. type upstreamMonitor struct { cfg *ctrld.Config diff --git a/internal/net/net.go b/internal/net/net.go index f4b5586..677e90d 100644 --- a/internal/net/net.go +++ b/internal/net/net.go @@ -157,6 +157,101 @@ type parallelDialerResult struct { err error } +const ( + // unreachableBackoffBase is the initial suppression window applied to a + // dial address after it returns a network-unreachable error (e.g. + // "connect: no route to host"). The window grows exponentially up to + // unreachableBackoffMax on repeated failures, and is cleared as soon as + // the address dials successfully. + unreachableBackoffBase = 5 * time.Second + // unreachableBackoffMax caps the suppression window so an address is + // always re-probed within a bounded interval, preserving recovery when + // the route comes back. + unreachableBackoffMax = 60 * time.Second +) + +// Windows winsock codes for the unreachable errnos. A failing connect on +// Windows surfaces these raw WSA codes (WSAENETUNREACH/WSAEHOSTUNREACH), whereas +// syscall.ENETUNREACH/EHOSTUNREACH are Go's portable "invented" values +// (APPLICATION_ERROR + iota) that never equal them. Matching these explicitly is +// therefore required for the classifier to detect unreachable errors on Windows; +// errors.Is against the syscall.* constants alone would not. +// +// https://learn.microsoft.com/en-us/windows/win32/winsock/windows-sockets-error-codes-2 +var ( + windowsENETUNREACH = syscall.Errno(10051) + windowsEHOSTUNREACH = syscall.Errno(10065) +) + +// IsUnreachable reports whether err indicates the destination network or host +// has no route (ENETUNREACH/EHOSTUNREACH). These are the errors produced when +// an endpoint's address family is available locally but unroutable, e.g. an +// IPv6 DoH endpoint while the host has IPv6 but no route to it. +func IsUnreachable(err error) bool { + if err == nil { + return false + } + var opErr *net.OpError + if errors.As(err, &opErr) { + return errors.Is(opErr.Err, syscall.ENETUNREACH) || + errors.Is(opErr.Err, syscall.EHOSTUNREACH) || + errors.Is(opErr.Err, windowsENETUNREACH) || + errors.Is(opErr.Err, windowsEHOSTUNREACH) + } + return false +} + +type unreachableEntry struct { + until time.Time + backoff time.Duration +} + +// unreachableTracker records dial addresses that recently failed with a +// network-unreachable error so ParallelDialer can temporarily stop hammering +// them. This prevents an unroutable endpoint from generating a sustained dial +// /health-check storm, while still re-probing each address once its bounded +// backoff window expires so genuine recovery is never permanently blocked. +type unreachableTracker struct { + mu sync.Mutex + entries map[string]unreachableEntry +} + +var unreachable = &unreachableTracker{entries: make(map[string]unreachableEntry)} + +// suppressed reports whether addr is currently within its unreachable backoff +// window and should be skipped. +func (t *unreachableTracker) suppressed(addr string, now time.Time) bool { + t.mu.Lock() + defer t.mu.Unlock() + e, ok := t.entries[addr] + return ok && now.Before(e.until) +} + +// markUnreachable extends the suppression window for addr using a bounded +// exponential backoff. +func (t *unreachableTracker) markUnreachable(addr string, now time.Time) { + t.mu.Lock() + defer t.mu.Unlock() + e := t.entries[addr] + if e.backoff == 0 { + e.backoff = unreachableBackoffBase + } else { + e.backoff *= 2 + if e.backoff > unreachableBackoffMax { + e.backoff = unreachableBackoffMax + } + } + e.until = now.Add(e.backoff) + t.entries[addr] = e +} + +// markReachable clears any suppression for addr after a successful dial. +func (t *unreachableTracker) markReachable(addr string) { + t.mu.Lock() + defer t.mu.Unlock() + delete(t.entries, addr) +} + type ParallelDialer struct { net.Dialer } @@ -165,26 +260,63 @@ func (d *ParallelDialer) DialContext(ctx context.Context, network string, addrs if len(addrs) == 0 { return nil, errors.New("empty addresses") } + + // Skip addresses that recently returned a network-unreachable error so an + // unroutable endpoint (e.g. an IPv6 DoH address while the host has IPv6 + // but no route to it) does not generate a sustained dial storm. Suppression + // is bounded: once an address's backoff window expires it is re-probed, so + // genuine recovery is preserved. + now := time.Now() + live := make([]string, 0, len(addrs)) + var suppressed int + for _, addr := range addrs { + if unreachable.suppressed(addr, now) { + suppressed++ + continue + } + live = append(live, addr) + } + if len(live) == 0 { + // Every candidate is within its unreachable backoff window. Fail fast + // and quietly instead of re-dialing known-unroutable addresses; the + // windows expire and re-probe shortly, so recovery still happens. + logger.Debug().Msgf("skipping %d unreachable address(es), all in backoff", suppressed) + // TODO: the errno here is hardcoded to EHOSTUNREACH, but the actual + // failure that triggered suppression may have been ENETUNREACH. This is + // harmless today (IsUnreachable treats both the same and nothing else + // inspects the errno), but if these errors are ever recorded/reported we + // should retain the real error in unreachableEntry and surface it here. + return nil, &net.OpError{Op: "dial", Net: network, Err: syscall.EHOSTUNREACH} + } + if suppressed > 0 { + logger.Debug().Msgf("skipping %d unreachable address(es) in backoff", suppressed) + } + ctx, cancel := context.WithCancel(ctx) defer cancel() done := make(chan struct{}) defer close(done) - ch := make(chan *parallelDialerResult, len(addrs)) + ch := make(chan *parallelDialerResult, len(live)) var wg sync.WaitGroup - wg.Add(len(addrs)) + wg.Add(len(live)) go func() { wg.Wait() close(ch) }() - for _, addr := range addrs { + for _, addr := range live { go func(addr string) { defer wg.Done() logger.Debug().Msgf("dialing to %s", addr) conn, err := d.Dialer.DialContext(ctx, network, addr) if err != nil { logger.Debug().Msgf("failed to dial %s: %v", addr, err) + if IsUnreachable(err) { + unreachable.markUnreachable(addr, time.Now()) + } + } else { + unreachable.markReachable(addr) } select { case ch <- ¶llelDialerResult{conn: conn, err: err}: diff --git a/internal/net/net_test.go b/internal/net/net_test.go index 7df3e09..38187c5 100644 --- a/internal/net/net_test.go +++ b/internal/net/net_test.go @@ -2,10 +2,77 @@ package net import ( "context" + "net" + "syscall" "testing" "time" ) +func TestIsUnreachable(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"enetunreach", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ENETUNREACH}, true}, + {"ehostunreach", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.EHOSTUNREACH}, true}, + {"windows enetunreach", &net.OpError{Op: "dial", Net: "tcp", Err: windowsENETUNREACH}, true}, + {"windows ehostunreach", &net.OpError{Op: "dial", Net: "tcp", Err: windowsEHOSTUNREACH}, true}, + {"connection refused", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}, false}, + {"not an opError", syscall.ENETUNREACH, false}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + if got := IsUnreachable(tc.err); got != tc.want { + t.Errorf("IsUnreachable(%v) = %v, want %v", tc.err, got, tc.want) + } + }) + } +} + +func TestUnreachableTracker(t *testing.T) { + tr := &unreachableTracker{entries: make(map[string]unreachableEntry)} + const addr = "[2606:1a40::22]:443" + now := time.Unix(0, 0) + + // Not suppressed before any failure. + if tr.suppressed(addr, now) { + t.Fatal("addr suppressed before any failure") + } + + // First failure suppresses for the base window. + tr.markUnreachable(addr, now) + if !tr.suppressed(addr, now.Add(unreachableBackoffBase-time.Millisecond)) { + t.Fatal("addr not suppressed within base backoff window") + } + if tr.suppressed(addr, now.Add(unreachableBackoffBase)) { + t.Fatal("addr still suppressed at end of base backoff window") + } + + // Backoff grows exponentially and is capped at the max. + tr.markUnreachable(addr, now) + if got := tr.entries[addr].backoff; got != 2*unreachableBackoffBase { + t.Fatalf("backoff after second failure = %v, want %v", got, 2*unreachableBackoffBase) + } + for i := 0; i < 10; i++ { + tr.markUnreachable(addr, now) + } + if got := tr.entries[addr].backoff; got != unreachableBackoffMax { + t.Fatalf("backoff not capped: got %v, want %v", got, unreachableBackoffMax) + } + + // A successful dial clears suppression entirely. + tr.markReachable(addr) + if tr.suppressed(addr, now) { + t.Fatal("addr still suppressed after markReachable") + } + if _, ok := tr.entries[addr]; ok { + t.Fatal("entry not removed after markReachable") + } +} + func TestProbeStackTimeout(t *testing.T) { done := make(chan struct{}) started := make(chan struct{})