mirror of
https://github.com/Control-D-Inc/ctrld.git
synced 2026-07-16 13:17:19 +02:00
fix: back off unroutable IPv6 DoH upstream health-check spam
When IPv6 is available locally but the selected IPv6 DoH endpoint is unroutable (e.g. dialing [2606:1a40::22]:443 returns "no route to host" while IPv4 stays usable), ctrld re-bootstrapped and re-dialed the endpoint every ~2s. A weekend soak produced ~46.7k "no route to host" lines, with the dial/health-check loop dominating the log during bad windows. Add bounded backoff/suppression for network-unreachable endpoints at two levels: - ParallelDialer (internal/net): track dial addresses that fail with ENETUNREACH/EHOSTUNREACH and skip them for an exponentially growing, bounded window (5s -> 60s). A successful dial clears the entry immediately, so recovery is preserved when the route returns. When every candidate is suppressed the dial fails fast and quietly instead of hammering known-unroutable addresses. - Upstream recovery loop (cmd/cli): demote unreachable check failures to debug and back off the retry cadence (2s -> 60s) for an unreachable streak; any other failure resets to the base cadence. The new IsUnreachable classifier lives in internal/net and is reused by cmd/cli's errNetworkError, so the unreachable-errno matching has a single definition. Note the explicit winsock constants (10051/10065) are required on Windows: syscall.ENETUNREACH/EHOSTUNREACH are Go's portable "invented" values and never equal the raw WSA codes a failing connect surfaces. Suppression and backoff are always bounded, so IPv6 is never disabled until restart and recovers on its own once the route is back. Split-stack selection and the #549 macOS intercept recovery work are untouched. Adds unit tests for the classifier, the dialer's suppression tracker, and the recovery backoff schedule.
This commit is contained in:
+135
-3
@@ -157,6 +157,101 @@ type parallelDialerResult struct {
|
||||
err error
|
||||
}
|
||||
|
||||
const (
|
||||
// unreachableBackoffBase is the initial suppression window applied to a
|
||||
// dial address after it returns a network-unreachable error (e.g.
|
||||
// "connect: no route to host"). The window grows exponentially up to
|
||||
// unreachableBackoffMax on repeated failures, and is cleared as soon as
|
||||
// the address dials successfully.
|
||||
unreachableBackoffBase = 5 * time.Second
|
||||
// unreachableBackoffMax caps the suppression window so an address is
|
||||
// always re-probed within a bounded interval, preserving recovery when
|
||||
// the route comes back.
|
||||
unreachableBackoffMax = 60 * time.Second
|
||||
)
|
||||
|
||||
// Windows winsock codes for the unreachable errnos. A failing connect on
|
||||
// Windows surfaces these raw WSA codes (WSAENETUNREACH/WSAEHOSTUNREACH), whereas
|
||||
// syscall.ENETUNREACH/EHOSTUNREACH are Go's portable "invented" values
|
||||
// (APPLICATION_ERROR + iota) that never equal them. Matching these explicitly is
|
||||
// therefore required for the classifier to detect unreachable errors on Windows;
|
||||
// errors.Is against the syscall.* constants alone would not.
|
||||
//
|
||||
// https://learn.microsoft.com/en-us/windows/win32/winsock/windows-sockets-error-codes-2
|
||||
var (
|
||||
windowsENETUNREACH = syscall.Errno(10051)
|
||||
windowsEHOSTUNREACH = syscall.Errno(10065)
|
||||
)
|
||||
|
||||
// IsUnreachable reports whether err indicates the destination network or host
|
||||
// has no route (ENETUNREACH/EHOSTUNREACH). These are the errors produced when
|
||||
// an endpoint's address family is available locally but unroutable, e.g. an
|
||||
// IPv6 DoH endpoint while the host has IPv6 but no route to it.
|
||||
func IsUnreachable(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
var opErr *net.OpError
|
||||
if errors.As(err, &opErr) {
|
||||
return errors.Is(opErr.Err, syscall.ENETUNREACH) ||
|
||||
errors.Is(opErr.Err, syscall.EHOSTUNREACH) ||
|
||||
errors.Is(opErr.Err, windowsENETUNREACH) ||
|
||||
errors.Is(opErr.Err, windowsEHOSTUNREACH)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type unreachableEntry struct {
|
||||
until time.Time
|
||||
backoff time.Duration
|
||||
}
|
||||
|
||||
// unreachableTracker records dial addresses that recently failed with a
|
||||
// network-unreachable error so ParallelDialer can temporarily stop hammering
|
||||
// them. This prevents an unroutable endpoint from generating a sustained dial
|
||||
// /health-check storm, while still re-probing each address once its bounded
|
||||
// backoff window expires so genuine recovery is never permanently blocked.
|
||||
type unreachableTracker struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]unreachableEntry
|
||||
}
|
||||
|
||||
var unreachable = &unreachableTracker{entries: make(map[string]unreachableEntry)}
|
||||
|
||||
// suppressed reports whether addr is currently within its unreachable backoff
|
||||
// window and should be skipped.
|
||||
func (t *unreachableTracker) suppressed(addr string, now time.Time) bool {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
e, ok := t.entries[addr]
|
||||
return ok && now.Before(e.until)
|
||||
}
|
||||
|
||||
// markUnreachable extends the suppression window for addr using a bounded
|
||||
// exponential backoff.
|
||||
func (t *unreachableTracker) markUnreachable(addr string, now time.Time) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
e := t.entries[addr]
|
||||
if e.backoff == 0 {
|
||||
e.backoff = unreachableBackoffBase
|
||||
} else {
|
||||
e.backoff *= 2
|
||||
if e.backoff > unreachableBackoffMax {
|
||||
e.backoff = unreachableBackoffMax
|
||||
}
|
||||
}
|
||||
e.until = now.Add(e.backoff)
|
||||
t.entries[addr] = e
|
||||
}
|
||||
|
||||
// markReachable clears any suppression for addr after a successful dial.
|
||||
func (t *unreachableTracker) markReachable(addr string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
delete(t.entries, addr)
|
||||
}
|
||||
|
||||
type ParallelDialer struct {
|
||||
net.Dialer
|
||||
}
|
||||
@@ -165,26 +260,63 @@ func (d *ParallelDialer) DialContext(ctx context.Context, network string, addrs
|
||||
if len(addrs) == 0 {
|
||||
return nil, errors.New("empty addresses")
|
||||
}
|
||||
|
||||
// Skip addresses that recently returned a network-unreachable error so an
|
||||
// unroutable endpoint (e.g. an IPv6 DoH address while the host has IPv6
|
||||
// but no route to it) does not generate a sustained dial storm. Suppression
|
||||
// is bounded: once an address's backoff window expires it is re-probed, so
|
||||
// genuine recovery is preserved.
|
||||
now := time.Now()
|
||||
live := make([]string, 0, len(addrs))
|
||||
var suppressed int
|
||||
for _, addr := range addrs {
|
||||
if unreachable.suppressed(addr, now) {
|
||||
suppressed++
|
||||
continue
|
||||
}
|
||||
live = append(live, addr)
|
||||
}
|
||||
if len(live) == 0 {
|
||||
// Every candidate is within its unreachable backoff window. Fail fast
|
||||
// and quietly instead of re-dialing known-unroutable addresses; the
|
||||
// windows expire and re-probe shortly, so recovery still happens.
|
||||
logger.Debug().Msgf("skipping %d unreachable address(es), all in backoff", suppressed)
|
||||
// TODO: the errno here is hardcoded to EHOSTUNREACH, but the actual
|
||||
// failure that triggered suppression may have been ENETUNREACH. This is
|
||||
// harmless today (IsUnreachable treats both the same and nothing else
|
||||
// inspects the errno), but if these errors are ever recorded/reported we
|
||||
// should retain the real error in unreachableEntry and surface it here.
|
||||
return nil, &net.OpError{Op: "dial", Net: network, Err: syscall.EHOSTUNREACH}
|
||||
}
|
||||
if suppressed > 0 {
|
||||
logger.Debug().Msgf("skipping %d unreachable address(es) in backoff", suppressed)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
ch := make(chan *parallelDialerResult, len(addrs))
|
||||
ch := make(chan *parallelDialerResult, len(live))
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(len(addrs))
|
||||
wg.Add(len(live))
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(ch)
|
||||
}()
|
||||
|
||||
for _, addr := range addrs {
|
||||
for _, addr := range live {
|
||||
go func(addr string) {
|
||||
defer wg.Done()
|
||||
logger.Debug().Msgf("dialing to %s", addr)
|
||||
conn, err := d.Dialer.DialContext(ctx, network, addr)
|
||||
if err != nil {
|
||||
logger.Debug().Msgf("failed to dial %s: %v", addr, err)
|
||||
if IsUnreachable(err) {
|
||||
unreachable.markUnreachable(addr, time.Now())
|
||||
}
|
||||
} else {
|
||||
unreachable.markReachable(addr)
|
||||
}
|
||||
select {
|
||||
case ch <- ¶llelDialerResult{conn: conn, err: err}:
|
||||
|
||||
Reference in New Issue
Block a user