mirror of
https://github.com/Control-D-Inc/ctrld.git
synced 2026-07-16 13:17:19 +02:00
fix: back off unroutable IPv6 DoH upstream health-check spam
When IPv6 is available locally but the selected IPv6 DoH endpoint is unroutable (e.g. dialing [2606:1a40::22]:443 returns "no route to host" while IPv4 stays usable), ctrld re-bootstrapped and re-dialed the endpoint every ~2s. A weekend soak produced ~46.7k "no route to host" lines, with the dial/health-check loop dominating the log during bad windows. Add bounded backoff/suppression for network-unreachable endpoints at two levels: - ParallelDialer (internal/net): track dial addresses that fail with ENETUNREACH/EHOSTUNREACH and skip them for an exponentially growing, bounded window (5s -> 60s). A successful dial clears the entry immediately, so recovery is preserved when the route returns. When every candidate is suppressed the dial fails fast and quietly instead of hammering known-unroutable addresses. - Upstream recovery loop (cmd/cli): demote unreachable check failures to debug and back off the retry cadence (2s -> 60s) for an unreachable streak; any other failure resets to the base cadence. The new IsUnreachable classifier lives in internal/net and is reused by cmd/cli's errNetworkError, so the unreachable-errno matching has a single definition. Note the explicit winsock constants (10051/10065) are required on Windows: syscall.ENETUNREACH/EHOSTUNREACH are Go's portable "invented" values and never equal the raw WSA codes a failing connect surfaces. Suppression and backoff are always bounded, so IPv6 is never disabled until restart and recovers on its own once the route is back. Split-stack selection and the #549 macOS intercept recovery work are untouched. Adds unit tests for the classifier, the dialer's suppression tracker, and the recovery backoff schedule.
This commit is contained in:
@@ -2,10 +2,77 @@ package net
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIsUnreachable(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"nil", nil, false},
|
||||
{"enetunreach", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ENETUNREACH}, true},
|
||||
{"ehostunreach", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.EHOSTUNREACH}, true},
|
||||
{"windows enetunreach", &net.OpError{Op: "dial", Net: "tcp", Err: windowsENETUNREACH}, true},
|
||||
{"windows ehostunreach", &net.OpError{Op: "dial", Net: "tcp", Err: windowsEHOSTUNREACH}, true},
|
||||
{"connection refused", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}, false},
|
||||
{"not an opError", syscall.ENETUNREACH, false},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := IsUnreachable(tc.err); got != tc.want {
|
||||
t.Errorf("IsUnreachable(%v) = %v, want %v", tc.err, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnreachableTracker(t *testing.T) {
|
||||
tr := &unreachableTracker{entries: make(map[string]unreachableEntry)}
|
||||
const addr = "[2606:1a40::22]:443"
|
||||
now := time.Unix(0, 0)
|
||||
|
||||
// Not suppressed before any failure.
|
||||
if tr.suppressed(addr, now) {
|
||||
t.Fatal("addr suppressed before any failure")
|
||||
}
|
||||
|
||||
// First failure suppresses for the base window.
|
||||
tr.markUnreachable(addr, now)
|
||||
if !tr.suppressed(addr, now.Add(unreachableBackoffBase-time.Millisecond)) {
|
||||
t.Fatal("addr not suppressed within base backoff window")
|
||||
}
|
||||
if tr.suppressed(addr, now.Add(unreachableBackoffBase)) {
|
||||
t.Fatal("addr still suppressed at end of base backoff window")
|
||||
}
|
||||
|
||||
// Backoff grows exponentially and is capped at the max.
|
||||
tr.markUnreachable(addr, now)
|
||||
if got := tr.entries[addr].backoff; got != 2*unreachableBackoffBase {
|
||||
t.Fatalf("backoff after second failure = %v, want %v", got, 2*unreachableBackoffBase)
|
||||
}
|
||||
for i := 0; i < 10; i++ {
|
||||
tr.markUnreachable(addr, now)
|
||||
}
|
||||
if got := tr.entries[addr].backoff; got != unreachableBackoffMax {
|
||||
t.Fatalf("backoff not capped: got %v, want %v", got, unreachableBackoffMax)
|
||||
}
|
||||
|
||||
// A successful dial clears suppression entirely.
|
||||
tr.markReachable(addr)
|
||||
if tr.suppressed(addr, now) {
|
||||
t.Fatal("addr still suppressed after markReachable")
|
||||
}
|
||||
if _, ok := tr.entries[addr]; ok {
|
||||
t.Fatal("entry not removed after markReachable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeStackTimeout(t *testing.T) {
|
||||
done := make(chan struct{})
|
||||
started := make(chan struct{})
|
||||
|
||||
Reference in New Issue
Block a user