fix: recover upstreams from a partial bootstrap lookup

A bootstrap lookup that returned only one address family, such as an
AAAA-only answer for a split-stack Control D upstream, was accepted as
is, and re-bootstrap rebuilt the transports from the same lists, so A
queries failed with "empty addresses" until a restart.

Bootstrap now checks the result against what the ip_stack needs and
fills a missing family from the Control D direct-IP table and bootstrap
server. If it is still missing, the next re-bootstrap looks it up again,
and a split transport borrows the other family meanwhile.

After the DoH direct-IP fallback fires, DoH and DoH3 also dial the
Control D direct IPs for the ip_stack's families when the bootstrap
addresses are empty or fail, keeping the domain for TLS. A working list
always wins, so nothing is pinned. DoT always verifies the server as
the upstream domain.

config.go

@@ -307,6 +307,13 @@ type UpstreamConfig struct {
 	u                  *url.URL
 	fallbackOnce       sync.Once
 	uid                string
+
+	// bootstrapIncomplete is set while the bootstrap addresses lack a family
+	// the ip_stack needs; the next re-bootstrap looks them up again.
+	bootstrapIncomplete atomic.Bool
+	// directFallback is set once FallbackToDirectIP has fired: DoH and DoH3
+	// then also try the Control D direct IPs when the bootstrap addresses fail.
+	directFallback atomic.Bool
 }
 
 // ListenerConfig specifies the networks configuration that ctrld will run on.
@@ -461,6 +468,10 @@ func (uc *UpstreamConfig) UID() string {
 	return uc.uid
 }
 
+// bootstrapLookupFn resolves domain for bootstrapping. It is a variable so
+// tests control the lookup result.
+var bootstrapLookupFn = lookupIP
+
 // SetupBootstrapIP manually find all available IPs of the upstream.
 // The first usable IP will be used as bootstrap IP of the upstream.
 // The upstream domain will be looked up using following orders:
@@ -469,49 +480,131 @@ func (uc *UpstreamConfig) UID() string {
 // - Direct IPs table for ControlD upstreams.
 // - ControlD Bootstrap DNS 76.76.2.22
 //
-// The setup process will block until there's usable IPs found.
+// The later sources fill in what the earlier ones could not: the whole list,
+// and also an address family the ip_stack needs that a lookup left out, such
+// as an AAAA-only answer for a split-stack upstream, whose A queries need IPv4.
+//
+// The setup process will block until there's usable IPs found. A result that
+// still lacks a needed family is used as it is, and the next re-bootstrap
+// looks the missing family up again.
 func (uc *UpstreamConfig) SetupBootstrapIP() {
 	b := backoff.NewBackoff("setupBootstrapIP", func(format string, args ...any) {}, 10*time.Second)
-	isControlD := uc.IsControlD()
 	nss := initDefaultOsResolver()
 	for {
-		uc.bootstrapIPs = lookupIP(uc.Domain, uc.Timeout, nss)
-		// For ControlD upstream, the bootstrap IPs could not be RFC 1918 addresses,
-		// filtering them out here to prevent weird behavior.
-		if isControlD {
-			n := 0
-			for _, ip := range uc.bootstrapIPs {
-				netIP := net.ParseIP(ip)
-				if netIP != nil && !netIP.IsPrivate() {
-					uc.bootstrapIPs[n] = ip
-					n++
-				}
-			}
-			uc.bootstrapIPs = uc.bootstrapIPs[:n]
-			if len(uc.bootstrapIPs) == 0 {
-				uc.bootstrapIPs = bootstrapIPsFromControlDDomain(uc.Domain)
-				ProxyLogger.Load().Warn().Msgf("no record found for %q, lookup from direct IP table", uc.Domain)
-			}
-		}
-		if len(uc.bootstrapIPs) == 0 {
-			ProxyLogger.Load().Warn().Msgf("no record found for %q, using bootstrap server: %s", uc.Domain, PremiumDNSBoostrapIP)
-			uc.bootstrapIPs = lookupIP(uc.Domain, uc.Timeout, []string{net.JoinHostPort(PremiumDNSBoostrapIP, "53")})
-
-		}
-		if len(uc.bootstrapIPs) > 0 {
+		if ips := uc.lookupBootstrapIPs(nss); len(ips) > 0 {
+			uc.setBootstrapIPs(ips)
 			break
 		}
 		ProxyLogger.Load().Warn().Msg("could not resolve bootstrap IPs, retrying...")
 		b.BackOff(context.Background(), errors.New("no bootstrap IPs"))
 	}
-	for _, ip := range uc.bootstrapIPs {
-		if ctrldnet.IsIPv6(ip) {
-			uc.bootstrapIPs6 = append(uc.bootstrapIPs6, ip)
-		} else {
-			uc.bootstrapIPs4 = append(uc.bootstrapIPs4, ip)
+	ProxyLogger.Load().Debug().Msgf("bootstrap IPs: %v", uc.bootstrapIPs)
+}
+
+// lookupBootstrapIPs runs one round of the bootstrap lookup through nss and the
+// fallback sources, and returns what it found, which may be incomplete for the
+// ip_stack or empty.
+func (uc *UpstreamConfig) lookupBootstrapIPs(nss []string) []string {
+	isControlD := uc.IsControlD()
+	ips := bootstrapLookupFn(uc.Domain, uc.Timeout, nss)
+	// For ControlD upstream, the bootstrap IPs could not be RFC 1918 addresses,
+	// filtering them out here to prevent weird behavior.
+	if isControlD {
+		n := 0
+		for _, ip := range ips {
+			netIP := net.ParseIP(ip)
+			if netIP != nil && !netIP.IsPrivate() {
+				ips[n] = ip
+				n++
+			}
+		}
+		ips = ips[:n]
+		if !uc.bootstrapComplete(ips) {
+			ProxyLogger.Load().Warn().Msgf("incomplete records for %q (%v), filling in from direct IP table", uc.Domain, ips)
+			ips = mergeMissingFamilies(ips, bootstrapIPsFromControlDDomain(uc.Domain))
 		}
 	}
-	ProxyLogger.Load().Debug().Msgf("bootstrap IPs: %v", uc.bootstrapIPs)
+	if !uc.bootstrapComplete(ips) {
+		ProxyLogger.Load().Warn().Msgf("incomplete records for %q (%v), using bootstrap server: %s", uc.Domain, ips, PremiumDNSBoostrapIP)
+		ips = mergeMissingFamilies(ips, bootstrapLookupFn(uc.Domain, uc.Timeout, []string{net.JoinHostPort(PremiumDNSBoostrapIP, "53")}))
+	}
+	return ips
+}
+
+// setBootstrapIPs replaces the bootstrap addresses, and records whether they
+// cover the families the ip_stack needs.
+func (uc *UpstreamConfig) setBootstrapIPs(ips []string) {
+	ips4, ips6 := splitAddressFamilies(ips)
+	uc.bootstrapIPs, uc.bootstrapIPs4, uc.bootstrapIPs6 = ips, ips4, ips6
+	complete := uc.bootstrapComplete(ips)
+	uc.bootstrapIncomplete.Store(!complete)
+	if !complete {
+		ProxyLogger.Load().Warn().Msgf("bootstrap IPs for %q lack an address family needed by ip_stack %q: %v; will look up again on re-bootstrap", uc.Domain, uc.IPStack, ips)
+	}
+}
+
+// bootstrapComplete reports whether ips cover the address families the
+// upstream's ip_stack needs: IPv4 for v4, IPv6 for v6, either for both, and
+// for split IPv4 for A queries plus IPv6 for the others when the host has IPv6.
+// An IPv4-only host never needs IPv6.
+func (uc *UpstreamConfig) bootstrapComplete(ips []string) bool {
+	ips4, ips6 := splitAddressFamilies(ips)
+	switch uc.IPStack {
+	case IpStackV4:
+		return len(ips4) > 0
+	case IpStackV6:
+		return len(ips6) > 0
+	case IpStackSplit:
+		return len(ips4) > 0 && (len(ips6) > 0 || !HasIPv6())
+	}
+	return len(ips) > 0
+}
+
+// splitAddressFamilies returns the IPv4 and the IPv6 addresses of ips.
+func splitAddressFamilies(ips []string) (ips4, ips6 []string) {
+	for _, ip := range ips {
+		if ctrldnet.IsIPv6(ip) {
+			ips6 = append(ips6, ip)
+		} else {
+			ips4 = append(ips4, ip)
+		}
+	}
+	return ips4, ips6
+}
+
+// mergeMissingFamilies adds to ips the addresses of extra whose family ips
+// does not have at all. A family ips already has is kept as it is.
+func mergeMissingFamilies(ips, extra []string) []string {
+	ips4, ips6 := splitAddressFamilies(ips)
+	extra4, extra6 := splitAddressFamilies(extra)
+	merged := append([]string{}, ips...)
+	if len(ips4) == 0 {
+		merged = append(merged, extra4...)
+	}
+	if len(ips6) == 0 {
+		merged = append(merged, extra6...)
+	}
+	return merged
+}
+
+// refreshIncompleteBootstrap looks the bootstrap addresses up again, once, when
+// they lack a family the ip_stack needs. Rebuilding the transports from the
+// same incomplete lists would never recover; this picks up the missing family
+// once DNS is reachable again. Families it still cannot find keep their current
+// addresses.
+func (uc *UpstreamConfig) refreshIncompleteBootstrap() {
+	if uc.BootstrapIP != "" || !uc.bootstrapIncomplete.Load() {
+		return
+	}
+	ips := uc.lookupBootstrapIPs(initDefaultOsResolver())
+	if !uc.bootstrapComplete(ips) {
+		ips = mergeMissingFamilies(ips, uc.bootstrapIPs)
+	}
+	if len(ips) == 0 {
+		return
+	}
+	uc.setBootstrapIPs(ips)
+	ProxyLogger.Load().Debug().Msgf("refreshed bootstrap IPs for %q: %v", uc.Domain, ips)
 }
 
 // ReBootstrap re-setup the bootstrap IP and the transport.
@@ -541,6 +634,7 @@ func (uc *UpstreamConfig) ForceReBootstrap() {
 		return
 	}
 	ProxyLogger.Load().Debug().Msgf("force re-bootstrapping upstream transport for %v", uc)
+	uc.refreshIncompleteBootstrap()
 	uc.SetupTransport()
 	// Clear any pending lazy re-bootstrap flag so ensureSetupTransport()
 	// doesn't redundantly recreate the transport we just built.
@@ -644,15 +738,26 @@ func (uc *UpstreamConfig) SetupTransport() {
 	uc.doqConnPool = uc.newDOQConnPool(ips)
 	uc.dotClientPool = uc.newDOTClientPool(ips)
 	if uc.IPStack == IpStackSplit {
-		uc.transport4 = uc.newDOHTransport(uc.bootstrapIPs4)
-		uc.http3RoundTripper4 = uc.newDOH3Transport(uc.bootstrapIPs4)
-		uc.doqConnPool4 = uc.newDOQConnPool(uc.bootstrapIPs4)
-		uc.dotClientPool4 = uc.newDOTClientPool(uc.bootstrapIPs4)
+		// Split picks the transport by query type, not a hard family limit:
+		// while a family is missing, its queries borrow the other family's
+		// addresses rather than fail on an empty transport until the
+		// re-bootstrap finds it (refreshIncompleteBootstrap).
+		ips4, ips6 := uc.bootstrapIPs4, uc.bootstrapIPs6
+		if len(ips4) == 0 {
+			ips4 = ips6
+		}
+		if len(ips6) == 0 {
+			ips6 = ips4
+		}
+		uc.transport4 = uc.newDOHTransport(ips4)
+		uc.http3RoundTripper4 = uc.newDOH3Transport(ips4)
+		uc.doqConnPool4 = uc.newDOQConnPool(ips4)
+		uc.dotClientPool4 = uc.newDOTClientPool(ips4)
 		if HasIPv6() {
-			uc.transport6 = uc.newDOHTransport(uc.bootstrapIPs6)
-			uc.http3RoundTripper6 = uc.newDOH3Transport(uc.bootstrapIPs6)
-			uc.doqConnPool6 = uc.newDOQConnPool(uc.bootstrapIPs6)
-			uc.dotClientPool6 = uc.newDOTClientPool(uc.bootstrapIPs6)
+			uc.transport6 = uc.newDOHTransport(ips6)
+			uc.http3RoundTripper6 = uc.newDOH3Transport(ips6)
+			uc.doqConnPool6 = uc.newDOQConnPool(ips6)
+			uc.dotClientPool6 = uc.newDOTClientPool(ips6)
 		} else {
 			uc.transport6 = uc.transport4
 			uc.http3RoundTripper6 = uc.http3RoundTripper4
@@ -667,6 +772,7 @@ func (uc *UpstreamConfig) ensureSetupTransport() {
 		uc.SetupTransport()
 	})
 	if uc.rebootstrap.CompareAndSwap(rebootstrapStarted, rebootstrapInProgress) {
+		uc.refreshIncompleteBootstrap()
 		uc.SetupTransport()
 		uc.rebootstrap.Store(rebootstrapNotStarted)
 	}
@@ -715,13 +821,17 @@ func (uc *UpstreamConfig) newDOHTransport(addrs []string) *http.Transport {
 		pd := &ctrldnet.ParallelDialer{}
 		pd.Timeout = dialerTimeout
 		pd.KeepAlive = dialerTimeout
-		dialAddrs := make([]string, len(addrs))
-		for i := range addrs {
-			dialAddrs[i] = net.JoinHostPort(addrs[i], port)
-		}
-		conn, err := pd.DialContext(ctx, network, dialAddrs, ProxyLogger.Load())
+		conn, err := pd.DialContext(ctx, network, joinHostPorts(addrs, port), ProxyLogger.Load())
 		if err != nil {
-			return nil, err
+			direct := uc.directFallbackIPs()
+			if len(direct) == 0 {
+				return nil, err
+			}
+			Log(ctx, ProxyLogger.Load().Debug().Err(err), "bootstrap addresses failed, dialing direct IPs %v", direct)
+			var directErr error
+			if conn, directErr = pd.DialContext(ctx, network, joinHostPorts(direct, port), ProxyLogger.Load()); directErr != nil {
+				return nil, errors.Join(err, directErr)
+			}
 		}
 		Log(ctx, ProxyLogger.Load().Debug(), "sending doh request to: %s", conn.RemoteAddr())
 		return conn, nil
@@ -920,7 +1030,42 @@ func (uc *UpstreamConfig) Context(ctx context.Context) (context.Context, context
 	return context.WithCancel(ctx)
 }
 
-// FallbackToDirectIP changes ControlD upstream endpoint to use direct IP instead of domain.
+// controlDDirectIPsFn returns the direct IPs of a Control D domain. It is a
+// variable so tests can point the fallback at local listeners.
+var controlDDirectIPsFn = bootstrapIPsFromControlDDomain
+
+// directFallbackIPs returns the Control D direct IPs that DoH and DoH3 try
+// when the bootstrap addresses fail, once FallbackToDirectIP has fired, or nil.
+// They are limited to the families the ip_stack dials: an IPv6-only upstream
+// must not be sent to an IPv4 literal it cannot reach.
+func (uc *UpstreamConfig) directFallbackIPs() []string {
+	if !uc.directFallback.Load() {
+		return nil
+	}
+	ips4, ips6 := splitAddressFamilies(controlDDirectIPsFn(uc.Domain))
+	switch uc.IPStack {
+	case IpStackV4:
+		return ips4
+	case IpStackV6:
+		return ips6
+	}
+	return append(ips4, ips6...)
+}
+
+// joinHostPorts returns ips joined with port.
+func joinHostPorts(ips []string, port string) []string {
+	addrs := make([]string, len(ips))
+	for i, ip := range ips {
+		addrs[i] = net.JoinHostPort(ip, port)
+	}
+	return addrs
+}
+
+// FallbackToDirectIP lets a ControlD DoH/DoH3 upstream also dial its direct
+// IPs, which DoH and DoH3 try only when the bootstrap addresses are empty or
+// all fail. The bootstrap list keeps precedence, so a working or refreshed
+// list is used again as soon as it can be, and nothing is pinned. The request
+// URL keeps the domain, so TLS still verifies the certificate for it.
 func (uc *UpstreamConfig) FallbackToDirectIP() bool {
 	if !uc.IsControlD() {
 		return false
@@ -931,17 +1076,12 @@ func (uc *UpstreamConfig) FallbackToDirectIP() bool {
 
 	done := false
 	uc.fallbackOnce.Do(func() {
-		var ip string
-		switch {
-		case dns.IsSubDomain(PremiumDnsDomain, uc.Domain):
-			ip = PremiumDNSBoostrapIP
-		case dns.IsSubDomain(FreeDnsDomain, uc.Domain):
-			ip = FreeDNSBoostrapIP
-		default:
+		ips := controlDDirectIPsFn(uc.Domain)
+		if len(ips) == 0 {
 			return
 		}
-		ProxyLogger.Load().Warn().Msgf("using direct IP for %q: %s", uc.Endpoint, ip)
-		uc.u.Host = ip
+		ProxyLogger.Load().Warn().Msgf("falling back to direct IPs for %q when its bootstrap addresses fail: %v", uc.Endpoint, ips)
+		uc.directFallback.Store(true)
 		done = true
 	})
 	return done

config_bootstrap_test.go

@@ -0,0 +1,334 @@
+package ctrld
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"net"
+	"net/http"
+	"slices"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/miekg/dns"
+	"github.com/quic-go/quic-go"
+	"github.com/quic-go/quic-go/http3"
+)
+
+// useBootstrapLookup replaces the bootstrap lookup. system answers lookups
+// through the system nameservers, controlD lookups through the Control D
+// bootstrap server; the test changes them through the returned pointers.
+func useBootstrapLookup(t *testing.T, system, controlD []string) (*[]string, *[]string) {
+	t.Helper()
+	sys, cd := append([]string{}, system...), append([]string{}, controlD...)
+	old := bootstrapLookupFn
+	bootstrapLookupFn = func(domain string, timeout int, nss []string) []string {
+		if slices.Equal(nss, []string{net.JoinHostPort(PremiumDNSBoostrapIP, "53")}) {
+			return append([]string{}, cd...)
+		}
+		return append([]string{}, sys...)
+	}
+	t.Cleanup(func() { bootstrapLookupFn = old })
+	return &sys, &cd
+}
+
+// useIPv6 pins whether the host has IPv6. HasIPv6 probes once per process, so
+// the probe runs first and the pinned value replaces its result.
+func useIPv6(t *testing.T, available bool) {
+	t.Helper()
+	old := HasIPv6()
+	SetIPv6Available(available)
+	t.Cleanup(func() { SetIPv6Available(old) })
+}
+
+func newBootstrapTestUpstream(typ, endpoint, ipStack string) *UpstreamConfig {
+	uc := &UpstreamConfig{Type: typ, Endpoint: endpoint, Timeout: 1000}
+	uc.Init()
+	uc.IPStack = ipStack
+	return uc
+}
+
+// Bootstrap results must cover the address families the ip_stack needs (#624).
+// A partial lookup is filled in from the Control D direct-IP table and the
+// Control D bootstrap server; what is still missing marks the upstream for a
+// new lookup instead of becoming a permanently empty transport. An IPv4-only
+// host never needs IPv6.
+func TestSetupBootstrapIPCompletesPerStack(t *testing.T) {
+	const (
+		cdEndpoint    = "https://dns.controld.com/abcdef"
+		otherEndpoint = "https://dns.example.com/dns-query"
+		v4, v6        = "192.0.2.10", "2001:db8::10"
+	)
+	for _, tc := range []struct {
+		name           string
+		endpoint       string
+		ipStack        string
+		ipv6           bool
+		system, cdBoot []string
+		want4, want6   []string
+		wantIncomplete bool
+	}{
+		{name: "Control D split, AAAA only", endpoint: cdEndpoint, ipStack: IpStackSplit, ipv6: true,
+			system: []string{"2606:1a40::99"}, want4: []string{PremiumDNSBoostrapIP}, want6: []string{"2606:1a40::99"}},
+		{name: "Control D split, A only", endpoint: cdEndpoint, ipStack: IpStackSplit, ipv6: true,
+			system: []string{"76.76.2.99"}, want4: []string{"76.76.2.99"}, want6: []string{PremiumDNSBoostrapIPv6}},
+		{name: "Control D split, A only, IPv4-only host", endpoint: cdEndpoint, ipStack: IpStackSplit, ipv6: false,
+			system: []string{"76.76.2.99"}, want4: []string{"76.76.2.99"}},
+		{name: "Control D split, complete", endpoint: cdEndpoint, ipStack: IpStackSplit, ipv6: true,
+			system: []string{"2606:1a40::99", "76.76.2.99"}, want4: []string{"76.76.2.99"}, want6: []string{"2606:1a40::99"}},
+		{name: "other v4, AAAA only", endpoint: otherEndpoint, ipStack: IpStackV4, ipv6: true,
+			system: []string{v6}, want6: []string{v6}, wantIncomplete: true},
+		{name: "other v6, A only", endpoint: otherEndpoint, ipStack: IpStackV6, ipv6: true,
+			system: []string{v4}, want4: []string{v4}, wantIncomplete: true},
+		{name: "other v4, complete", endpoint: otherEndpoint, ipStack: IpStackV4, ipv6: true,
+			system: []string{v4}, want4: []string{v4}},
+		{name: "other both, AAAA only", endpoint: otherEndpoint, ipStack: IpStackBoth, ipv6: true,
+			system: []string{v6}, want6: []string{v6}},
+		{name: "other split, AAAA only, bootstrap server has A", endpoint: otherEndpoint, ipStack: IpStackSplit, ipv6: true,
+			system: []string{v6}, cdBoot: []string{v4}, want4: []string{v4}, want6: []string{v6}},
+		{name: "other split, AAAA only, nowhere has A", endpoint: otherEndpoint, ipStack: IpStackSplit, ipv6: true,
+			system: []string{v6}, want6: []string{v6}, wantIncomplete: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			useIPv6(t, tc.ipv6)
+			useBootstrapLookup(t, tc.system, tc.cdBoot)
+			uc := newBootstrapTestUpstream(ResolverTypeDOH, tc.endpoint, tc.ipStack)
+			uc.SetupBootstrapIP()
+			if !slices.Equal(uc.bootstrapIPs4, tc.want4) || !slices.Equal(uc.bootstrapIPs6, tc.want6) {
+				t.Fatalf("bootstrap IPs = v4 %v, v6 %v; want v4 %v, v6 %v", uc.bootstrapIPs4, uc.bootstrapIPs6, tc.want4, tc.want6)
+			}
+			if got := uc.bootstrapIncomplete.Load(); got != tc.wantIncomplete {
+				t.Fatalf("bootstrapIncomplete = %v, want %v", got, tc.wantIncomplete)
+			}
+		})
+	}
+}
+
+// After a partial lookup, the transport recovers once the missing family can
+// be resolved again, without a restart: a re-bootstrap looks it up instead of
+// rebuilding the transport from the same incomplete lists. Until then a split
+// transport borrows the other family rather than staying empty.
+func TestReBootstrapRefreshesMissingFamily(t *testing.T) {
+	const v4, v6 = "192.0.2.10", "2001:db8::10"
+	for _, tc := range []struct {
+		name          string
+		ipStack       string
+		wantInterimA4 []string // addresses A queries dial before recovery
+	}{
+		{name: "v4", ipStack: IpStackV4, wantInterimA4: nil},
+		{name: "split", ipStack: IpStackSplit, wantInterimA4: []string{v6}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			useIPv6(t, true)
+			system, _ := useBootstrapLookup(t, []string{v6}, nil)
+			uc := newBootstrapTestUpstream(ResolverTypeDOT, "dot.example.com:853", tc.ipStack)
+			t.Cleanup(uc.CloseTransports)
+			uc.SetupBootstrapIP()
+			if !uc.bootstrapIncomplete.Load() {
+				t.Fatal("an AAAA-only result was accepted as complete")
+			}
+			if got := uc.dotTransport(dns.TypeA).addrs; !slices.Equal(got, tc.wantInterimA4) {
+				t.Fatalf("A-query addresses before recovery = %v, want %v", got, tc.wantInterimA4)
+			}
+
+			// DNS is still broken: a re-bootstrap keeps what it has.
+			uc.ReBootstrap()
+			if got := uc.dotTransport(dns.TypeA).addrs; !slices.Equal(got, tc.wantInterimA4) {
+				t.Fatalf("A-query addresses while DNS is down = %v, want %v", got, tc.wantInterimA4)
+			}
+
+			// DNS is back: the next re-bootstrap picks up the missing family.
+			*system = []string{v6, v4}
+			uc.ReBootstrap()
+			if got := uc.dotTransport(dns.TypeA).addrs; !slices.Equal(got, []string{v4}) {
+				t.Fatalf("A-query addresses after recovery = %v, want [%s]", got, v4)
+			}
+			if uc.bootstrapIncomplete.Load() {
+				t.Fatal("still incomplete after the missing family was found")
+			}
+		})
+	}
+}
+
+// A complete result is not looked up again on every re-bootstrap.
+func TestReBootstrapKeepsCompleteResult(t *testing.T) {
+	useIPv6(t, true)
+	lookups := 0
+	old := bootstrapLookupFn
+	bootstrapLookupFn = func(string, int, []string) []string {
+		lookups++
+		return []string{"192.0.2.10", "2001:db8::10"}
+	}
+	t.Cleanup(func() { bootstrapLookupFn = old })
+	uc := newBootstrapTestUpstream(ResolverTypeDOT, "dot.example.com:853", IpStackSplit)
+	t.Cleanup(uc.CloseTransports)
+	uc.SetupBootstrapIP()
+	before := lookups
+	uc.ReBootstrap()
+	_ = uc.dotTransport(dns.TypeA)
+	if lookups != before {
+		t.Fatalf("a complete result was looked up again: %d lookups", lookups-before)
+	}
+}
+
+// useDirectIPs replaces the Control D direct-IP table.
+func useDirectIPs(t *testing.T, ips ...string) {
+	t.Helper()
+	old := controlDDirectIPsFn
+	controlDDirectIPsFn = func(string) []string { return append([]string{}, ips...) }
+	t.Cleanup(func() { controlDDirectIPsFn = old })
+}
+
+// listenLocal listens on ip:port, or on a free port when port is "0", and
+// returns the port. It skips the test when ip is not available here.
+func listenLocal(t *testing.T, ip, port string) string {
+	t.Helper()
+	ln, err := net.Listen("tcp", net.JoinHostPort(ip, port))
+	if err != nil {
+		t.Skipf("cannot listen on %s: %v", ip, err)
+	}
+	t.Cleanup(func() { ln.Close() })
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			c.Close()
+		}
+	}()
+	return strconv.Itoa(ln.Addr().(*net.TCPAddr).Port)
+}
+
+func dialDOH(t *testing.T, transport *http.Transport, port string) (string, error) {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	conn, err := transport.DialContext(ctx, "tcp", net.JoinHostPort("dns.controld.com", port))
+	if err != nil {
+		return "", err
+	}
+	defer conn.Close()
+	host, _, _ := net.SplitHostPort(conn.RemoteAddr().String())
+	return host, nil
+}
+
+// The DoH direct-IP fallback adds the Control D direct IPs as a backup for
+// when the bootstrap addresses are empty or all fail. It never replaces a
+// working list, so it is not pinned until restart, and it only dials the
+// families the ip_stack dials.
+func TestFallbackToDirectIPIsABackup(t *testing.T) {
+	t.Run("v6 upstream keeps dialing its IPv6 list after the fallback", func(t *testing.T) {
+		port := listenLocal(t, "::1", "0")
+		useDirectIPs(t, "127.0.0.1")
+		uc := newBootstrapTestUpstream(ResolverTypeDOH, "https://dns.controld.com/abcdef", IpStackV6)
+		transport := uc.newDOHTransport([]string{"::1"})
+		t.Cleanup(transport.CloseIdleConnections)
+		if !uc.FallbackToDirectIP() {
+			t.Fatal("FallbackToDirectIP did not fire for a Control D upstream")
+		}
+		if got, err := dialDOH(t, transport, port); err != nil || got != "::1" {
+			t.Fatalf("dialed %q (err %v), want the bootstrap address ::1", got, err)
+		}
+		if uc.u.Host != "dns.controld.com" {
+			t.Fatalf("request host = %q, want the domain kept for TLS", uc.u.Host)
+		}
+	})
+
+	t.Run("v6 upstream with no addresses only gets IPv6 direct IPs", func(t *testing.T) {
+		port := listenLocal(t, "::1", "0")
+		useDirectIPs(t, "127.0.0.1", "::1")
+		uc := newBootstrapTestUpstream(ResolverTypeDOH, "https://dns.controld.com/abcdef", IpStackV6)
+		transport := uc.newDOHTransport(nil)
+		t.Cleanup(transport.CloseIdleConnections)
+		uc.FallbackToDirectIP()
+		if got := uc.directFallbackIPs(); !slices.Equal(got, []string{"::1"}) {
+			t.Fatalf("direct IPs for a v6 upstream = %v, want only [::1]", got)
+		}
+		if got, err := dialDOH(t, transport, port); err != nil || got != "::1" {
+			t.Fatalf("dialed %q (err %v), want the IPv6 direct IP ::1", got, err)
+		}
+	})
+
+	t.Run("direct IPs only after the list fails, and the list wins again once reachable", func(t *testing.T) {
+		port := listenLocal(t, "127.0.0.1", "0") // the direct IP
+		useDirectIPs(t, "127.0.0.1")
+		uc := newBootstrapTestUpstream(ResolverTypeDOH, "https://dns.controld.com/abcdef", IpStackSplit)
+		transport := uc.newDOHTransport([]string{"::1"}) // nothing listens there yet
+		t.Cleanup(transport.CloseIdleConnections)
+
+		if _, err := dialDOH(t, transport, port); err == nil {
+			t.Fatal("a failed list was retried on the direct IPs before the fallback fired")
+		}
+		uc.FallbackToDirectIP()
+		if got, err := dialDOH(t, transport, port); err != nil || got != "127.0.0.1" {
+			t.Fatalf("dialed %q (err %v), want the direct IP after the list failed", got, err)
+		}
+
+		// The bootstrap address is reachable again: it is dialed, not the direct IP.
+		listenLocal(t, "::1", port)
+		if got, err := dialDOH(t, transport, port); err != nil || got != "::1" {
+			t.Fatalf("dialed %q (err %v), want the recovered bootstrap address ::1", got, err)
+		}
+	})
+	t.Run("doh3 dials the direct IPs after the list fails", func(t *testing.T) {
+		cert := generateTestCertificate(t)
+		ln, err := quic.ListenAddr("127.0.0.1:0", &tls.Config{
+			Certificates: []tls.Certificate{cert.tlsCert},
+			NextProtos:   []string{http3.NextProtoH3},
+			MinVersion:   tls.VersionTLS13,
+		}, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(func() { ln.Close() })
+		go func() {
+			for {
+				if _, err := ln.Accept(context.Background()); err != nil {
+					return
+				}
+			}
+		}()
+		port := strconv.Itoa(ln.Addr().(*net.UDPAddr).Port)
+		useDirectIPs(t, "127.0.0.1")
+		uc := newBootstrapTestUpstream(ResolverTypeDOH3, "https://dns.controld.com/abcdef", IpStackV4)
+		rt := uc.newDOH3Transport(nil).(*http3.Transport)
+		t.Cleanup(func() { _ = rt.Close() })
+
+		pool := x509.NewCertPool()
+		pool.AddCert(cert.cert)
+		tlsCfg := &tls.Config{RootCAs: pool, ServerName: "localhost", NextProtos: []string{http3.NextProtoH3}, MinVersion: tls.VersionTLS13}
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		addr := net.JoinHostPort("dns.controld.com", port)
+
+		if _, err := rt.Dial(ctx, addr, tlsCfg, nil); err == nil {
+			t.Fatal("dial with no addresses succeeded before the fallback")
+		}
+		uc.FallbackToDirectIP()
+		conn, err := rt.Dial(ctx, addr, tlsCfg, nil)
+		if err != nil {
+			t.Fatalf("doh3 dial after the fallback: %v", err)
+		}
+		defer conn.CloseWithError(0, "")
+		if host, _, _ := net.SplitHostPort(conn.RemoteAddr().String()); host != "127.0.0.1" {
+			t.Fatalf("doh3 dialed %q, want the direct IP 127.0.0.1", host)
+		}
+	})
+}
+
+// DoT verifies the server as the upstream domain on every dial path, including
+// the empty-address path that dials the endpoint: tls.Client fails every
+// handshake without ServerName, and verification must stay on.
+func TestDOTClientPoolServerName(t *testing.T) {
+	uc := newBootstrapTestUpstream(ResolverTypeDOT, "dot.example.com:853", IpStackBoth)
+	p := newDOTClientPool(uc, nil)
+	t.Cleanup(p.Close)
+	if p.tlsConfig.ServerName != "dot.example.com" {
+		t.Fatalf("ServerName = %q, want the upstream domain", p.tlsConfig.ServerName)
+	}
+	if p.tlsConfig.InsecureSkipVerify {
+		t.Fatal("certificate verification is disabled")
+	}
+}

config_quic.go

@@ -42,17 +42,23 @@ func (uc *UpstreamConfig) newDOH3Transport(addrs []string) http.RoundTripper {
 			closeUDPConnWhenDone(conn, udpConn)
 			return conn, nil
 		}
-		dialAddrs := make([]string, len(addrs))
-		for i := range addrs {
-			dialAddrs[i] = net.JoinHostPort(addrs[i], port)
-		}
 		pd := &quicParallelDialer{}
-		conn, err := pd.Dial(ctx, dialAddrs, tlsCfg, cfg)
+		conn, err := pd.Dial(ctx, joinHostPorts(addrs, port), tlsCfg, cfg)
 		if err != nil {
-			return nil, err
+			direct := uc.directFallbackIPs()
+			if len(direct) == 0 {
+				return nil, err
+			}
+			ProxyLogger.Load().Debug().Err(err).Msgf("bootstrap addresses failed, dialing direct IPs %v", direct)
+			var directErr error
+			if conn, directErr = pd.Dial(ctx, joinHostPorts(direct, port), tlsCfg, cfg); directErr != nil {
+				return nil, errors.Join(err, directErr)
+			}
 		}
 		ProxyLogger.Load().Debug().Msgf("sending doh3 request to: %s", conn.RemoteAddr())
-		return conn, err
+		// err may still hold the bootstrap failure the direct dial recovered
+		// from; returning it would make http3 drop the connection.
+		return conn, nil
 	}
 	runtime.SetFinalizer(rt, func(rt *http3.Transport) {
 		rt.CloseIdleConnections()

dot.go

@@ -71,13 +71,13 @@ func newDOTClientPool(uc *UpstreamConfig, addrs []string) *dotConnPool {
 	// regardless of the machine DNS status.
 	dialer := newDialer(net.JoinHostPort(controldPublicDns, "53"))
 
+	// Every dial path verifies the server as uc.Domain: a bootstrap_ip, the
+	// bootstrap list, and also the empty-list path, which dials the endpoint.
+	// tls.Client needs ServerName there too; without it every handshake fails.
 	tlsConfig := &tls.Config{
 		RootCAs:    uc.certPool,
 		MinVersion: tls.VersionTLS12,
-	}
-
-	if uc.BootstrapIP != "" {
-		tlsConfig.ServerName = uc.Domain
+		ServerName: uc.Domain,
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())