Rework DoH/DoH3 transport setup/bootstrapping

The current transport setup is using mutex lock for synchronization.
This could work ok in normal device, but on low capacity routers, this
high contention may affect the performance, causing ctrld hangs.

Instead of using mutex lock, using atomic operation for synchronization
yield a better performance:

 - There's no lock, so other requests won't be blocked. And even theses
   requests use old broken transport, it would be fine, because the
   client will retry them later.

 - The setup transport is now done once, on demand when the transport is
   accessed, or when signal rebootsrapping. The first call to
   dohTransport will block others, but the transport is warmup before
   ctrld start serving requests, so client requests won't be affected.

That helps ctrld handling the requests better when running on low
capacity device.

Further more, the transport configuration is also tweaked for better
default performance:

 - MaxIdleConnsPerHost is set to 100 (default is 2), which allows more
   connections to be reused, reduce the load to open/close connections
   on demand. See [1] for a real example.

 - Due to the raising of MaxIdleConnsPerHost, once the transport is
   GC-ed, it must explicitly close its idle connections.

 - TLS client session cache is now enabled.

Last but not least, the upstream ping process is also reworked. DoH
transport is an HTTP transport, so doing a HEAD request is enough to
warmup the transport, instead of doing a full DNS query.

[1]: https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/274

cmd/ctrld/prog.go

@@ -91,7 +91,7 @@ func (p *prog) run() {
 			mainLog.Info().Str("bootstrap_ip", uc.BootstrapIP).Msgf("Using bootstrap IP for upstream.%s", n)
 		}
 		uc.SetCertPool(rootCertPool)
-		uc.SetupTransport()
+		go uc.Ping()
 	}
 
 	go p.watchLinkState()

config.go

@@ -5,13 +5,16 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
+	"io"
 	"math/rand"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
+	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/go-playground/validator/v10"
@@ -154,11 +157,12 @@ type UpstreamConfig struct {
 	SendClientInfo *bool `mapstructure:"send_client_info" toml:"send_client_info,omitempty"`
 
 	g                  singleflight.Group
-	mu                 sync.Mutex
+	rebootstrap        atomic.Bool
 	bootstrapIPs       []string
 	bootstrapIPs4      []string
 	bootstrapIPs6      []string
 	transport          *http.Transport
+	transportOnce      sync.Once
 	transport4         *http.Transport
 	transport6         *http.Transport
 	http3RoundTripper  http.RoundTripper
@@ -306,20 +310,11 @@ func (uc *UpstreamConfig) ReBootstrap() {
 	}
 	_, _, _ = uc.g.Do("ReBootstrap", func() (any, error) {
 		ProxyLog.Debug().Msg("re-bootstrapping upstream ip")
-		uc.setupTransportWithoutPingUpstream()
+		uc.rebootstrap.Store(true)
 		return true, nil
 	})
 }
 
-func (uc *UpstreamConfig) setupTransportWithoutPingUpstream() {
-	switch uc.Type {
-	case ResolverTypeDOH:
-		uc.setupDOHTransportWithoutPingUpstream()
-	case ResolverTypeDOH3:
-		uc.setupDOH3TransportWithoutPingUpstream()
-	}
-}
-
 // SetupTransport initializes the network transport used to connect to upstream server.
 // For now, only DoH upstream is supported.
 func (uc *UpstreamConfig) SetupTransport() {
@@ -332,14 +327,31 @@ func (uc *UpstreamConfig) SetupTransport() {
 }
 
 func (uc *UpstreamConfig) setupDOHTransport() {
-	uc.setupDOHTransportWithoutPingUpstream()
+	switch uc.IPStack {
-	go uc.pingUpstream()
+	case IpStackBoth, "":
+		uc.transport = uc.newDOHTransport(uc.bootstrapIPs)
+	case IpStackV4:
+		uc.transport = uc.newDOHTransport(uc.bootstrapIPs4)
+	case IpStackV6:
+		uc.transport = uc.newDOHTransport(uc.bootstrapIPs6)
+	case IpStackSplit:
+		uc.transport4 = uc.newDOHTransport(uc.bootstrapIPs4)
+		if hasIPv6() {
+			uc.transport6 = uc.newDOHTransport(uc.bootstrapIPs6)
+		} else {
+			uc.transport6 = uc.transport4
+		}
+		uc.transport = uc.newDOHTransport(uc.bootstrapIPs)
+	}
 }
 
 func (uc *UpstreamConfig) newDOHTransport(addrs []string) *http.Transport {
 	transport := http.DefaultTransport.(*http.Transport).Clone()
-	transport.IdleConnTimeout = 5 * time.Second
+	transport.MaxIdleConnsPerHost = 100
-	transport.TLSClientConfig = &tls.Config{RootCAs: uc.certPool}
+	transport.TLSClientConfig = &tls.Config{
+		RootCAs:            uc.certPool,
+		ClientSessionCache: tls.NewLRUClientSessionCache(0),
+	}
 
 	dialerTimeoutMs := 2000
 	if uc.Timeout > 0 && uc.Timeout < dialerTimeoutMs {
@@ -368,44 +380,39 @@ func (uc *UpstreamConfig) newDOHTransport(addrs []string) *http.Transport {
 		Log(ctx, ProxyLog.Debug(), "sending doh request to: %s", conn.RemoteAddr())
 		return conn, nil
 	}
+	runtime.SetFinalizer(transport, func(transport *http.Transport) {
+		transport.CloseIdleConnections()
+	})
 	return transport
 }
 
-func (uc *UpstreamConfig) setupDOHTransportWithoutPingUpstream() {
+// Ping warms up the connection to DoH/DoH3 upstream.
-	uc.mu.Lock()
+func (uc *UpstreamConfig) Ping() {
-	defer uc.mu.Unlock()
+	switch uc.Type {
-	switch uc.IPStack {
+	case ResolverTypeDOH, ResolverTypeDOH3:
-	case IpStackBoth, "":
+	default:
-		uc.transport = uc.newDOHTransport(uc.bootstrapIPs)
-	case IpStackV4:
-		uc.transport = uc.newDOHTransport(uc.bootstrapIPs4)
-	case IpStackV6:
-		uc.transport = uc.newDOHTransport(uc.bootstrapIPs6)
-	case IpStackSplit:
-		uc.transport4 = uc.newDOHTransport(uc.bootstrapIPs4)
-		if hasIPv6() {
-			uc.transport6 = uc.newDOHTransport(uc.bootstrapIPs6)
-		} else {
-			uc.transport6 = uc.transport4
-		}
-
-		uc.transport = uc.newDOHTransport(uc.bootstrapIPs)
-	}
-}
-
-func (uc *UpstreamConfig) pingUpstream() {
-	// Warming up the transport by querying a test packet.
-	dnsResolver, err := NewResolver(uc)
-	if err != nil {
-		ProxyLog.Error().Err(err).Msgf("failed to create resolver for upstream: %s", uc.Name)
 		return
 	}
-	msg := new(dns.Msg)
+
-	msg.SetQuestion(".", dns.TypeNS)
+	ping := func(t http.RoundTripper) {
-	msg.MsgHdr.RecursionDesired = true
+		if t == nil {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			return
-	defer cancel()
+		}
-	_, _ = dnsResolver.Resolve(ctx, msg)
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		req, _ := http.NewRequestWithContext(ctx, "HEAD", uc.Endpoint, nil)
+		resp, _ := t.RoundTrip(req)
+		if resp == nil {
+			return
+		}
+		defer resp.Body.Close()
+		_, _ = io.Copy(io.Discard, resp.Body)
+	}
+
+	for _, typ := range []uint16{dns.TypeA, dns.TypeAAAA} {
+		ping(uc.dohTransport(typ))
+		ping(uc.doh3Transport(typ))
+	}
 }
 
 func (uc *UpstreamConfig) isControlD() bool {
@@ -424,8 +431,12 @@ func (uc *UpstreamConfig) isControlD() bool {
 }
 
 func (uc *UpstreamConfig) dohTransport(dnsType uint16) http.RoundTripper {
-	uc.mu.Lock()
+	uc.transportOnce.Do(func() {
-	defer uc.mu.Unlock()
+		uc.SetupTransport()
+	})
+	if uc.rebootstrap.CompareAndSwap(true, false) {
+		uc.SetupTransport()
+	}
 	switch uc.IPStack {
 	case IpStackBoth, IpStackV4, IpStackV6:
 		return uc.transport

config_quic.go

@@ -19,8 +19,24 @@ import (
 )
 
 func (uc *UpstreamConfig) setupDOH3Transport() {
-	uc.setupDOH3TransportWithoutPingUpstream()
+	switch uc.IPStack {
-	go uc.pingUpstream()
+	case IpStackBoth, "":
+		uc.http3RoundTripper = uc.newDOH3Transport(uc.bootstrapIPs)
+	case IpStackV4:
+		uc.http3RoundTripper = uc.newDOH3Transport(uc.bootstrapIPs4)
+	case IpStackV6:
+		uc.http3RoundTripper = uc.newDOH3Transport(uc.bootstrapIPs6)
+	case IpStackSplit:
+		uc.http3RoundTripper4 = uc.newDOH3Transport(uc.bootstrapIPs4)
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		if ctrldnet.IPv6Available(ctx) {
+			uc.http3RoundTripper6 = uc.newDOH3Transport(uc.bootstrapIPs6)
+		} else {
+			uc.http3RoundTripper6 = uc.http3RoundTripper4
+		}
+		uc.http3RoundTripper = uc.newDOH3Transport(uc.bootstrapIPs)
+	}
 }
 
 func (uc *UpstreamConfig) newDOH3Transport(addrs []string) http.RoundTripper {
@@ -58,32 +74,13 @@ func (uc *UpstreamConfig) newDOH3Transport(addrs []string) http.RoundTripper {
 	return rt
 }
 
-func (uc *UpstreamConfig) setupDOH3TransportWithoutPingUpstream() {
-	uc.mu.Lock()
-	defer uc.mu.Unlock()
-	switch uc.IPStack {
-	case IpStackBoth, "":
-		uc.http3RoundTripper = uc.newDOH3Transport(uc.bootstrapIPs)
-	case IpStackV4:
-		uc.http3RoundTripper = uc.newDOH3Transport(uc.bootstrapIPs4)
-	case IpStackV6:
-		uc.http3RoundTripper = uc.newDOH3Transport(uc.bootstrapIPs6)
-	case IpStackSplit:
-		uc.http3RoundTripper4 = uc.newDOH3Transport(uc.bootstrapIPs4)
-		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
-		defer cancel()
-		if ctrldnet.IPv6Available(ctx) {
-			uc.http3RoundTripper6 = uc.newDOH3Transport(uc.bootstrapIPs6)
-		} else {
-			uc.http3RoundTripper6 = uc.http3RoundTripper4
-		}
-		uc.http3RoundTripper = uc.newDOH3Transport(uc.bootstrapIPs)
-	}
-}
-
 func (uc *UpstreamConfig) doh3Transport(dnsType uint16) http.RoundTripper {
-	uc.mu.Lock()
+	uc.transportOnce.Do(func() {
-	defer uc.mu.Unlock()
+		uc.SetupTransport()
+	})
+	if uc.rebootstrap.CompareAndSwap(true, false) {
+		uc.SetupTransport()
+	}
 	switch uc.IPStack {
 	case IpStackBoth, IpStackV4, IpStackV6:
 		return uc.http3RoundTripper

config_quic_free.go

@@ -6,5 +6,4 @@ import "net/http"
 
 func (uc *UpstreamConfig) setupDOH3Transport() {}
 
-func (uc *UpstreamConfig) setupDOH3TransportWithoutPingUpstream()         {}
 func (uc *UpstreamConfig) doh3Transport(dnsType uint16) http.RoundTripper { return nil }