all: retry the request more agressively

For better recovery and dealing with network stack changes, this commit
change the request flow to:

failure of any kind -> recreate transport/re-bootstrap -> retry once

That would make ctrld recover from all scenarios in theory.
This commit is contained in:
Cuong Manh Le
2023-03-03 00:47:21 +07:00
committed by Cuong Manh Le
parent 262dcb1dff
commit fb20d443c1
6 changed files with 36 additions and 35 deletions

View File

@@ -14,14 +14,12 @@ import (
"github.com/go-playground/validator/v10"
"github.com/miekg/dns"
"github.com/spf13/viper"
"golang.org/x/sync/singleflight"
"github.com/Control-D-Inc/ctrld/internal/dnsrcode"
ctrldnet "github.com/Control-D-Inc/ctrld/internal/net"
)
// ErrUpstreamFailed indicates that ctrld failed to connect to an upstream.
var ErrUpstreamFailed = errors.New("could not connect to upstream")
// SetConfigName set the config name that ctrld will look for.
func SetConfigName(v *viper.Viper, name string) {
v.SetConfigName(name)
@@ -108,6 +106,7 @@ type UpstreamConfig struct {
transport *http.Transport `mapstructure:"-" toml:"-"`
http3RoundTripper http.RoundTripper `mapstructure:"-" toml:"-"`
g singleflight.Group
// guard BootstrapIP
mu sync.Mutex
}
@@ -154,6 +153,22 @@ func (uc *UpstreamConfig) Init() {
}
}
// ReBootstrap re-setup the bootstrap IP and the transport.
func (uc *UpstreamConfig) ReBootstrap() {
_, _, _ = uc.g.Do("rebootstrap", func() (any, error) {
ProxyLog.Debug().Msg("re-bootstrapping upstream ip")
ctrldnet.Reset()
err := uc.SetupBootstrapIP()
if err != nil {
ProxyLog.Error().Err(err).Msg("re-bootstrapping failed")
} else {
ProxyLog.Debug().Msgf("bootstrap ip set to: %s", uc.BootstrapIP)
}
uc.SetupTransport()
return err == nil, err
})
}
// SetupTransport initializes the network transport used to connect to upstream server.
// For now, only DoH upstream is supported.
func (uc *UpstreamConfig) SetupTransport() {
@@ -208,8 +223,8 @@ func (uc *UpstreamConfig) setupDOHTransport() {
uc.transport.IdleConnTimeout = 5 * time.Second
uc.transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
dialer := &net.Dialer{
Timeout: 5 * time.Second,
KeepAlive: 5 * time.Second,
Timeout: 2 * time.Second,
KeepAlive: 2 * time.Second,
}
Log(ctx, ProxyLog.Debug(), "debug dial context %s - %s - %s", addr, network, bootstrapDNS)
// if we have a bootstrap ip set, use it to avoid DNS lookup
@@ -219,12 +234,7 @@ func (uc *UpstreamConfig) setupDOHTransport() {
}
}
Log(ctx, ProxyLog.Debug(), "sending doh request to: %s", addr)
conn, err := dialer.DialContext(ctx, network, addr)
if err != nil {
Log(ctx, ProxyLog.Debug().Err(err), "could not dial to upstream")
return nil, ErrUpstreamFailed
}
return conn, nil
return dialer.DialContext(ctx, network, addr)
}
uc.pingUpstream()