mirror of
https://github.com/Control-D-Inc/ctrld.git
synced 2026-09-04 13:36:35 +02:00
Both API requests and binary downloads retry against a hard-coded IP when the attempt via hostname fails. Both then overwrote the first error with the fallback's, so only the last failure was reported. That discarded the diagnosis. During the Firewall Mode incident the hostname attempt was denied locally - WSAEACCES, "An attempt was made to access a socket in a way forbidden by its access permissions", which means the host is blocking ctrld - while the direct-ip fallback failed with an unreachable IPv6 route. What surfaced to the operator was "dial tcp6: no route to host", pointing at a routing problem that did not exist, while the WSAEACCES that named the real cause was visible only in debug logs. Report both failures instead, keeping the error chain intact so errors.Is still matches either one. Also switch the final wrap in doWithRetry from %v to %w, which had been flattening the chain even when a single error was reported. Add coverage for both paths, including that the fallback is still attempted and that a successful fallback returns no error. This also changes retry classification, which is worth stating explicitly because it is not obvious from "report both errors". processCDFlags decides whether to keep backing off with errUrlNetworkError, which uses errors.As - and errors.As returns the first match in the tree. Wrapping the hostname attempt first therefore hands the predicate that attempt's failure, where previously only the fallback's error survived to be classified. The effect is intended. A locally denied socket (WSAEACCES) is not a transient network error, so preflight now fails fast and reports instead of retrying against a firewall that is not going to clear on its own - the incident logged 256 retry cycles doing exactly that. The case that justifies retrying forever, a network unreachable on both attempts at boot, is unchanged. Both classifications are pinned by tests, along with the wrap order they depend on at each composition site, so reversing it fails loudly rather than silently restoring the old behaviour.
123 lines
4.2 KiB
Go
123 lines
4.2 KiB
Go
package cli
|
|
|
|
import (
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
// AppCallback provides hooks for injecting certain functionalities
|
|
// from mobile platforms to main ctrld cli.
|
|
// This allows mobile applications to customize behavior without modifying core CLI code
|
|
type AppCallback struct {
|
|
HostName func() string
|
|
LanIp func() string
|
|
MacAddress func() string
|
|
Exit func(error string)
|
|
}
|
|
|
|
// AppConfig allows overwriting ctrld cli flags from mobile platforms.
|
|
// This provides a clean interface for mobile apps to configure ctrld behavior
|
|
type AppConfig struct {
|
|
CdUID string
|
|
ProvisionID string
|
|
CustomHostname string
|
|
HomeDir string
|
|
UpstreamProto string
|
|
Verbose int
|
|
LogPath string
|
|
}
|
|
|
|
// Network and HTTP configuration constants
|
|
const (
|
|
// defaultHTTPTimeout provides reasonable timeout for HTTP operations
|
|
// This prevents hanging requests while allowing sufficient time for network delays
|
|
defaultHTTPTimeout = 30 * time.Second
|
|
|
|
// defaultMaxRetries provides retry attempts for failed HTTP requests
|
|
// This improves reliability in unstable network conditions
|
|
defaultMaxRetries = 3
|
|
|
|
// downloadServerIp is the fallback IP for download operations
|
|
// This ensures downloads work even when DNS resolution fails
|
|
downloadServerIp = "23.171.240.151"
|
|
)
|
|
|
|
// httpClientWithFallback returns an HTTP client configured with timeout and IPv4 fallback
|
|
// This ensures reliable HTTP operations by preferring IPv4 and handling timeouts gracefully
|
|
func httpClientWithFallback(timeout time.Duration) *http.Client {
|
|
return &http.Client{
|
|
Timeout: timeout,
|
|
Transport: &http.Transport{
|
|
// Prefer IPv4 over IPv6
|
|
// This improves compatibility with networks that have IPv6 issues
|
|
DialContext: (&net.Dialer{
|
|
Timeout: 10 * time.Second,
|
|
KeepAlive: 30 * time.Second,
|
|
FallbackDelay: 1 * time.Millisecond, // Very small delay to prefer IPv4
|
|
}).DialContext,
|
|
},
|
|
}
|
|
}
|
|
|
|
// doWithRetry performs an HTTP request with retries
|
|
// This improves reliability by automatically retrying failed requests with exponential backoff
|
|
func doWithRetry(req *http.Request, maxRetries int, ip string) (*http.Response, error) {
|
|
return doWithRetryClient(httpClientWithFallback(defaultHTTPTimeout), req, maxRetries, ip)
|
|
}
|
|
|
|
// doWithRetryClient is doWithRetry with an injectable client, so the retry and
|
|
// error-composition behaviour can be tested without real network access.
|
|
func doWithRetryClient(client *http.Client, req *http.Request, maxRetries int, ip string) (*http.Response, error) {
|
|
var lastErr error
|
|
var ipReq *http.Request
|
|
if ip != "" {
|
|
ipReq = req.Clone(req.Context())
|
|
ipReq.Host = ip
|
|
ipReq.URL.Host = ip
|
|
}
|
|
for attempt := 0; attempt < maxRetries; attempt++ {
|
|
if attempt > 0 {
|
|
// Linear backoff reduces server load and improves success rate
|
|
time.Sleep(time.Second * time.Duration(attempt+1))
|
|
}
|
|
|
|
resp, err := client.Do(req)
|
|
if err == nil {
|
|
return resp, nil
|
|
}
|
|
// Keep the hostname attempt's error: it carries the diagnosis (on Windows,
|
|
// a local firewall denying the socket shows up here as WSAEACCES), while the
|
|
// direct-ip fallback often fails for an unrelated reason such as an
|
|
// unreachable IPv6 route.
|
|
attemptErr := err
|
|
if ipReq != nil {
|
|
mainLog.Load().Warn().Err(err).Msgf("Dial to %q failed", req.Host)
|
|
mainLog.Load().Warn().Msgf("Fallback to direct ip to download prod version: %q", ip)
|
|
resp, fallbackErr := client.Do(ipReq)
|
|
if fallbackErr == nil {
|
|
return resp, nil
|
|
}
|
|
attemptErr = fmt.Errorf("%w; fallback to direct ip %s failed: %w", attemptErr, ip, fallbackErr)
|
|
}
|
|
|
|
lastErr = attemptErr
|
|
mainLog.Load().Debug().Err(attemptErr).
|
|
Str("method", req.Method).
|
|
Str("url", req.URL.String()).
|
|
Msgf("HTTP request attempt %d/%d failed", attempt+1, maxRetries)
|
|
}
|
|
return nil, fmt.Errorf("failed after %d attempts to %s %s: %w", maxRetries, req.Method, req.URL, lastErr)
|
|
}
|
|
|
|
// Helper for making GET requests with retries
|
|
// This provides a simplified interface for common GET operations with built-in retry logic
|
|
func getWithRetry(url string, ip string) (*http.Response, error) {
|
|
req, err := http.NewRequest(http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return doWithRetry(req, defaultMaxRetries, ip)
|
|
}
|