Files
ctrld/cmd/cli/library.go
T
Cuong Manh Le 052b057756 all: keep the first-attempt error when the direct-ip fallback also fails
Both API requests and binary downloads retry against a hard-coded IP when the
attempt via hostname fails. Both then overwrote the first error with the
fallback's, so only the last failure was reported.

That discarded the diagnosis. During the Firewall Mode incident the hostname
attempt was denied locally - WSAEACCES, "An attempt was made to access a socket
in a way forbidden by its access permissions", which means the host is blocking
ctrld - while the direct-ip fallback failed with an unreachable IPv6 route. What
surfaced to the operator was "dial tcp6: no route to host", pointing at a
routing problem that did not exist, while the WSAEACCES that named the real
cause was visible only in debug logs.

Report both failures instead, keeping the error chain intact so errors.Is still
matches either one. Also switch the final wrap in doWithRetry from %v to %w,
which had been flattening the chain even when a single error was reported.

Add coverage for both paths, including that the fallback is still attempted and
that a successful fallback returns no error.

This also changes retry classification, which is worth stating explicitly because
it is not obvious from "report both errors". processCDFlags decides whether to
keep backing off with errUrlNetworkError, which uses errors.As - and errors.As
returns the first match in the tree. Wrapping the hostname attempt first
therefore hands the predicate that attempt's failure, where previously only the
fallback's error survived to be classified.

The effect is intended. A locally denied socket (WSAEACCES) is not a transient
network error, so preflight now fails fast and reports instead of retrying
against a firewall that is not going to clear on its own - the incident logged
256 retry cycles doing exactly that. The case that justifies retrying forever, a
network unreachable on both attempts at boot, is unchanged. Both classifications
are pinned by tests, along with the wrap order they depend on at each composition
site, so reversing it fails loudly rather than silently restoring the old
behaviour.
2026-08-14 15:29:32 +07:00

123 lines
4.2 KiB
Go

package cli
import (
"fmt"
"net"
"net/http"
"time"
)
// AppCallback provides hooks for injecting certain functionalities
// from mobile platforms to main ctrld cli.
// This allows mobile applications to customize behavior without modifying core CLI code
type AppCallback struct {
HostName func() string
LanIp func() string
MacAddress func() string
Exit func(error string)
}
// AppConfig allows overwriting ctrld cli flags from mobile platforms.
// This provides a clean interface for mobile apps to configure ctrld behavior
type AppConfig struct {
CdUID string
ProvisionID string
CustomHostname string
HomeDir string
UpstreamProto string
Verbose int
LogPath string
}
// Network and HTTP configuration constants
const (
// defaultHTTPTimeout provides reasonable timeout for HTTP operations
// This prevents hanging requests while allowing sufficient time for network delays
defaultHTTPTimeout = 30 * time.Second
// defaultMaxRetries provides retry attempts for failed HTTP requests
// This improves reliability in unstable network conditions
defaultMaxRetries = 3
// downloadServerIp is the fallback IP for download operations
// This ensures downloads work even when DNS resolution fails
downloadServerIp = "23.171.240.151"
)
// httpClientWithFallback returns an HTTP client configured with timeout and IPv4 fallback
// This ensures reliable HTTP operations by preferring IPv4 and handling timeouts gracefully
func httpClientWithFallback(timeout time.Duration) *http.Client {
return &http.Client{
Timeout: timeout,
Transport: &http.Transport{
// Prefer IPv4 over IPv6
// This improves compatibility with networks that have IPv6 issues
DialContext: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
FallbackDelay: 1 * time.Millisecond, // Very small delay to prefer IPv4
}).DialContext,
},
}
}
// doWithRetry performs an HTTP request with retries
// This improves reliability by automatically retrying failed requests with exponential backoff
func doWithRetry(req *http.Request, maxRetries int, ip string) (*http.Response, error) {
return doWithRetryClient(httpClientWithFallback(defaultHTTPTimeout), req, maxRetries, ip)
}
// doWithRetryClient is doWithRetry with an injectable client, so the retry and
// error-composition behaviour can be tested without real network access.
func doWithRetryClient(client *http.Client, req *http.Request, maxRetries int, ip string) (*http.Response, error) {
var lastErr error
var ipReq *http.Request
if ip != "" {
ipReq = req.Clone(req.Context())
ipReq.Host = ip
ipReq.URL.Host = ip
}
for attempt := 0; attempt < maxRetries; attempt++ {
if attempt > 0 {
// Linear backoff reduces server load and improves success rate
time.Sleep(time.Second * time.Duration(attempt+1))
}
resp, err := client.Do(req)
if err == nil {
return resp, nil
}
// Keep the hostname attempt's error: it carries the diagnosis (on Windows,
// a local firewall denying the socket shows up here as WSAEACCES), while the
// direct-ip fallback often fails for an unrelated reason such as an
// unreachable IPv6 route.
attemptErr := err
if ipReq != nil {
mainLog.Load().Warn().Err(err).Msgf("Dial to %q failed", req.Host)
mainLog.Load().Warn().Msgf("Fallback to direct ip to download prod version: %q", ip)
resp, fallbackErr := client.Do(ipReq)
if fallbackErr == nil {
return resp, nil
}
attemptErr = fmt.Errorf("%w; fallback to direct ip %s failed: %w", attemptErr, ip, fallbackErr)
}
lastErr = attemptErr
mainLog.Load().Debug().Err(attemptErr).
Str("method", req.Method).
Str("url", req.URL.String()).
Msgf("HTTP request attempt %d/%d failed", attempt+1, maxRetries)
}
return nil, fmt.Errorf("failed after %d attempts to %s %s: %w", maxRetries, req.Method, req.URL, lastErr)
}
// Helper for making GET requests with retries
// This provides a simplified interface for common GET operations with built-in retry logic
func getWithRetry(url string, ip string) (*http.Response, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
return doWithRetry(req, defaultMaxRetries, ip)
}