all: permit ctrld's own endpoints in Firewall Mode

Firewall Mode permits only what ctrld resolved through its own listener. The API transport resolves api.controld.com through the OS nameservers and falls back to hardcoded addresses, so nothing ever teaches the allowlist about it and ctrld's own block-all filters deny its control-plane socket. The upgrade download server has the same shape: performUpgrade spawns a detached child process, and the WFP filters carry no process condition, so the service blocks its own upgrade. Permit both permanently, at startup and on reload. For the API that means the resolved addresses and the transport's direct fallbacks - the fallbacks are what it dials when DNS is unusable, which is the state a blocked ctrld is in. For the download server only the fallback IP is needed, since its hostname lookup does go through the listener and is learned. APIDomain/APIEndpointIPs are exported so the permitted set and the dialed set cannot drift apart. Call initPlatformFirewall on reload even when enforcement is already up. AddPermanent fires no change callback, so an address permitted by a reload reached memory only while the platform never heard about it. Each platform's re-entry is a refresh: Windows reinstalls the permanent filters it is missing, macOS returns early. Also keep every dial attempt in the transport's error. It returned only the last stage, an unroutable IPv6 address reporting "no route to host", hiding the IPv4 WSAEACCES that named the real cause. The direct IPs are still always dialed, so the API stays reachable without DNS; only a duplicate dial of an address the resolver already returned is dropped.
2026-09-04 13:36:35 +02:00 · 2026-08-28 14:02:24 +07:00
parent 4113064680
commit f96868c266
5 changed files with 452 additions and 47 deletions
@@ -0,0 +1,178 @@
+package controld
+
+import (
+	"errors"
+	"net"
+	"slices"
+	"strings"
+	"syscall"
+	"testing"
+)
+
+// TestJoinAttemptErrorsKeepsEveryAttempt pins the diagnosis the incident lost.
+//
+// The transport dials several address families in turn. The IPv4 attempt is the
+// one that says "the host is blocking ctrld"; the last attempt is usually an IPv6
+// address that is simply unroutable and reports "no route to host". Returning only
+// the last error is what turned a self-inflicted block into a phantom routing
+// problem in the logs, and sent the investigation after a network fault that did
+// not exist.
+func TestJoinAttemptErrorsKeepsEveryAttempt(t *testing.T) {
+	blocked := &net.OpError{Op: "dial", Net: "tcp4", Err: wsaEACCES}
+	unroutable := &net.OpError{Op: "dial", Net: "tcp6", Err: syscall.EHOSTUNREACH}
+
+	err := joinAttemptErrors([]error{
+		wrapAttempt("resolved ipv4", blocked),
+		wrapAttempt("direct ipv6", unroutable),
+	})
+	if err == nil {
+		t.Fatal("joinAttemptErrors() = nil for two failed attempts")
+	}
+
+	msg := err.Error()
+	for _, want := range []string{"resolved ipv4", "direct ipv6"} {
+		if !strings.Contains(msg, want) {
+			t.Errorf("error text does not name the %q attempt: %s", want, msg)
+		}
+	}
+	if !errors.Is(err, wsaEACCES) {
+		t.Errorf("the IPv4 socket denial did not survive; a caller can no longer tell a local block from a routing failure: %s", msg)
+	}
+	if !errors.Is(err, syscall.EHOSTUNREACH) {
+		t.Errorf("the last attempt's error did not survive: %s", msg)
+	}
+	if strings.Contains(msg, "\n") {
+		t.Errorf("the joined error spans lines, which breaks one-record-per-failure logging: %q", msg)
+	}
+}
+
+// TestJoinAttemptErrorsSingleAndEmpty covers the degenerate inputs: one attempt is
+// returned untouched, and no attempt at all still has to be an error rather than a
+// nil the dialer would hand back as a successful connection.
+func TestJoinAttemptErrorsSingleAndEmpty(t *testing.T) {
+	only := errors.New("only attempt")
+	if got := joinAttemptErrors([]error{only}); !errors.Is(got, only) {
+		t.Errorf("joinAttemptErrors() = %v, want the single attempt unwrapped", got)
+	}
+	if got := joinAttemptErrors(nil); got == nil {
+		t.Error("joinAttemptErrors(nil) = nil; the dialer would report success with no connection")
+	}
+}
+
+// TestAPIDialStagesAlwaysDialTheDirectIPs is the guarantee the direct addresses
+// exist for: when DNS is unusable, ctrld must still reach the API.
+//
+// Whatever resolution returns - nothing, stale addresses, one family only - every
+// direct address is dialed. The only thing the duplicate trim removes is a second
+// dial of an address an earlier stage already covers.
+func TestAPIDialStagesAlwaysDialTheDirectIPs(t *testing.T) {
+	const (
+		directV4 = apiDomainComIPv4
+		directV6 = apiDomainComIPv6
+	)
+	v4, v6 := []string{directV4}, []string{directV6}
+
+	tests := []struct {
+		name     string
+		resolved []string
+	}{
+		{"resolution returned nothing", nil},
+		{"resolution returned the direct ips", []string{directV4, directV6}},
+		{"resolution returned stale ips", []string{"203.0.113.10", "2001:db8::1"}},
+		{"resolution returned ipv4 only", []string{"203.0.113.10"}},
+		{"resolution returned ipv6 only", []string{"2001:db8::1"}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			stages := apiDialStages(tt.resolved, v4, v6)
+
+			var dialed []string
+			for _, stage := range stages {
+				if len(stage.ips) == 0 {
+					t.Errorf("stage %q has no address; it would dial nothing", stage.what)
+				}
+				dialed = append(dialed, stage.ips...)
+			}
+			for _, direct := range []string{directV4, directV6} {
+				if !slices.Contains(dialed, direct) {
+					t.Errorf("the direct address %s is never dialed; the API is unreachable without DNS", direct)
+				}
+				if n := count(dialed, direct); n != 1 {
+					t.Errorf("the direct address %s is dialed %d times, want exactly 1", direct, n)
+				}
+			}
+			for _, ip := range tt.resolved {
+				if !slices.Contains(dialed, ip) {
+					t.Errorf("the resolved address %s is never dialed", ip)
+				}
+			}
+		})
+	}
+}
+
+// TestAPIDialStagesTryIPv4First pins the order: the IPv4 stages come before the
+// IPv6 ones. IPv6 at these hosts is commonly unroutable, and its "no route to
+// host" is what used to be the only error a failure reported.
+func TestAPIDialStagesTryIPv4First(t *testing.T) {
+	stages := apiDialStages([]string{"203.0.113.10", "2001:db8::1"},
+		[]string{apiDomainComIPv4}, []string{apiDomainComIPv6})
+
+	var order []string
+	for _, stage := range stages {
+		order = append(order, stage.network)
+	}
+	want := []string{"tcp4", "tcp4", "tcp6", "tcp6"}
+	if len(order) != len(want) {
+		t.Fatalf("stage networks = %v, want %v", order, want)
+	}
+	for i := range want {
+		if order[i] != want[i] {
+			t.Fatalf("stage networks = %v, want %v", order, want)
+		}
+	}
+}
+
+func count(haystack []string, needle string) int {
+	var n int
+	for _, s := range haystack {
+		if s == needle {
+			n++
+		}
+	}
+	return n
+}
+
+// TestNotInSkipsAlreadyDialedAddresses covers the duplicate-dial trim: LookupIP
+// normally answers with the direct addresses, so dialing both lists doubles every
+// failure for no added chance of success.
+func TestNotInSkipsAlreadyDialedAddresses(t *testing.T) {
+	if got := notIn([]string{apiDomainComIPv4}, []string{apiDomainComIPv4}); len(got) != 0 {
+		t.Errorf("notIn() = %v, want empty: the address was already dialed", got)
+	}
+	if got := notIn([]string{apiDomainComIPv4}, []string{"203.0.113.10"}); len(got) != 1 {
+		t.Errorf("notIn() = %v, want the direct address kept when it was not dialed", got)
+	}
+	if got := notIn([]string{apiDomainComIPv4}, nil); len(got) != 1 {
+		t.Errorf("notIn() = %v, want the direct address kept when nothing resolved", got)
+	}
+}
+
+// TestAPIEndpointIPsCoverEveryDialedAddress ties the Firewall Mode allowlist to the
+// transport. Firewall Mode permits APIEndpointIPs; the transport dials
+// apiDirectIPs. If one grows an address the other does not, ctrld starts blocking
+// its own control plane again, which is precisely the 38-hour outage.
+func TestAPIEndpointIPsCoverEveryDialedAddress(t *testing.T) {
+	for _, dev := range []bool{false, true} {
+		permitted := APIEndpointIPs(dev)
+		v4, v6 := apiDirectIPs(dev)
+		for _, ip := range append(append([]string{}, v4...), v6...) {
+			if !slices.Contains(permitted, ip) {
+				t.Errorf("cdDev=%v: the transport dials %s but APIEndpointIPs does not report it, so Firewall Mode will not permit it", dev, ip)
+			}
+		}
+		if len(permitted) != len(v4)+len(v6) {
+			t.Errorf("cdDev=%v: APIEndpointIPs = %v, but the split halves are %v/%v", dev, permitted, v4, v6)
+		}
+	}
+}
@@ -11,6 +11,7 @@ import (
 	"net"
 	"net/http"
 	"runtime"
+	"slices"
 	"strings"
 	"time"

@@ -322,20 +323,49 @@ func ParseRawUID(rawUID string) (string, string) {
 	return uid, clientID
 }

+// APIDomain returns the ControlD API hostname for the environment.
+func APIDomain(cdDev bool) string {
+	if cdDev {
+		return apiDomainDev
+	}
+	return apiDomainCom
+}
+
+// APIEndpointIPs returns the addresses the API transport dials directly when the
+// hostname cannot be resolved.
+//
+// Exported because Firewall Mode has to permit them: it blocks every destination
+// ctrld did not resolve through its own listener, and the API is resolved through
+// the OS resolver by LookupIP instead, so nothing ever teaches the allowlist about
+// it. Left unpermitted, ctrld's own block-all filters deny its API socket - which
+// is what stranded the 2026-08-16 Windows run with 920 WSAEACCES denials and not
+// one successful configuration refresh in 38 hours.
+func APIEndpointIPs(cdDev bool) []string {
+	if cdDev {
+		return []string{apiDomainDevIPv4}
+	}
+	return []string{apiDomainComIPv4, apiDomainComIPv6}
+}
+
+// apiDirectIPs splits APIEndpointIPs into its IPv4 and IPv6 halves.
+func apiDirectIPs(cdDev bool) (v4, v6 []string) {
+	for _, ip := range APIEndpointIPs(cdDev) {
+		if strings.Contains(ip, ":") {
+			v6 = append(v6, ip)
+		} else {
+			v4 = append(v4, ip)
+		}
+	}
+	return v4, v6
+}
+
 // apiTransport returns an HTTP transport for connecting to ControlD API endpoint.
 func apiTransport(loggerCtx context.Context, cdDev bool) *http.Transport {
 	transport := http.DefaultTransport.(*http.Transport).Clone()
 	transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
-		apiDomain := apiDomainCom
-		apiIpsV4 := []string{apiDomainComIPv4}
-		apiIpsV6 := []string{apiDomainComIPv6}
-		apiIPs := []string{apiDomainComIPv4, apiDomainComIPv6}
-		if cdDev {
-			apiDomain = apiDomainDev
-			apiIpsV4 = []string{apiDomainDevIPv4}
-			apiIpsV6 = []string{}
-			apiIPs = []string{apiDomainDevIPv4}
-		}
+		apiDomain := APIDomain(cdDev)
+		apiIpsV4, apiIpsV6 := apiDirectIPs(cdDev)
+		apiIPs := APIEndpointIPs(cdDev)

 		ips := ctrld.LookupIP(loggerCtx, apiDomain)
 		if len(ips) == 0 {
@@ -344,18 +374,6 @@ func apiTransport(loggerCtx context.Context, cdDev bool) *http.Transport {
 			ips = apiIPs
 		}

-		// Separate IPv4 and IPv6 addresses
-		// This separation is needed because different network stacks may have different
-		// connectivity to IPv4 vs IPv6, so we try them separately for better reliability
-		var ipv4s, ipv6s []string
-		for _, ip := range ips {
-			if strings.Contains(ip, ":") {
-				ipv6s = append(ipv6s, ip)
-			} else {
-				ipv4s = append(ipv4s, ip)
-			}
-		}
-
 		dial := func(ctx context.Context, network string, addrs []string) (net.Conn, error) {
 			d := &ctrldnet.ParallelDialer{}
 			logger := ctrld.LoggerFromCtx(loggerCtx)
@@ -363,25 +381,21 @@ func apiTransport(loggerCtx context.Context, cdDev bool) *http.Transport {
 		}
 		_, port, _ := net.SplitHostPort(addr)

-		// Try IPv4 first
-		if len(ipv4s) > 0 {
-			if conn, err := dial(ctx, "tcp4", addrsFromPort(ipv4s, port)); err == nil {
+		var attempts []error
+		for _, stage := range apiDialStages(ips, apiIpsV4, apiIpsV6) {
+			conn, err := dial(ctx, stage.network, addrsFromPort(stage.ips, port))
+			if err == nil {
 				return conn, nil
 			}
+			attempts = append(attempts, wrapAttempt(stage.what, err))
 		}
-		// Fallback to direct IPv4
-		if conn, err := dial(ctx, "tcp4", addrsFromPort(apiIpsV4, port)); err == nil {
-			return conn, nil
-		}
-
-		// Fallback to IPv6 if available
-		if len(ipv6s) > 0 {
-			if conn, err := dial(ctx, "tcp6", addrsFromPort(ipv6s, port)); err == nil {
-				return conn, nil
-			}
-		}
-		// Fallback to direct IPv6
-		return dial(ctx, "tcp6", addrsFromPort(apiIpsV6, port))
+		// Every attempt is reported, not just the last one. The stage that
+		// diagnoses a local block is the IPv4 one - on Windows a firewall denying
+		// ctrld's own socket surfaces there as WSAEACCES - while the last stage is
+		// an IPv6 address that is commonly unroutable and fails with a bare "no
+		// route to host". Returning only that turned a self-inflicted block into a
+		// phantom routing problem and sent an incident investigation the wrong way.
+		return nil, joinAttemptErrors(attempts)
 	}
 	if runtime.GOOS == "android" {
 		transport.TLSClientConfig = &tls.Config{RootCAs: certs.CACertPool(), MinVersion: tls.VersionTLS12}
@@ -389,6 +403,105 @@ func apiTransport(loggerCtx context.Context, cdDev bool) *http.Transport {
 	return transport
 }

+// apiDialStage is one attempt in the API transport's fallback order.
+type apiDialStage struct {
+	what    string
+	network string
+	ips     []string
+}
+
+// apiDialStages plans the dial order for one API connection: resolved IPv4, the
+// direct IPv4, then the same for IPv6. The families are attempted separately
+// because a host can have working connectivity to one and not the other.
+//
+// Every direct address is always dialed. It is the address that has to work when
+// DNS does not, so it is dropped from its own stage only when it is already in
+// the resolved list and the earlier stage therefore dials it anyway - dialing it
+// twice doubles the failures without adding a chance of success. If resolution
+// returns nothing, or returns addresses that are stale or wrong, the direct
+// stages still carry the full direct list.
+func apiDialStages(resolved, directV4, directV6 []string) []apiDialStage {
+	// Different network stacks may have different connectivity to IPv4 vs IPv6.
+	var ipv4s, ipv6s []string
+	for _, ip := range resolved {
+		if strings.Contains(ip, ":") {
+			ipv6s = append(ipv6s, ip)
+		} else {
+			ipv4s = append(ipv4s, ip)
+		}
+	}
+	stages := []apiDialStage{
+		{"resolved ipv4", "tcp4", ipv4s},
+		{"direct ipv4", "tcp4", notIn(directV4, ipv4s)},
+		{"resolved ipv6", "tcp6", ipv6s},
+		{"direct ipv6", "tcp6", notIn(directV6, ipv6s)},
+	}
+	out := make([]apiDialStage, 0, len(stages))
+	for _, stage := range stages {
+		if len(stage.ips) > 0 {
+			out = append(out, stage)
+		}
+	}
+	return out
+}
+
+// notIn returns the members of ips that are absent from seen.
+//
+// The direct-IP stages exist for when the hostname does not resolve, and LookupIP
+// usually answers with those very addresses, so dialing both lists doubles the
+// failures for no added chance of success.
+func notIn(ips, seen []string) []string {
+	if len(seen) == 0 {
+		return ips
+	}
+	var out []string
+	for _, ip := range ips {
+		if !slices.Contains(seen, ip) {
+			out = append(out, ip)
+		}
+	}
+	return out
+}
+
+// wrapAttempt labels one dial attempt's failure with the stage that produced it,
+// so a joined error says which family and which address list failed how.
+func wrapAttempt(what string, err error) error {
+	return fmt.Errorf("%s: %w", what, err)
+}
+
+// joinAttemptErrors combines the dial attempts into one error.
+func joinAttemptErrors(attempts []error) error {
+	switch len(attempts) {
+	case 0:
+		return errors.New("no api address to dial")
+	case 1:
+		return attempts[0]
+	}
+	return &dialAttemptsError{attempts: attempts}
+}
+
+// dialAttemptsError carries every attempt the API dialer made.
+//
+// errors.Join would do the same for errors.Is, but renders one attempt per line,
+// and these end up in a single log record; this keeps them on one line. Unwrap
+// returns all of them, so a caller testing for a specific errno - a local socket
+// denial rather than an unroutable address - finds it wherever in the sequence it
+// happened, not only if it happened last.
+type dialAttemptsError struct {
+	attempts []error
+}
+
+func (e *dialAttemptsError) Error() string {
+	msgs := make([]string, 0, len(e.attempts))
+	for _, err := range e.attempts {
+		msgs = append(msgs, err.Error())
+	}
+	return strings.Join(msgs, "; ")
+}
+
+// Unwrap exposes every attempt to errors.Is and errors.As.
+func (e *dialAttemptsError) Unwrap() []error { return e.attempts }
+
 func addrsFromPort(ips []string, port string) []string {
 	addrs := make([]string, len(ips))
 	for i, ip := range ips {