//go:build darwin package cli import ( "fmt" "net" "net/netip" "os/exec" "sort" "strings" "sync" "time" ) // pfFirewallState holds the state for pf-based firewall mode enforcement on macOS. // When firewall mode is active, we maintain a pf table of allowed IPs and add // block/pass rules to the ctrld anchor that enforce the allowlist. type pfFirewallState struct { // mu protects batch accumulation. mu sync.Mutex // pendingAdds and pendingRemoves accumulate changes for batched pf updates. pendingAdds []netip.Addr pendingRemoves []netip.Addr // batchTimer fires after the accumulation window to flush pending changes. batchTimer *time.Timer // lastForwardedSources is the forwarded-source set (auto-detected VM networks + // configured) that pf has actually accepted, and lastForwardedKey its // order-independent signature. reconcileForwardedSources compares against these // to rebuild the anchor when guests appear/disappear, and to know which subnets // gained or lost trust so their stale pf states can be dropped. Both advance only // after a successful anchor load, so a failed reload is retried, never latched. lastForwardedSources []forwardedSource lastForwardedKey string // applyForwardedMu serializes the compare-reload-record sequence in // applyForwardedSourceChange, so a watchdog tick and a network change cannot both // rebuild the anchor for the same transition or interleave their snapshot updates. // Separate from mu, which must not be held across pfctl execution. applyForwardedMu sync.Mutex } // forwardedSourceWarnTracker dedupes the "unusable configured entry" warning by // signature. Config is re-parsed on every anchor build and every watchdog tick, so // warning unconditionally would repeat the same line for the life of the process; // comparing signatures still reports an entry a config reload has just introduced. // // Package-level rather than a pfFirewallState field because it has to work before // that state exists: initPlatformFirewall defers pf enforcement until intercept mode // starts, so Firewall Mode can be on with no pfFirewallState yet. var forwardedSourceWarnTracker struct { mu sync.Mutex key string } const ( // pfFirewallTable is the pf table name for dynamically-allowed IPs. pfFirewallTable = "ctrld_allowed" // pfFirewallBatchInterval is the accumulation window for batching pf table updates. // Short enough for responsiveness, long enough to avoid per-DNS-response pfctl calls. pfFirewallBatchInterval = 200 * time.Millisecond ) // hypervisorVMNetPrefixes are interface-name prefixes that are specific to a // virtualization vendor's private VM/NAT network, and therefore reliable proof of // VM ownership by name alone. // // Two vendors still put the subnet directly on such an interface: Parallels (vnic0) // and VirtualBox host-only (vboxnet0). vmnet* covers legacy kext-based VMware Fusion // on Intel. They are also what proves ownership of a bridge they are a member of - // see bridgeHasVMMember. // // On current macOS this is not sufficient on its own. vmnet.framework - which backs // Virtualization.framework guests (UTM, Docker Desktop, Multipass, Fusion 12.1+ NAT) // - puts the RFC1918 gateway address on a bridge10x interface and attaches the // vendor-named vmenet* interface as an address-less member of it. Matching on name // alone therefore never fires for those stacks, which is why bridge membership is // checked too. var hypervisorVMNetPrefixes = []string{ "vmnet", // VMware Fusion (legacy kext-based, Intel) "vmenet", // vmnet.framework member interface (UTM, Docker, Multipass, Fusion 12.1+) "vnic", // Parallels Desktop "vboxnet", // VirtualBox host-only } // isHypervisorVMNetIface reports whether an interface name is a known // vendor-specific VM/NAT network (see hypervisorVMNetPrefixes). func isHypervisorVMNetIface(name string) bool { for _, prefix := range hypervisorVMNetPrefixes { if strings.HasPrefix(name, prefix) { return true } } return false } // isBridgeIface reports whether name is a macOS bridge interface. // // A bridge name is deliberately NOT treated as proof of anything: macOS uses this // namespace for Thunderbolt/aggregated links too (this repo's own tunnel-change code // lists bridge0 among physical interfaces), and auto-trusting those could force-route // unrelated same-subnet traffic. Ownership comes from the member list instead. func isBridgeIface(name string) bool { return strings.HasPrefix(name, "bridge") } // bridgeMembersFn returns a bridge's member interfaces. Indirected so detection can // be tested without a hypervisor or ifconfig. var bridgeMembersFn = bridgeMembers // bridgeMembers returns the member interfaces of a macOS bridge, via ifconfig. // There is no address-family-independent syscall for this that does not mean // hand-rolling SIOCGDRVSPEC/BRDGGIFS structs, and this runs at most once per // candidate bridge per anchor build. func bridgeMembers(name string) []string { out, err := exec.Command("ifconfig", name).CombinedOutput() if err != nil { return nil } return parseBridgeMembers(string(out)) } // parseBridgeMembers extracts member interface names from ifconfig output, which // lists each as a line of the form "\tmember: vmenet0 flags=3". func parseBridgeMembers(ifconfigOut string) []string { var members []string for _, line := range strings.Split(ifconfigOut, "\n") { fields := strings.Fields(line) if len(fields) < 2 || fields[0] != "member:" { continue } members = append(members, fields[1]) } return members } // bridgeHasVMMember reports whether a bridge's member list proves VM/NAT ownership, // i.e. whether a vendor-specific VM interface is bridged into it. // // This is the ownership evidence a bridge name lacks. A vmnet.framework bridge has a // vmenet* member; a Thunderbolt bridge has en* members and so is never trusted. func bridgeHasVMMember(members []string) bool { for _, m := range members { if isHypervisorVMNetIface(m) { return true } } return false } // firewallFlushPlatform flushes the pf table on macOS. func (p *prog) firewallFlushPlatform() { p.pfFirewallFlushTable() } // shutdownPlatformFirewall removes macOS firewall-mode dynamic state. The pf // anchor itself is rebuilt by DNS intercept without firewall rules once // p.allowList is nil. func (p *prog) shutdownPlatformFirewall() { p.pfFirewallFlushTable() if p.dnsInterceptState == nil { return } var vpnExemptions []vpnDNSExemption if p.vpnDNS != nil { vpnExemptions = p.vpnDNS.CurrentExemptions() } rulesStr := p.buildPFAnchorRules(vpnExemptions) if err := writePFAnchorFile(rulesStr); err != nil { p.Warn().Err(err).Msg("Firewall: failed to write pf anchor during shutdown") return } if out, err := exec.Command("pfctl", "-a", pfAnchorName, "-f", pfAnchorFile).CombinedOutput(); err != nil { p.Warn().Err(err).Str("output", strings.TrimSpace(string(out))).Msg("Firewall: failed to reload pf anchor during shutdown") } } // initPlatformFirewall initializes macOS-specific firewall enforcement (pf tables). func (p *prog) initPlatformFirewall() { if _, ok := p.platformFirewallState.(*pfFirewallState); ok { return } // pf enforcement is only meaningful when intercept mode is active - // without it, we have no pf anchor to add rules to. if dnsIntercept && p.dnsInterceptState != nil { p.initPFFirewall() } else { p.Info().Msg("Firewall: pf enforcement deferred until intercept mode starts") } } // initPFFirewall sets up pf-based firewall mode enforcement. Called from // initFirewallAllowList() when running on macOS with intercept mode active. // // Architecture: // - Creates a pf table for dynamic IP allowlisting // - Registers batch change callbacks on the AllowList // - The actual pf anchor rules are injected via buildPFFirewallRules() which // is called from buildPFAnchorRules() when firewall mode is active // // The table approach (vs. per-IP pass rules) is critical for performance: // pfctl table operations are O(log n) and don't require a full anchor reload. func (p *prog) initPFFirewall() { state := &pfFirewallState{} p.platformFirewallState = state // Register batch callback - AllowList reaper and FlushDomain use this. p.allowList.SetOnBatchChange(func(added []netip.Addr, removed []netip.Addr) { state.mu.Lock() defer state.mu.Unlock() if len(added) > 0 { state.pendingAdds = append(state.pendingAdds, added...) } if len(removed) > 0 { state.pendingRemoves = append(state.pendingRemoves, removed...) } state.scheduleBatchFlush(p) }) // Register individual change callback - Add() and Remove() use this. p.allowList.SetOnChange(func(ip netip.Addr, added bool) { state.mu.Lock() defer state.mu.Unlock() if added { state.pendingAdds = append(state.pendingAdds, ip) } else { state.pendingRemoves = append(state.pendingRemoves, ip) } state.scheduleBatchFlush(p) }) // DNS responses may have populated the allowlist before platform callbacks // were registered. Bulk-load that snapshot so pf starts with the same view // as the in-memory allowlist. p.pfFirewallPopulateTable() // Seed the forwarded-source snapshot with what the anchor was just built with, // so the first reconcile only fires on a real subsequent change. sources := p.forwardedSources() state.lastForwardedSources = sources state.lastForwardedKey = forwardedSourceSetKey(sources) // Report the effective trust set, including the configured entries. Without this // an admin who sets firewall_forwarded_sources has no way to confirm it took // effect short of reading pf rules. p.logForwardedSources(sources) p.Info().Msg("Firewall: pf table enforcement initialized") } // scheduleBatchFlush starts or resets the batch timer. Must be called with state.mu held. func (s *pfFirewallState) scheduleBatchFlush(p *prog) { if s.batchTimer != nil { return } s.batchTimer = time.AfterFunc(pfFirewallBatchInterval, func() { s.flushBatch(p) }) } // flushBatch applies accumulated pf table changes in a single pfctl call per direction. func (s *pfFirewallState) flushBatch(p *prog) { s.mu.Lock() adds := s.pendingAdds removes := s.pendingRemoves s.pendingAdds = nil s.pendingRemoves = nil s.batchTimer = nil s.mu.Unlock() if len(adds) == 0 && len(removes) == 0 { return } // Collapse add/remove deltas into the current primary allowlist state. This // avoids leaving pf opposite the allowlist when an Add and Remove for the // same IP land in one batch window. ipsToSync := make(map[netip.Addr]struct{}, len(adds)+len(removes)) for _, ip := range adds { ipsToSync[ip] = struct{}{} } for _, ip := range removes { ipsToSync[ip] = struct{}{} } var tableAdds, tableRemoves []string for ip := range ipsToSync { if p.allowList != nil && p.allowList.Contains(ip) { tableAdds = append(tableAdds, ip.String()) } else { tableRemoves = append(tableRemoves, ip.String()) } } // Apply additions. if len(tableAdds) > 0 { // pfctl -t -T add accepts multiple IPs space-separated. args := append([]string{"-a", pfAnchorName, "-t", pfFirewallTable, "-T", "add"}, tableAdds...) if out, err := exec.Command("pfctl", args...).CombinedOutput(); err != nil { p.Warn().Err(err).Str("output", string(out)). Msgf("Firewall: failed to add %d IPs to pf table", len(tableAdds)) } else { p.Debug().Msgf("Firewall: added %d IPs to pf table %s", len(tableAdds), pfFirewallTable) } } // Apply removals. if len(tableRemoves) > 0 { args := append([]string{"-a", pfAnchorName, "-t", pfFirewallTable, "-T", "delete"}, tableRemoves...) if out, err := exec.Command("pfctl", args...).CombinedOutput(); err != nil { // Not a hard error - the IP may have already been removed (e.g., by a Flush). p.Debug().Err(err).Str("output", string(out)). Msgf("Firewall: failed to remove %d IPs from pf table (may already be gone)", len(tableRemoves)) } else { p.Debug().Msgf("Firewall: removed %d IPs from pf table %s", len(tableRemoves), pfFirewallTable) } } } // pfFirewallFlushTable removes all entries from the pf firewall table. // Called on network changes and config reloads before the AllowList is flushed. func (p *prog) pfFirewallFlushTable() { if state, ok := p.platformFirewallState.(*pfFirewallState); ok && state != nil { state.mu.Lock() if state.batchTimer != nil { state.batchTimer.Stop() state.batchTimer = nil } state.pendingAdds = nil state.pendingRemoves = nil state.mu.Unlock() } out, err := exec.Command("pfctl", "-a", pfAnchorName, "-t", pfFirewallTable, "-T", "flush").CombinedOutput() if err != nil { p.Debug().Err(err).Str("output", string(out)).Msg("Firewall: failed to flush pf table (may not exist yet)") } else { p.Info().Msg("Firewall: flushed pf table " + pfFirewallTable) } } // pfFirewallPopulateTable bulk-loads all currently allowed IPs into the pf table. // Called after anchor rule installation to ensure the table is populated. func (p *prog) pfFirewallPopulateTable() { if p.allowList == nil { return } ips := p.allowList.AllowedIPs() if len(ips) == 0 { return } ipStrs := make([]string, 0, len(ips)) for _, ip := range ips { ipStrs = append(ipStrs, ip.String()) } args := append([]string{"-a", pfAnchorName, "-t", pfFirewallTable, "-T", "add"}, ipStrs...) if out, err := exec.Command("pfctl", args...).CombinedOutput(); err != nil { p.Warn().Err(err).Str("output", string(out)). Msgf("Firewall: failed to populate pf table with %d IPs", len(ips)) } else { p.Info().Msgf("Firewall: populated pf table with %d allowed IPs", len(ips)) } } // buildPFFirewallRules generates the pf rules for firewall mode enforcement. // These rules are appended to the anchor by buildPFAnchorRules() when firewall // mode is active. // // The strategy is: // - Define table (dynamically populated via pfctl -T add/delete) // - Block all outbound traffic by default (after DNS intercept rules) // - Pass outbound to IPs in // - Pass outbound from ctrld's group (already handled by blanket exemption) // - Pass loopback, link-local, multicast (already handled by permanent allowlist, // but explicit pf rules prevent kernel-level blocking before our check) // // IMPORTANT: These rules must come AFTER the DNS intercept rules in the anchor // so that DNS itself still works (DNS is how IPs get into the allowlist). func buildPFFirewallRules() string { var rules strings.Builder rules.WriteString("\n# --- Firewall Mode: DNS-resolved IP allowlist enforcement ---\n") rules.WriteString("# Only IPs resolved by ctrld are allowed for outbound connections.\n") rules.WriteString("# Table is dynamically populated from DNS responses.\n\n") // Declare the table. pfctl -T add/delete operates on this table dynamically. fmt.Fprintf(&rules, "table <%s> persist\n\n", pfFirewallTable) // Pass traffic to allowed IPs (both IPv4 and IPv6). rules.WriteString("# Allow outbound to DNS-resolved IPs.\n") fmt.Fprintf(&rules, "pass out quick inet proto { tcp, udp } from any to <%s>\n", pfFirewallTable) fmt.Fprintf(&rules, "pass out quick inet6 proto { tcp, udp } from any to <%s>\n\n", pfFirewallTable) // Allow ICMP/ICMPv6 - needed for path MTU discovery, ping, etc. rules.WriteString("# Allow ICMP (path MTU discovery, ping, etc.)\n") rules.WriteString("pass out quick inet proto icmp\n") rules.WriteString("pass out quick inet6 proto icmp6\n\n") // Allow all loopback traffic (safety net - permanent allowlist covers this too). rules.WriteString("# Allow all loopback traffic.\n") rules.WriteString("pass out quick on lo0\n") rules.WriteString("pass in quick on lo0\n\n") // Allow RFC1918 and link-local - these are in the permanent allowlist but // explicit pf rules prevent the block rule below from catching them. rules.WriteString("# Allow private/link-local ranges (LAN, printers, NAS, mDNS, DHCP).\n") rules.WriteString("pass out quick inet proto { tcp, udp } from any to 10.0.0.0/8\n") rules.WriteString("pass out quick inet proto { tcp, udp } from any to 172.16.0.0/12\n") rules.WriteString("pass out quick inet proto { tcp, udp } from any to 192.168.0.0/16\n") rules.WriteString("pass out quick inet proto { tcp, udp } from any to 169.254.0.0/16\n") rules.WriteString("pass out quick inet proto { tcp, udp } from any to 100.64.0.0/10\n") rules.WriteString("pass out quick inet6 proto { tcp, udp } from any to fe80::/10\n\n") // Allow multicast (mDNS, SSDP, etc.). rules.WriteString("# Allow multicast (mDNS, SSDP, etc.).\n") rules.WriteString("pass out quick inet proto { tcp, udp } from any to 224.0.0.0/4\n") rules.WriteString("pass out quick inet6 proto { tcp, udp } from any to ff00::/8\n\n") // Allow DHCP (UDP 67/68) - needed for network configuration. rules.WriteString("# Allow DHCP.\n") rules.WriteString("pass out quick inet proto udp from any port 68 to any port 67\n\n") // Block everything else. This is the enforcement rule. // "block return" sends TCP RST / ICMP unreachable so apps fail fast instead of timing out. rules.WriteString("# Block all other outbound traffic (IPs not resolved by ctrld).\n") rules.WriteString("block return out quick inet proto { tcp, udp } from any to any\n") rules.WriteString("block return out quick inet6 proto { tcp, udp } from any to any\n") return rules.String() } // forwardedSource is a trusted VM/container source subnet. iface is the ingress // interface for an auto-detected source, used to scope its pf rules to that // interface so unrelated same-subnet traffic on other interfaces is unaffected. // iface is empty for an operator-configured source, which matches on the source // CIDR alone — an explicit opt-in the admin is responsible for. type forwardedSource struct { prefix netip.Prefix iface string } // firewallForwardedSources parses the operator-configured forwarded-workload // source subnets (service.firewall_forwarded_sources), dropping - with a warning - // any entry that is not a valid CIDR, or is not IPv4, so one bad line never voids the // whole set. These augment auto-detection (see forwardedSources) and are the supported // way to trust generic bridge* stacks (Multipass, Docker Desktop). // // The IPv4 restriction is not cosmetic: forwarded DNS can only be redirected to // ctrld's IPv4 intercept listener, and an IPv6 source would otherwise produce pf rules // whose address family contradicts their source literal, which makes pfctl reject the // entire anchor and take DNS interception down with it. func (p *prog) firewallForwardedSources() []forwardedSource { sources, rejected := parseForwardedSourceConfig(p.cfg.Service.FirewallForwardedSources) p.warnRejectedForwardedSources(rejected) return sources } // rejectedForwardedSource is a configured entry that could not be used, with the // reason to report to the operator. type rejectedForwardedSource struct { value string reason string } // parseForwardedSourceConfig parses configured entries, returning the usable sources // and the rejected ones. It is pure and silent: callers decide when a rejection is // worth logging, because this runs on every anchor build and every watchdog tick. func parseForwardedSourceConfig(raw []string) ([]forwardedSource, []rejectedForwardedSource) { if len(raw) == 0 { return nil, nil } out := make([]forwardedSource, 0, len(raw)) var rejected []rejectedForwardedSource for _, s := range raw { pfx, err := netip.ParsePrefix(strings.TrimSpace(s)) if err != nil { rejected = append(rejected, rejectedForwardedSource{ value: s, reason: "not a valid CIDR (want e.g. 192.168.64.0/24): " + err.Error(), }) continue } if !pfx.Addr().Is4() { rejected = append(rejected, rejectedForwardedSource{ value: s, reason: "not IPv4 - forwarded-workload DNS interception is IPv4-only (ctrld's intercept listener is IPv4)", }) continue } out = append(out, forwardedSource{prefix: pfx.Masked()}) } return out, rejected } // warnRejectedForwardedSources reports unusable configured entries, but only when the // set of rejections changes (see forwardedSourceWarnTracker). func (p *prog) warnRejectedForwardedSources(rejected []rejectedForwardedSource) { key := rejectedForwardedSourcesKey(rejected) forwardedSourceWarnTracker.mu.Lock() unchanged := forwardedSourceWarnTracker.key == key forwardedSourceWarnTracker.key = key forwardedSourceWarnTracker.mu.Unlock() if unchanged { return } for _, r := range rejected { p.Warn().Str("value", r.value). Msgf("Firewall: ignoring firewall_forwarded_sources entry - %s", r.reason) } } // rejectedForwardedSourcesKey returns an order-independent signature of a rejection // set, so re-parsing an unchanged config is recognised as nothing new to report. func rejectedForwardedSourcesKey(rejected []rejectedForwardedSource) string { if len(rejected) == 0 { return "" } parts := make([]string, 0, len(rejected)) for _, r := range rejected { parts = append(parts, r.value+"|"+r.reason) } sort.Strings(parts) return strings.Join(parts, ",") } // detectForwardedSources auto-discovers VM/NAT networks on the host so the common // case works with no configuration. // // An interface qualifies only if it is up, carries an RFC1918 IPv4 network, AND its // VM ownership can be proven one of two ways: // // - its own name is vendor-specific (hypervisorVMNetPrefixes): Parallels, legacy // Fusion, VirtualBox host-only; // - it is a bridge whose member list contains a vendor-specific interface // (bridgeHasVMMember): every vmnet.framework stack, where the address lives on // bridge10x and the vendor-named vmenet* member has none. // // Ownership proof is the security boundary: a bridge is trusted for what is bridged // into it, never for its name. The RFC1918 filter is the second boundary - a stack // presenting a public range is never auto-trusted. Each detected source keeps its // ingress interface so its pf rules stay scoped to it. // // Addresses are checked before membership so the ifconfig call is only made for a // bridge that could actually qualify; a Thunderbolt bridge with no RFC1918 address // costs nothing. func (p *prog) detectForwardedSources() []forwardedSource { ifaces, err := net.Interfaces() if err != nil { p.Warn().Err(err).Msg("Firewall: could not enumerate interfaces for forwarded-source detection") return nil } var out []forwardedSource for _, ifi := range ifaces { if ifi.Flags&net.FlagUp == 0 { continue } addrs, err := ifi.Addrs() if err != nil { continue } sources, reason := forwardedSourcesForIface(ifi.Name, addrs, bridgeMembersFn) for _, src := range sources { out = append(out, src) // Debug, not Info: detection re-runs on every anchor build. The effective // set is reported once by logForwardedSources at init and on every change. p.Debug().Str("iface", src.iface).Str("subnet", src.prefix.String()).Str("reason", reason). Msg("Firewall: auto-detected VM/container network for forwarded DNS") } } return out } // forwardedSourcesForIface decides whether one up interface is an auto-trusted // forwarded-workload source, and returns its subnets plus why it qualified. // // Split out from interface enumeration so the trust decision - which is a security // boundary - is testable against synthetic interfaces, including the cases that must // NOT qualify: a Thunderbolt bridge, a public-range VM network, an address-less // vendor interface. // // members is only consulted for a bridge that already has an RFC1918 IPv4 address, so // no subprocess runs for the address-less or public bridges on a typical host. func forwardedSourcesForIface(name string, addrs []net.Addr, members func(string) []string) ([]forwardedSource, string) { vendorNamed := isHypervisorVMNetIface(name) if !vendorNamed && !isBridgeIface(name) { return nil, "" } prefixes := privateIPv4Prefixes(addrs) if len(prefixes) == 0 { return nil, "" } reason := "vendor VM interface" if !vendorNamed { mem := members(name) if !bridgeHasVMMember(mem) { return nil, "" } reason = "bridge with VM member " + strings.Join(vmMembers(mem), ",") } out := make([]forwardedSource, 0, len(prefixes)) for _, pfx := range prefixes { out = append(out, forwardedSource{prefix: pfx, iface: name}) } return out, reason } // privateIPv4Prefixes returns the masked RFC1918 IPv4 networks among addrs, skipping // public and IPv6 addresses. func privateIPv4Prefixes(addrs []net.Addr) []netip.Prefix { var out []netip.Prefix for _, a := range addrs { ipnet, ok := a.(*net.IPNet) if !ok { continue } pfx, err := netip.ParsePrefix(ipnet.String()) if err != nil { continue } pfx = pfx.Masked() if !pfx.Addr().Is4() || !pfx.Addr().IsPrivate() { continue } out = append(out, pfx) } return out } // vmMembers returns the vendor VM interfaces among a bridge's members, for logging // which member made the bridge trusted. func vmMembers(members []string) []string { var out []string for _, m := range members { if isHypervisorVMNetIface(m) { out = append(out, m) } } return out } // forwardedSources returns the effective trusted set: auto-detected vendor VM // networks UNION operator-configured subnets, de-duplicated by prefix (a subnet // that is both auto-detected and configured keeps the interface-scoped // auto-detected form). Config augments auto-detection; it never disables it. func (p *prog) forwardedSources() []forwardedSource { var out []forwardedSource seen := make(map[netip.Prefix]struct{}) for _, group := range [][]forwardedSource{ p.detectForwardedSources(), p.firewallForwardedSources(), } { for _, src := range group { if _, dup := seen[src.prefix]; dup { continue } seen[src.prefix] = struct{}{} out = append(out, src) } } return out } // forwardedSourceSetKey returns a deterministic, order-independent signature of a // forwarded-source set. reconcileForwardedSources compares it across time to detect // when VM/container interfaces appear or disappear. func forwardedSourceSetKey(sources []forwardedSource) string { parts := make([]string, 0, len(sources)) for _, s := range sources { parts = append(parts, s.iface+"|"+s.prefix.String()) } sort.Strings(parts) return strings.Join(parts, ",") } // currentForwardedSources returns the effective forwarded-source set, or nil when // Firewall Mode is off. // // The gate matters because detection is not free: it enumerates interfaces, execs // ifconfig for each candidate bridge, and re-parses config. Those rules are only ever // emitted in firewall mode, so doing any of it with firewall mode off is pure waste - // and it would report unusable config entries on a path where nothing wants them. // Anchor rebuilds happen on every tunnel change, watchdog restore and VPN DNS update // regardless of firewall mode, so this is the difference between zero work and work on // every one of them. func (p *prog) currentForwardedSources() []forwardedSource { if !p.firewallModeEnabled() { return nil } return p.forwardedSources() } // buildPFForwardedSourceRulesFor generates the pf rules that make the given // VM/container (forwarded/NATed) source subnets first-class Firewall Mode clients // WITHOUT an interface-wide bypass. Returns "" for an empty input so anchor // behavior is unchanged when there are no sources. // // For each source subnet: // // - Plaintext DNS (port 53) is force-routed through ctrld's loopback listener // (route-to lo0, which then hits the existing rdr-on-lo0 redirect). ctrld // therefore observes and policy-enforces every guest resolution, and the // resolved IP lands in . The guest's subsequent egress to that // IP is permitted by the existing "pass out ... to " rule: NAT // rewrites the guest source to the host, which that rule's "from any" covers. // - DoT (port 853) to any resolver is blocked so the guest cannot swap in an // alternate encrypted resolver to escape policy. DoH over 443 is // indistinguishable from ordinary HTTPS and is a documented limitation. // // Auto-detected sources carry their ingress interface and are scoped with // "on ", so an unrelated interface on the same private range is never // affected. Configured sources (admin opt-in) match on the source CIDR alone. // All matches are on the pre-NAT guest source (inbound): after NAT the source is // the host and could no longer be told apart. This is an explicit, per-subnet // trust boundary - a direct public IP the guest never resolved through ctrld stays // blocked, so the guest cannot bypass Control D policy. // // IPv4 only, and strictly per address family: ctrld's intercept listener is IPv4, so // IPv6 guest DNS cannot be redirected here - the anchor's existing // "block out ... inet6 ... port 53" rule forces guests to fall back to interceptable // IPv4 DNS, and guest IPv6 egress to an alternate resolver is covered by the blanket // IPv6 block plus the fact that such a resolver never enters . // Consequently only IPv4 sources produce rules, and each rule's address family // matches its source literal: pf rejects a whole anchor over a single // "inet6 ... from 192.168.x.0/24" mismatch, which would take DNS interception down // with it. Non-IPv4 sources are skipped here as a backstop; firewallForwardedSources // already drops them at parse time with a warning. func buildPFForwardedSourceRulesFor(sources []forwardedSource, listenerIP string) string { sources = ipv4ForwardedSources(sources) if len(sources) == 0 { return "" } var rules strings.Builder rules.WriteString("\n# --- Firewall Mode: forwarded workload (VM/container) DNS interception ---\n") rules.WriteString("# VM/container source subnets (auto-detected vendor VM networks + configured).\n") rules.WriteString("# Their DNS is forced through ctrld so guest resolutions are policy-enforced and\n") fmt.Fprintf(&rules, "# populate <%s>; egress to allowed IPs is then permitted by the allowlist rule\n", pfFirewallTable) rules.WriteString("# below. Auto-detected sources are scoped to their ingress interface; this is an\n") rules.WriteString("# explicit, per-subnet trust boundary - NOT an interface-wide permit.\n\n") for _, src := range sources { cidr := src.prefix.String() on := "" label := "configured (source-CIDR scope)" if src.iface != "" { on = "on " + src.iface + " " label = "auto-detected on " + src.iface } fmt.Fprintf(&rules, "# %s - %s\n", cidr, label) // Force guest plaintext DNS (port 53) onto loopback, where the existing // rdr-on-lo0 rule redirects it to ctrld. Matched inbound (pre-NAT); "quick" // so it wins over the blanket block that buildPFFirewallRules appends after. fmt.Fprintf(&rules, "pass in quick %sroute-to lo0 inet proto udp from %s to ! %s port 53\n", on, cidr, listenerIP) fmt.Fprintf(&rules, "pass in quick %sroute-to lo0 inet proto tcp from %s to ! %s port 53\n", on, cidr, listenerIP) // Block DoT so the guest cannot escape ctrld via an alternate encrypted // resolver. DoH over 443 is indistinguishable from HTTPS - documented limitation. // inet only: the source literal is IPv4, and pf refuses to load an anchor // containing an inet6 rule with an IPv4 source. fmt.Fprintf(&rules, "block return in quick %sinet proto { tcp, udp } from %s to any port 853\n\n", on, cidr) } return rules.String() } // ipv4ForwardedSources returns the IPv4 subset of sources. Forwarded-workload DNS // interception is IPv4-only (the intercept listener is IPv4), and every emitted rule // must match its source's address family or pfctl rejects the entire anchor. func ipv4ForwardedSources(sources []forwardedSource) []forwardedSource { out := make([]forwardedSource, 0, len(sources)) for _, src := range sources { if src.prefix.Addr().Is4() { out = append(out, src) } } return out } // reconcileForwardedSources rebuilds the pf anchor when the effective forwarded- // source set (auto-detected VM networks + configured) has changed since the last // build, and drops the pf states of every subnet whose trust changed. // // VM/container interfaces appear and disappear at runtime (guest start/stop) and no // existing path rebuilds an otherwise-intact anchor for that: ensurePFAnchorActive() // returns early while the rules still exist, checkTunnelInterfaceChanges() tracks // only tunnel interfaces, and pfInterceptMonitor() rebuilds only after the host // interception probe fails. Without this, a guest started after ctrld would never be // trusted, and a stopped guest's subnet would stay trusted until an unrelated // rebuild. Invoked from the network-change paths and, as a time bound when no event // fires, from the pf watchdog tick. // // A reload failure is not latched: the applied snapshot only advances once pf has // actually accepted the new anchor, so the next reconcile (at the latest the next // watchdog tick) retries the same change instead of treating it as done. func (p *prog) reconcileForwardedSources() { if !p.firewallModeEnabled() || p.dnsInterceptState == nil { return } state, ok := p.platformFirewallState.(*pfFirewallState) if !ok || state == nil { return } sources := p.forwardedSources() reload := func() error { return p.reloadForwardedSourceAnchor(sources) } gained, lost, changed, err := state.applyForwardedSourceChange(sources, reload) if !changed { return } if err != nil { p.Warn().Err(err).Strs("gained_trust", prefixStrings(gained)).Strs("lost_trust", prefixStrings(lost)). Msg("Firewall: forwarded-source set changed but the pf anchor reload failed — keeping the previously applied set so the next reconcile retries") return } p.Info().Strs("gained_trust", prefixStrings(gained)).Strs("lost_trust", prefixStrings(lost)). Msg("Firewall: forwarded-source set changed (VM/container start/stop), pf anchor rebuilt") // Restate the whole effective set, so one log line always answers "what is trusted // right now" without replaying every earlier transition. p.logForwardedSources(sources) // Rules only govern new states, so kill the states of the affected subnets: // a subnet that lost trust must stop using states created while it was trusted, // and one that just gained it must have its pre-existing (un-intercepted) DNS // states re-evaluated instead of running until they expire. The guest simply // re-establishes the connections under the new rules. Only reached after a // successful load - killing states against the old anchor would just have them // recreated under the very rules the change was meant to replace. p.killForwardedSourceStates(append(gained, lost...)) } // applyForwardedSourceChange performs one reconcile step: compare cur against the // last applied forwarded-source set and, when it differs, install it via reload. // It reports which subnets gained and lost trust (trust identity is (subnet, // interface scope), so a subnet that stays but changes scope appears in both lists), // whether there was anything to do at all, and reload's error. // // The applied snapshot advances ONLY after reload returns nil. A failed write or // pfctl load therefore leaves the previous set recorded, so the running anchor and // the snapshot cannot diverge and the change is retried on the next reconcile rather // than silently dropped. The whole check-reload-record sequence is serialized so // concurrent callers (watchdog tick vs. network change) cannot both rebuild or // interleave their snapshot updates. // // Split out from reconcileForwardedSources so the guest start/stop lifecycle and the // failure-then-retry path are deterministically testable without pf or a hypervisor. func (s *pfFirewallState) applyForwardedSourceChange(cur []forwardedSource, reload func() error) (gained, lost []netip.Prefix, changed bool, err error) { s.applyForwardedMu.Lock() defer s.applyForwardedMu.Unlock() key := forwardedSourceSetKey(cur) s.mu.Lock() applied, appliedKey := s.lastForwardedSources, s.lastForwardedKey s.mu.Unlock() if key == appliedKey { return nil, nil, false, nil } gained = forwardedSubnetsNotIn(cur, applied) lost = forwardedSubnetsNotIn(applied, cur) if err := reload(); err != nil { return gained, lost, true, err } s.mu.Lock() s.lastForwardedSources = cur s.lastForwardedKey = key s.mu.Unlock() return gained, lost, true, nil } // forwardedSubnetsNotIn returns the subnets of a whose exact trust entry (subnet + // interface scope) is absent from b, de-duplicated. func forwardedSubnetsNotIn(a, b []forwardedSource) []netip.Prefix { inB := make(map[forwardedSource]struct{}, len(b)) for _, src := range b { inB[src] = struct{}{} } var out []netip.Prefix seen := make(map[netip.Prefix]struct{}, len(a)) for _, src := range a { if _, ok := inB[src]; ok { continue } if _, dup := seen[src.prefix]; dup { continue } seen[src.prefix] = struct{}{} out = append(out, src.prefix) } return out } // prefixStrings renders prefixes for logging. func prefixStrings(prefixes []netip.Prefix) []string { out := make([]string, 0, len(prefixes)) for _, pfx := range prefixes { out = append(out, pfx.String()) } return out } // forwardedSourceDescriptions renders the effective trust set for logging, naming each // subnet's origin so an admin can tell an auto-detected VM network (and the interface // its rules are scoped to) from an entry they configured. func forwardedSourceDescriptions(sources []forwardedSource) []string { out := make([]string, 0, len(sources)) for _, src := range sources { if src.iface != "" { out = append(out, src.prefix.String()+" (auto-detected on "+src.iface+")") continue } out = append(out, src.prefix.String()+" (configured)") } return out } // logForwardedSources reports the effective forwarded-workload trust set. Configured // entries were previously invisible in the log - only auto-detection said anything - // so an admin had no way to confirm firewall_forwarded_sources took effect. The empty // case is logged too, with what to do about it, because "no guest DNS interception" // looks identical to "feature silently did nothing". func (p *prog) logForwardedSources(sources []forwardedSource) { if len(sources) == 0 { p.Info().Msg("Firewall: no forwarded-workload (VM/container) sources — guest DNS is not intercepted. " + "Auto-detection needs an up interface with an RFC1918 IPv4 address that is either vendor-named " + "(vnic*, vboxnet*, vmnet*) or a bridge with a VM member (vmenet*); anything else must be listed " + "in service.firewall_forwarded_sources") return } p.Info().Int("count", len(sources)).Strs("sources", forwardedSourceDescriptions(sources)). Msg("Firewall: forwarded-workload (VM/container) DNS interception active for these source subnets") } // killForwardedSourceStates drops the pf state entries sourced from the given // subnets. Targeted (pfctl -k ) rather than a global state flush, so a // guest starting or stopping never resets unrelated host connections. func (p *prog) killForwardedSourceStates(prefixes []netip.Prefix) { for _, pfx := range prefixes { out, err := exec.Command("pfctl", "-k", pfx.String()).CombinedOutput() if err != nil { // Not a hard error - most often there simply are no matching states. p.Debug().Err(err).Str("subnet", pfx.String()).Str("output", strings.TrimSpace(string(out))). Msg("Firewall: could not kill pf states for changed forwarded source") continue } p.Info().Str("subnet", pfx.String()). Msg("Firewall: killed pf states for changed forwarded source") } } // recordAppliedForwardedSources records sources as the set pf is now enforcing. // // Called by the rebuild paths that are not the forwarded-source reconcile itself // (tunnel change, watchdog restore, VPN DNS exemptions, forced reload, startup): each // of those installs a full anchor that already contains the current set, so without // this the next reconcile would compare against a stale snapshot and redo the work. // // Takes applyForwardedMu so a record cannot land in the middle of a reconcile's // compare-reload-record sequence and be overwritten by it, or overwrite it. func (p *prog) recordAppliedForwardedSources(sources []forwardedSource) { state, ok := p.platformFirewallState.(*pfFirewallState) if !ok || state == nil { return } state.applyForwardedMu.Lock() defer state.applyForwardedMu.Unlock() state.mu.Lock() state.lastForwardedSources = sources state.lastForwardedKey = forwardedSourceSetKey(sources) state.mu.Unlock() } // reloadForwardedSourceAnchor rebuilds and reloads the ctrld pf anchor so the given // forwarded-source rules take effect. Mirrors the anchor reload used by the intercept // watchdog, but reports failure to the caller: whether pf actually accepted the anchor // decides whether the new source set may be recorded as applied. // // The set is passed in rather than re-detected, so what gets loaded is exactly what // the caller compared and will record. Re-detecting here could install a set that // differs from the recorded snapshot if an interface appeared in between. func (p *prog) reloadForwardedSourceAnchor(sources []forwardedSource) error { var vpnExemptions []vpnDNSExemption if p.vpnDNS != nil { vpnExemptions = p.vpnDNS.CurrentExemptions() } rulesStr := p.buildPFAnchorRulesWith(vpnExemptions, sources) if err := writePFAnchorFile(rulesStr); err != nil { return fmt.Errorf("write pf anchor %s: %w", pfAnchorFile, err) } if out, err := exec.Command("pfctl", "-a", pfAnchorName, "-f", pfAnchorFile).CombinedOutput(); err != nil { return fmt.Errorf("load pf anchor %s: %w (output: %s)", pfAnchorName, err, strings.TrimSpace(string(out))) } return nil }