From 779fe015f0b298f81130cc2d8f6d1e38c9906e9e Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Thu, 6 Aug 2026 11:51:55 +0000 Subject: [PATCH 01/16] fix: validate pf state before stabilization --- cmd/cli/dns_intercept_darwin.go | 868 ++++++++++-------- cmd/cli/dns_intercept_darwin_test.go | 613 +++++++++++++ ...s_intercept_ignored_change_windows_test.go | 20 + cmd/cli/dns_intercept_others.go | 9 +- cmd/cli/dns_intercept_settle.go | 16 +- cmd/cli/dns_intercept_settle_test.go | 9 +- cmd/cli/dns_intercept_windows.go | 10 +- cmd/cli/dns_proxy.go | 133 +-- cmd/cli/pf_ruleset.go | 79 ++ cmd/cli/pf_ruleset_test.go | 157 ++++ cmd/cli/prog.go | 102 +- cmd/cli/prog_intercept_fallback_test.go | 221 +++++ cmd/cli/vpn_dns.go | 103 ++- cmd/cli/vpn_dns_test.go | 163 +++- docs/pf-dns-intercept.md | 21 +- 15 files changed, 2036 insertions(+), 488 deletions(-) create mode 100644 cmd/cli/dns_intercept_ignored_change_windows_test.go create mode 100644 cmd/cli/pf_ruleset.go create mode 100644 cmd/cli/pf_ruleset_test.go create mode 100644 cmd/cli/prog_intercept_fallback_test.go diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go index 364a29a..76863b9 100644 --- a/cmd/cli/dns_intercept_darwin.go +++ b/cmd/cli/dns_intercept_darwin.go @@ -46,6 +46,15 @@ const ( // can turn a pf ruleset race into a fork/file-descriptor exhaustion loop. pfExecFailureBackoff = 5 * time.Second + // pfStabilizationMaxWait bounds the total period ordinary repair can be + // suppressed if PF never becomes stable or pfctl keeps failing. + pfStabilizationMaxWait = 90 * time.Second + + // pfIgnoredChangeReconcileInterval bounds leading-edge pf/VPN-DNS work + // during continuous ignored network-change storms. The 2s/4s delayed checks + // still provide trailing reconciliation once the storm stops. + pfIgnoredChangeReconcileInterval = pfAnchorRecheckDelayLong + // pfVPNInterfacePrefixes lists interface name prefixes that indicate VPN/tunnel // interfaces on macOS. Used to add interface-specific DNS intercept rules so that // VPN software with "pass out quick on " rules cannot bypass our intercept. @@ -199,6 +208,11 @@ func (p *prog) startDNSIntercept() error { if err := p.validateDNSIntercept(); err != nil { return err } + p.pfStabilizing.Store(false) + p.pfLastRestoreTime.Store(0) + p.pfBackoffMultiplier.Store(0) + p.pfExecBackoffUntil.Store(0) + p.pfIgnoredChangeLastReconcile.Store(0) // Set up _ctrld group for pf exemption scoping. This ensures that only ctrld's // own DNS queries (matching "group _ctrld" in pf rules) can bypass the redirect. @@ -258,20 +272,22 @@ func (p *prog) startDNSIntercept() error { out, err := exec.Command("pfctl", "-a", pfAnchorName, "-f", pfAnchorFile).CombinedOutput() if err != nil { - os.Remove(pfAnchorFile) + p.rollbackDNSInterceptStart() return fmt.Errorf("dns intercept: failed to load pf anchor: %w (output: %s)", err, strings.TrimSpace(string(out))) } mainLog.Load().Debug().Msgf("DNS intercept: loaded pf anchor %q from %s", pfAnchorName, pfAnchorFile) if err := p.ensurePFAnchorReference(); err != nil { - mainLog.Load().Warn().Err(err).Msg("DNS intercept: could not add anchor references to running pf ruleset — anchor may not be active") + p.rollbackDNSInterceptStart() + return fmt.Errorf("dns intercept: failed to activate pf anchor references: %w", err) } out, err = exec.Command("pfctl", "-e").CombinedOutput() if err != nil { outStr := strings.TrimSpace(string(out)) if !strings.Contains(outStr, "already enabled") { - mainLog.Load().Warn().Msgf("DNS intercept: pfctl -e returned: %s (err: %v) — pf may not be enabled", outStr, err) + p.rollbackDNSInterceptStart() + return fmt.Errorf("dns intercept: failed to enable pf: %w (output: %s)", err, outStr) } } @@ -289,8 +305,12 @@ func (p *prog) startDNSIntercept() error { mainLog.Load().Debug().Msgf("DNS intercept: active pf NAT/redirect rules:\n%s", strings.TrimSpace(string(out))) } - // Post-load verification: confirm everything actually took effect. - p.verifyPFState() + // Post-load verification: do not publish intercept mode unless PF is + // authoritatively active; the caller can then use interface-DNS fallback. + if !p.verifyPFState() { + p.rollbackDNSInterceptStart() + return fmt.Errorf("dns intercept: post-load PF verification failed") + } p.dnsInterceptState = &pfState{ anchorFile: pfAnchorFile, @@ -299,7 +319,9 @@ func (p *prog) startDNSIntercept() error { // Store the initial set of tunnel interfaces so we can detect changes later. p.mu.Lock() - p.lastTunnelIfaces = discoverTunnelInterfaces() + p.lastTunnelIfaces = discoverTunnelInterfacesForReconcile() + p.pendingTunnelIfaces = nil + p.hasPendingTunnelIfaces = false p.mu.Unlock() lc := p.cfg.FirstListener() @@ -316,6 +338,18 @@ func (p *prog) startDNSIntercept() error { return nil } +func (p *prog) rollbackDNSInterceptStart() { + if out, err := exec.Command("pfctl", "-a", pfAnchorName, "-F", "all").CombinedOutput(); err != nil { + mainLog.Load().Warn().Err(err).Msgf("DNS intercept: startup rollback could not flush anchor (output: %s)", strings.TrimSpace(string(out))) + } + if err := os.Remove(pfAnchorFile); err != nil && !os.IsNotExist(err) { + mainLog.Load().Warn().Err(err).Msg("DNS intercept: startup rollback could not remove anchor file") + } + if err := p.removePFAnchorReference(); err != nil { + mainLog.Load().Warn().Err(err).Msg("DNS intercept: startup rollback could not remove anchor references") + } +} + // ensurePFAnchorReference ensures the running pf ruleset includes our anchor // declarations. We dump the RUNNING ruleset via "pfctl -sr" (filter+scrub rules) // and "pfctl -sn" (NAT/rdr rules), check if our references exist, and if not, @@ -390,19 +424,19 @@ func (p *prog) ensurePFAnchorReference() error { pureFilterLines = append([]string{anchorRef}, pureFilterLines...) } - // Dump and clean pf options. VPN apps (e.g., Windscribe) set "set skip on { lo0 }" - // which disables pf processing on loopback, breaking our route-to + rdr mechanism. - // We strip lo0 and tunnel interfaces from the skip list before reloading. - cleanedOptions, hadLoopbackSkip := pfGetCleanedOptions() - if hadLoopbackSkip { - mainLog.Load().Info().Msg("DNS intercept: will reload pf options without lo0 in skip list") - } - - // Reassemble in pf's required order: options → scrub → translation → filtering. + // Reassemble in pf's required order: scrub → translation → filtering. + // + // KNOWN GAP: no options section is emitted, so this reload drops the running pf + // options - timeouts, limits and any third-party "set skip" directives - which by + // pf semantics revert to defaults. + // + // They cannot simply be carried across, because Apple's pfctl offers no way to read + // them: pfctl(8) accepts -s nat, queue, rules, Anchors, states, Sources, info, + // References, labels, timeouts, memory, Tables, osfp, Interfaces and all, with no + // options modifier. A fix therefore likely means not rebuilding the main ruleset + // from scratch at all. What macOS pf does to system options on a reload like this + // needs confirming on a host - tracked as follow-up to #573. var combined strings.Builder - if cleanedOptions != "" { - combined.WriteString(cleanedOptions) - } for _, line := range scrubLines { combined.WriteString(line + "\n") } @@ -444,139 +478,67 @@ func (p *prog) checkAnchorOrdering(filterLines []string, ourAnchorRef string) { } } -// pfGetCleanedOptions dumps the running pf options via "pfctl -sO" and returns -// them with lo0 removed from any "set skip on" directive. VPN apps like Windscribe -// set "set skip on { lo0 }" which tells pf to bypass ALL processing on -// loopback — this breaks our route-to + rdr interception mechanism which depends on -// lo0. We strip lo0 (and any known VPN tunnel interfaces) from the skip list so our -// rdr rules on lo0 can fire. Other options (timeouts, limits, etc.) are preserved. -// -// Returns the cleaned options as a string suitable for prepending to a pfctl -f reload, -// and a boolean indicating whether lo0 was found in the skip list (i.e., we needed to fix it). -func pfGetCleanedOptions() (string, bool) { - out, err := exec.Command("pfctl", "-sO").CombinedOutput() - if err != nil { - mainLog.Load().Debug().Err(err).Msg("DNS intercept: could not dump pf options") - return "", false - } - - var cleaned strings.Builder - hadLoopbackSkip := false - - for _, line := range strings.Split(string(out), "\n") { - line = strings.TrimSpace(line) - if line == "" || strings.Contains(line, "ALTQ") { - continue - } - - // Parse "set skip on { lo0 ipsec0 }" or "set skip on lo0" - if strings.HasPrefix(line, "set skip on") { - // Extract interface list from the skip directive. - skipPart := strings.TrimPrefix(line, "set skip on") - skipPart = strings.TrimSpace(skipPart) - skipPart = strings.Trim(skipPart, "{}") - skipPart = strings.TrimSpace(skipPart) - - ifaces := strings.Fields(skipPart) - var kept []string - for _, iface := range ifaces { - if iface == "lo0" { - hadLoopbackSkip = true - continue // Remove lo0 — we need pf to process lo0 for our rdr rules. - } - // Also remove VPN tunnel interfaces — we have explicit intercept - // rules for them in our anchor, so skipping defeats the purpose. - isTunnel := false - for _, prefix := range strings.Split(pfVPNInterfacePrefixes, ",") { - if strings.HasPrefix(iface, strings.TrimSpace(prefix)) { - isTunnel = true - break - } - } - if isTunnel { - mainLog.Load().Debug().Msgf("DNS intercept: removing tunnel interface %q from pf skip list", iface) - continue - } - kept = append(kept, iface) - } - - if len(kept) > 0 { - cleaned.WriteString(fmt.Sprintf("set skip on { %s }\n", strings.Join(kept, " "))) - } - // If no interfaces left, omit the skip directive entirely. - continue - } - - // Preserve all other options (timeouts, limits, etc.). - cleaned.WriteString(line + "\n") - } - - if hadLoopbackSkip { - mainLog.Load().Warn().Msg("DNS intercept: detected 'set skip on lo0' — another program (likely VPN software) " + - "disabled pf processing on loopback, which breaks our DNS interception. Removing lo0 from skip list.") - } - - return cleaned.String(), hadLoopbackSkip -} - -// pfFilterRuleLines filters pfctl output into actual pf rule lines, -// stripping stderr warnings (e.g., "No ALTQ support in kernel") and empty lines. -func pfFilterRuleLines(output string) []string { - var rules []string - for _, line := range strings.Split(output, "\n") { - line = strings.TrimSpace(line) - if line == "" { - continue - } - // Skip pfctl stderr warnings that appear in CombinedOutput. - if strings.Contains(line, "ALTQ") { - continue - } - rules = append(rules, line) - } - return rules -} - -// pfContainsRule checks if any line in the slice contains the given rule string. -// Uses substring matching because pfctl may append extra tokens like " all" to rules -// (e.g., `rdr-anchor "com.controld.ctrld" all`), which would fail exact matching. -func pfContainsRule(lines []string, rule string) bool { - for _, line := range lines { - if strings.Contains(line, rule) { - return true - } - } - return false -} - // stopDNSIntercept removes all pf rules and cleans up the DNS interception. func (p *prog) stopDNSIntercept() error { - if p.dnsInterceptState == nil { + state, ok := p.dnsInterceptState.(*pfState) + if !ok || state == nil { mainLog.Load().Debug().Msg("DNS intercept: no pf state to clean up") return nil } - mainLog.Load().Info().Msg("DNS intercept: shutting down pf redirect") + // Revoke lifecycle state and cancel future owners before waiting for any + // in-flight PF mutation. A mutation that already owns the lane may finish, + // but final cleanup always runs after it and no new owner can start. + p.dnsInterceptState = nil + pfShutdownStateRevokedForTest() + p.mu.Lock() + if p.pfStabilizeCancel != nil { + p.pfStabilizeCancel() + p.pfStabilizeCancel = nil + } + p.lastTunnelIfaces = nil + p.pendingTunnelIfaces = nil + p.hasPendingTunnelIfaces = false + p.mu.Unlock() + p.pfStabilizing.Store(false) - out, err := exec.Command("pfctl", "-a", p.dnsInterceptState.(*pfState).anchorName, "-F", "all").CombinedOutput() + p.pfDelayedRecheckMu.Lock() + for _, timer := range p.pfDelayedRecheckTimers { + if timer != nil { + timer.Stop() + } + } + p.pfDelayedRecheckTimers = nil + p.pfDelayedRecheckMu.Unlock() + + for !p.pfEnsureRunning.CompareAndSwap(false, true) { + time.Sleep(10 * time.Millisecond) + } + defer p.pfEnsureRunning.Store(false) + + mainLog.Load().Info().Msg("DNS intercept: shutting down pf redirect") + out, err := exec.Command("pfctl", "-a", state.anchorName, "-F", "all").CombinedOutput() if err != nil { mainLog.Load().Warn().Msgf("DNS intercept: failed to flush pf anchor %q: %v (output: %s)", - p.dnsInterceptState.(*pfState).anchorName, err, strings.TrimSpace(string(out))) + state.anchorName, err, strings.TrimSpace(string(out))) } else { - mainLog.Load().Debug().Msgf("DNS intercept: flushed pf anchor %q", p.dnsInterceptState.(*pfState).anchorName) + mainLog.Load().Debug().Msgf("DNS intercept: flushed pf anchor %q", state.anchorName) } - if err := os.Remove(p.dnsInterceptState.(*pfState).anchorFile); err != nil && !os.IsNotExist(err) { - mainLog.Load().Warn().Msgf("DNS intercept: failed to remove anchor file %s: %v", p.dnsInterceptState.(*pfState).anchorFile, err) + if err := os.Remove(state.anchorFile); err != nil && !os.IsNotExist(err) { + mainLog.Load().Warn().Msgf("DNS intercept: failed to remove anchor file %s: %v", state.anchorFile, err) } else { - mainLog.Load().Debug().Msgf("DNS intercept: removed anchor file %s", p.dnsInterceptState.(*pfState).anchorFile) + mainLog.Load().Debug().Msgf("DNS intercept: removed anchor file %s", state.anchorFile) } if err := p.removePFAnchorReference(); err != nil { mainLog.Load().Warn().Err(err).Msg("DNS intercept: failed to remove anchor references from running pf ruleset") } - p.dnsInterceptState = nil + p.pfLastRestoreTime.Store(0) + p.pfBackoffMultiplier.Store(0) + p.pfExecBackoffUntil.Store(0) + p.pfIgnoredChangeLastReconcile.Store(0) mainLog.Load().Info().Msg("DNS intercept: pf shutdown complete") return nil } @@ -598,6 +560,14 @@ func (p *prog) removePFAnchorReference() error { return fmt.Errorf("failed to dump running filter rules: %w (output: %s)", err, strings.TrimSpace(string(filterOut))) } + // Nothing of ours in the running ruleset means nothing to remove - and the reload + // below would reset system-wide pf options on the way past. Startup rollback reaches + // here after failures that happen before the references are ever added. + if !pfAnchorReferencesPresent(string(natOut), string(filterOut), pfAnchorName) { + mainLog.Load().Debug().Msg("DNS intercept: no anchor references in the running pf ruleset — nothing to remove") + return nil + } + // Filter and remove our lines. natLines := pfFilterRuleLines(string(natOut)) filterLines := pfFilterRuleLines(string(filterOut)) @@ -705,6 +675,8 @@ func discoverTunnelInterfaces() []string { return tunnels } +var discoverTunnelInterfacesForReconcile = discoverTunnelInterfaces + // dnsInterceptSupported reports whether DNS intercept mode is supported on this platform. func dnsInterceptSupported() bool { _, err := exec.LookPath("pfctl") @@ -761,6 +733,10 @@ func (p *prog) validateDNSIntercept() error { // // pf requires strict rule ordering: translation (rdr) BEFORE filtering (pass). func (p *prog) buildPFAnchorRules(vpnExemptions []vpnDNSExemption) string { + return p.buildPFAnchorRulesForTunnels(vpnExemptions, discoverTunnelInterfacesForReconcile()) +} + +func (p *prog) buildPFAnchorRulesForTunnels(vpnExemptions []vpnDNSExemption, tunnelIfaces []string) string { // Read the actual listener address from config. In intercept mode, ctrld may // be on a non-standard port (e.g., 127.0.0.1:5354) if mDNSResponder holds *:53. // The pf rdr rules must redirect to wherever ctrld is actually listening. @@ -950,7 +926,6 @@ func (p *prog) buildPFAnchorRules(vpnExemptions []vpnDNSExemption) string { // ruleset (not inside our anchor). Main ruleset rules are evaluated before ALL // anchors, making them impossible for another app to override without explicitly // removing them. See docs/dns-intercept-mode.md for details. - tunnelIfaces := discoverTunnelInterfaces() if len(tunnelIfaces) > 0 { rules.WriteString("# --- VPN/tunnel interface intercept rules ---\n") rules.WriteString("# Explicit intercept on tunnel interfaces prevents VPN apps from capturing\n") @@ -1030,7 +1005,7 @@ func (p *prog) buildPFAnchorRules(vpnExemptions []vpnDNSExemption) string { // verifyPFState checks that the pf ruleset is correctly configured after loading. // It verifies both the anchor references in the main ruleset and the rules within // our anchor. Failures are logged at ERROR level to make them impossible to miss. -func (p *prog) verifyPFState() { +func (p *prog) verifyPFState() bool { rdrAnchorRef := fmt.Sprintf("rdr-anchor \"%s\"", pfAnchorName) anchorRef := fmt.Sprintf("anchor \"%s\"", pfAnchorName) verified := true @@ -1038,6 +1013,7 @@ func (p *prog) verifyPFState() { // Check main ruleset for anchor references (rdr-anchor in translation rules). natOut, err := exec.Command("pfctl", "-sn").CombinedOutput() if err != nil { + p.pfBackoffResourceExhaustion(err, natOut, "verify NAT rules") mainLog.Load().Error().Err(err).Msg("DNS intercept: VERIFICATION FAILED — could not dump NAT rules") verified = false } else { @@ -1050,6 +1026,7 @@ func (p *prog) verifyPFState() { filterOut, err := exec.Command("pfctl", "-sr").CombinedOutput() if err != nil { + p.pfBackoffResourceExhaustion(err, filterOut, "verify filter rules") mainLog.Load().Error().Err(err).Msg("DNS intercept: VERIFICATION FAILED — could not dump filter rules") verified = false } else if !strings.Contains(string(filterOut), anchorRef) { @@ -1060,38 +1037,44 @@ func (p *prog) verifyPFState() { // Check our anchor has rules loaded. anchorFilter, err := exec.Command("pfctl", "-a", pfAnchorName, "-sr").CombinedOutput() if err != nil { + p.pfBackoffResourceExhaustion(err, anchorFilter, "verify anchor filter rules") mainLog.Load().Error().Err(err).Msg("DNS intercept: VERIFICATION FAILED — could not dump anchor filter rules") verified = false - } else if len(strings.TrimSpace(string(anchorFilter))) == 0 { + } else if pfRulesetEmpty(string(anchorFilter)) { mainLog.Load().Error().Msg("DNS intercept: VERIFICATION FAILED — anchor has no filter rules loaded") verified = false } anchorNat, err := exec.Command("pfctl", "-a", pfAnchorName, "-sn").CombinedOutput() if err != nil { + p.pfBackoffResourceExhaustion(err, anchorNat, "verify anchor NAT rules") mainLog.Load().Error().Err(err).Msg("DNS intercept: VERIFICATION FAILED — could not dump anchor NAT rules") verified = false - } else if len(strings.TrimSpace(string(anchorNat))) == 0 { + } else if pfRulesetEmpty(string(anchorNat)) { mainLog.Load().Error().Msg("DNS intercept: VERIFICATION FAILED — anchor has no NAT/redirect rules loaded") verified = false } - // Check that lo0 is not in the skip list — if it is, our rdr rules are dead. - optOut, err := exec.Command("pfctl", "-sO").CombinedOutput() - if err == nil { - for _, line := range strings.Split(string(optOut), "\n") { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "set skip on") && strings.Contains(line, "lo0") { - mainLog.Load().Error().Msg("DNS intercept: VERIFICATION FAILED — 'set skip on lo0' is active, rdr rules on loopback will not fire") - verified = false - break - } - } - } + // There is deliberately no "set skip on lo0" check here, even though such a skip + // would make our rdr rules on loopback dead. + // + // macOS offers no way to query it as text: pfctl(8) accepts -s nat, queue, rules, + // Anchors, states, Sources, info, References, labels, timeouts, memory, Tables, + // osfp, Interfaces and all - there is no options or skip modifier. Since this + // verification gates whether intercept mode may publish, a check that cannot + // succeed here would roll back every start. + // + // probePFIntercept() detects the condition functionally instead: it sends a real + // query from outside the _ctrld group and confirms the listener received the + // redirect, which cannot succeed if pf is bypassing loopback. An explicit check is + // follow-up work - pfctl(8) documents "-s Interfaces -v" as listing which + // interfaces have skip rules activated, but its output shape needs confirming on a + // host that actually has a skip configured before anything parses it. if verified { mainLog.Load().Info().Msg("DNS intercept: post-load verification passed — all pf rules confirmed active") } + return verified } // resetUpstreamTransports forces all DoH/DoT/DoQ upstreams to re-bootstrap their @@ -1122,32 +1105,164 @@ func (p *prog) resetUpstreamTransports() { } } +var ( + ensurePFAnchorReferenceForRestore = func(p *prog) error { + return p.ensurePFAnchorReference() + } + rebuildPFAnchorRulesForReconcile = func(p *prog, exemptions []vpnDNSExemption) ([]string, error) { + return p.rebuildPFAnchorRules(exemptions) + } + restorePFAnchorForReconcile = func(p *prog, reason string) pfAnchorCheckResult { + return p.restorePFAnchor(reason) + } + pfShutdownStateRevokedForTest = func() {} +) + +func (p *prog) rebuildPFAnchorRules(vpnExemptions []vpnDNSExemption) ([]string, error) { + tunnelIfaces := append([]string(nil), discoverTunnelInterfacesForReconcile()...) + rulesStr := p.buildPFAnchorRulesForTunnels(vpnExemptions, tunnelIfaces) + if err := os.WriteFile(pfAnchorFile, []byte(rulesStr), 0644); err != nil { + return nil, fmt.Errorf("write anchor file: %w", err) + } + out, err := exec.Command("pfctl", "-a", pfAnchorName, "-f", pfAnchorFile).CombinedOutput() + if err != nil { + return nil, fmt.Errorf("load rebuilt anchor: %w (output: %s)", err, strings.TrimSpace(string(out))) + } + flushPFStates() + return tunnelIfaces, nil +} + +func (p *prog) restorePFAnchor(reason string) pfAnchorCheckResult { + return p.restorePFAnchorWithTransportReset(reason, true) +} + +func (p *prog) restorePFAnchorWithTransportReset(reason string, resetTransports bool) pfAnchorCheckResult { + if p.dnsInterceptState == nil { + return pfAnchorCheckSkipped + } + var vpnExemptions []vpnDNSExemption + if p.vpnDNS != nil { + vpnExemptions = p.vpnDNS.CurrentExemptions() + } + mainLog.Load().Info().Str("reason", reason).Msg("DNS intercept: restoring pf anchor") + if err := ensurePFAnchorReferenceForRestore(p); err != nil { + p.pfBackoffResourceExhaustion(err, nil, "restore anchor references") + if resetTransports { + p.resetUpstreamTransports() + } + mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to restore anchor references") + return pfAnchorCheckFailed + } + + tunnelIfaces, err := rebuildPFAnchorRulesForReconcile(p, vpnExemptions) + if err != nil { + p.pfBackoffResourceExhaustion(err, nil, "rebuild anchor") + if resetTransports { + p.resetUpstreamTransports() + } + mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to rebuild anchor") + return pfAnchorCheckFailed + } + + // rebuildPFAnchorRules flushed PF states after the successful load. Reset + // transports even if the authoritative verification below fails, otherwise + // DoH can remain pinned to connections invalidated by that flush. + if resetTransports { + p.resetUpstreamTransports() + } + if !p.verifyPFState() { + mainLog.Load().Error().Str("reason", reason).Msg("DNS intercept: rebuilt anchor failed post-load verification") + return pfAnchorCheckFailed + } + if p.vpnDNS != nil { + p.vpnDNS.markInterceptExemptionsApplied(vpnExemptions) + } + p.commitPFReconcileState(tunnelIfaces) + p.pfLastRestoreTime.Store(time.Now().UnixMilli()) + mainLog.Load().Info().Str("reason", reason).Msg("DNS intercept: pf anchor restored successfully") + return pfAnchorCheckRestored +} + +func (p *prog) commitPFReconcileState(tunnelIfaces []string) { + p.mu.Lock() + defer p.mu.Unlock() + p.lastTunnelIfaces = append([]string(nil), tunnelIfaces...) + if p.hasPendingTunnelIfaces && stringSlicesEqual(p.pendingTunnelIfaces, tunnelIfaces) { + p.pendingTunnelIfaces = nil + p.hasPendingTunnelIfaces = false + } +} + +func (p *prog) clearPendingTunnelReconcile() { + p.mu.Lock() + p.pendingTunnelIfaces = nil + p.hasPendingTunnelIfaces = false + p.mu.Unlock() +} + +func (p *prog) hasPendingTunnelReconcile() bool { + p.mu.Lock() + defer p.mu.Unlock() + return p.hasPendingTunnelIfaces +} + +func (p *prog) reconcilePFAnchorForTunnelChange() pfAnchorCheckResult { + if p.pfExecBackoffActive() { + return pfAnchorCheckSkipped + } + if !p.pfEnsureRunning.CompareAndSwap(false, true) { + return pfAnchorCheckSkipped + } + defer p.pfEnsureRunning.Store(false) + return restorePFAnchorForReconcile(p, "tunnel interface change") +} + // checkTunnelInterfaceChanges compares the current set of active tunnel interfaces // against the last known set. If they differ (e.g., a VPN connected and created utun420), // it rebuilds and reloads the pf anchor rules to include interface-specific intercept // rules for the new interface. // -// Returns true if the anchor was rebuilt, false if no changes detected. -// This is called from the network change callback even when validInterfacesMap() -// reports no changes — because validInterfacesMap() only tracks physical hardware -// ports (en0, bridge0, etc.) and ignores tunnel interfaces (utun*, ipsec*, etc.). +// Returns true when a new tunnel state is observed or a pending state is +// successfully applied. Coalesced/deferred retries return false so ignored-event +// storms do not bypass the macOS VPN-DNS refresh limiter on every notification. func (p *prog) checkTunnelInterfaceChanges() bool { if p.dnsInterceptState == nil { return false } - current := discoverTunnelInterfaces() + current := append([]string(nil), discoverTunnelInterfacesForReconcile()...) p.mu.Lock() - prev := p.lastTunnelIfaces - changed := !stringSlicesEqual(prev, current) - if changed { - p.lastTunnelIfaces = current + prev := append([]string(nil), p.lastTunnelIfaces...) + stabilizing := p.pfStabilizing.Load() + newObservation := false + if p.hasPendingTunnelIfaces && stringSlicesEqual(p.pendingTunnelIfaces, current) { + if stabilizing { + p.mu.Unlock() + return false + } + if stringSlicesEqual(prev, current) { + p.pendingTunnelIfaces = nil + p.hasPendingTunnelIfaces = false + p.mu.Unlock() + return false + } + // The same pending state still differs from the applied baseline. Retry it + // after stabilization/failure instead of treating it as already applied. + } else { + if !p.hasPendingTunnelIfaces && stringSlicesEqual(prev, current) { + p.mu.Unlock() + return false + } + newObservation = true + p.pendingTunnelIfaces = append([]string(nil), current...) + p.hasPendingTunnelIfaces = true } p.mu.Unlock() - if !changed { - return false + if stringSlicesEqual(prev, current) { + p.clearPendingTunnelReconcile() + return true } // Detect NEW tunnel interfaces (not just any change). @@ -1164,39 +1279,26 @@ func (p *prog) checkTunnelInterfaceChanges() bool { } } + if stabilizing { + mainLog.Load().Debug().Msgf("DNS intercept: tunnel state changed during stabilization (was %v, now %v) — deferring anchor rebuild", prev, current) + return newObservation + } + if hasNewTunnel { - // A new VPN tunnel appeared. Enter stabilization mode — the VPN may be - // about to wipe our pf rules (Windscribe does this ~500ms after tunnel creation). - // We can't check pfAnchorIsWiped() here because the wipe hasn't happened yet. - // The stabilization loop will detect whether pf actually gets wiped: - // - If rules change (VPN touches pf): wait for stability, then restore. - // - If rules stay stable for the full wait (Tailscale): exit early and rebuild immediately. + // A new VPN tunnel appeared. The dedicated post-stabilization repair path + // rebuilds the anchor if no successful exemption update applies it first. p.pfStartStabilization() - return true + return newObservation } mainLog.Load().Info().Msgf("DNS intercept: tunnel interfaces changed (was %v, now %v) — rebuilding pf anchor rules", prev, current) - - // Rebuild anchor rules with the updated tunnel interface list. - // Pass current VPN DNS servers so exemptions are preserved for still-active VPNs. - var vpnExemptions []vpnDNSExemption - if p.vpnDNS != nil { - vpnExemptions = p.vpnDNS.CurrentExemptions() + result := p.reconcilePFAnchorForTunnelChange() + if result != pfAnchorCheckRestored { + // Keep the desired tunnel set pending. The next event, delayed check, or + // watchdog tick retries it without confusing pending with applied state. + mainLog.Load().Debug().Msgf("DNS intercept: tunnel rule rebuild deferred/failed (result: %d)", result) } - rulesStr := p.buildPFAnchorRules(vpnExemptions) - if err := os.WriteFile(pfAnchorFile, []byte(rulesStr), 0644); err != nil { - mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to write rebuilt anchor file") - return true - } - out, err := exec.Command("pfctl", "-a", pfAnchorName, "-f", pfAnchorFile).CombinedOutput() - if err != nil { - mainLog.Load().Error().Err(err).Msgf("DNS intercept: failed to reload rebuilt anchor (output: %s)", strings.TrimSpace(string(out))) - return true - } - - flushPFStates() // Clear stale states so new rules (incl. VPN DNS exemptions) take effect - mainLog.Load().Info().Msgf("DNS intercept: rebuilt pf anchor with %d tunnel interfaces", len(current)) - return true + return newObservation || result == pfAnchorCheckRestored } // stringSlicesEqual reports whether two string slices have the same elements in the same order. @@ -1216,11 +1318,9 @@ func stringSlicesEqual(a, b []string) bool { // until the VPN's ruleset stops changing. This prevents a death spiral where // ctrld and the VPN repeatedly overwrite each other's pf rules. func (p *prog) pfStartStabilization() { - if p.pfStabilizing.Load() { - // Already stabilizing — extending is handled by backoff. + if !p.pfStabilizing.CompareAndSwap(false, true) { return } - p.pfStabilizing.Store(true) multiplier := max(int(p.pfBackoffMultiplier.Load()), 1) baseStableTime := 6000 * time.Millisecond // 4 polls at 1.5s @@ -1245,9 +1345,17 @@ func (p *prog) pfStartStabilization() { // pfStabilizationLoop polls pfctl -sr hash until the ruleset is stable for the // required duration, then restores our anchor rules. func (p *prog) pfStabilizationLoop(ctx context.Context, stableRequired time.Duration) { + p.pfStabilizationLoopWithMaxWait(ctx, stableRequired, pfStabilizationMaxWait) +} + +func (p *prog) pfStabilizationLoopWithMaxWait(ctx context.Context, stableRequired, maxWaitDuration time.Duration) { defer p.pfStabilizing.Store(false) pollInterval := 1500 * time.Millisecond + pollTicker := time.NewTicker(pollInterval) + defer pollTicker.Stop() + maxWait := time.NewTimer(maxWaitDuration) + defer maxWait.Stop() var lastHash string stableSince := time.Time{} @@ -1258,12 +1366,21 @@ func (p *prog) pfStabilizationLoop(ctx context.Context, stableRequired time.Dura return case <-p.stopCh: return - case <-time.After(pollInterval): + case <-maxWait.C: + mainLog.Load().Warn().Dur("max_wait", maxWaitDuration). + Msg("DNS intercept: stabilization timed out — returning ownership to delayed/watchdog recovery") + p.scheduleDelayedRechecks() + return + case <-pollTicker.C: } + if p.pfExecBackoffActive() { + continue + } // Hash the current filter ruleset. out, err := exec.Command("pfctl", "-sr").CombinedOutput() if err != nil { + p.pfBackoffResourceExhaustion(err, out, "poll stabilization filter rules") continue } hash := fmt.Sprintf("%x", sha256.Sum256(out)) @@ -1282,65 +1399,65 @@ func (p *prog) pfStabilizationLoop(ctx context.Context, stableRequired time.Dura } if time.Since(stableSince) >= stableRequired { - // Stable long enough — restore our rules. - // Clear stabilizing flag BEFORE calling ensurePFAnchorActive so - // the guard inside that function doesn't suppress our restore. - p.pfStabilizing.Store(false) - mainLog.Load().Info().Msgf("DNS intercept: pf stable for %s — restoring anchor rules", stableRequired) - p.ensurePFAnchorActive() + // The active loop retains ownership until the dedicated post-stable + // repair finishes. It never re-enters stabilization recursively. + mainLog.Load().Info().Msgf("DNS intercept: pf stable for %s — reconciling anchor rules", stableRequired) + result := p.reconcilePFAnchorAfterStabilization() + if result != pfAnchorCheckRestored && result != pfAnchorCheckIntact { + p.scheduleDelayedRechecks() + } routes, domainlessServers, exemptions := p.refreshDNSAfterVPNSettle("pf_stabilized") if routes == 0 && domainlessServers == 0 && exemptions == 0 { p.scheduleDNSAfterVPNSettleRefresh("pf_stabilized_followup", pfAnchorRecheckDelayLong) } - p.pfLastRestoreTime.Store(time.Now().UnixMilli()) + if p.hasPendingTunnelReconcile() { + p.scheduleDelayedRechecks() + } return } } } -// ensurePFAnchorActive checks that our pf anchor references and rules are still -// present in the running ruleset. If anything is missing (e.g., another program -// like Windscribe desktop or macOS itself reloaded pf.conf), it restores them. -// -// Returns true if restoration was needed, false if everything was already intact. -// Called both on network changes (immediate) and by the periodic pfWatchdog. -func (p *prog) ensurePFAnchorActive() bool { +var runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { + return exec.Command("pfctl", args...).CombinedOutput() +} + +// ensurePFAnchorActive checks live PF state and returns an explicit outcome for +// callers that must distinguish intact, restored, deferred, skipped, and failed. +func (p *prog) ensurePFAnchorActive() pfAnchorCheckResult { + return p.ensurePFAnchorActiveWithPolicy(true, false) +} + +// reconcilePFAnchorAfterStabilization is owned by the active stabilization loop. +// It bypasses ordinary recent-wipe deferral and forces a rebuild only when tunnel +// state is still pending. Otherwise an intact anchor is left alone, avoiding a +// redundant global PF state flush after an earlier successful exemption update. +func (p *prog) reconcilePFAnchorAfterStabilization() pfAnchorCheckResult { + return p.ensurePFAnchorActiveWithPolicy(false, p.hasPendingTunnelReconcile()) +} + +func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebuild bool) pfAnchorCheckResult { if p.dnsInterceptState == nil { - return false + return pfAnchorCheckSkipped + } + // Ordinary callers must not compete with the loop that owns stabilization. + // Check before acquiring the singleflight guard so the post-stable owner is + // not blocked by a watchdog callback that would immediately skip. + if allowRecentWipeDeferral && p.pfStabilizing.Load() { + mainLog.Load().Debug().Msg("DNS intercept watchdog: suppressed — VPN stabilization in progress") + return pfAnchorCheckSkipped } if !p.pfEnsureRunning.CompareAndSwap(false, true) { mainLog.Load().Debug().Msg("DNS intercept watchdog: check already running, skipping duplicate") - return false + return pfAnchorCheckSkipped } defer p.pfEnsureRunning.Store(false) if p.pfExecBackoffActive() { - return false + return pfAnchorCheckSkipped } - - // While stabilizing (VPN connecting), suppress all restores. - // The stabilization loop will restore once pf settles. - if p.pfStabilizing.Load() { - mainLog.Load().Debug().Msg("DNS intercept watchdog: suppressed — VPN stabilization in progress") - return false - } - - // Check if our last restore was very recent and got wiped again. - // This indicates a VPN reconnect cycle — enter stabilization with backoff. - if lastRestore := p.pfLastRestoreTime.Load(); lastRestore > 0 { - elapsed := time.Since(time.UnixMilli(lastRestore)) - if elapsed < 10*time.Second { - // Rules were wiped within 10s of our last restore — VPN is fighting us. - p.pfBackoffMultiplier.Add(1) - mainLog.Load().Warn().Msgf("DNS intercept: rules wiped %s after restore — entering stabilization (backoff multiplier: %d)", - elapsed, p.pfBackoffMultiplier.Load()) - p.pfStartStabilization() - return false - } - // Rules survived >10s — reset backoff - if p.pfBackoffMultiplier.Load() > 0 { - p.pfBackoffMultiplier.Store(0) - } + if allowRecentWipeDeferral && p.pfStabilizing.Load() { + return pfAnchorCheckSkipped } rdrAnchorRef := fmt.Sprintf("rdr-anchor \"%s\"", pfAnchorName) @@ -1348,13 +1465,11 @@ func (p *prog) ensurePFAnchorActive() bool { needsRestore := false // Check 1: anchor references in the main ruleset. - natOut, err := exec.Command("pfctl", "-sn").CombinedOutput() + natOut, err := runPFAnchorCheckCommand("-sn") if err != nil { - if p.pfBackoffResourceExhaustion(err, natOut, "dump NAT rules") { - return false - } + p.pfBackoffResourceExhaustion(err, natOut, "dump NAT rules") mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump NAT rules") - return false + return pfAnchorCheckFailed } natStr := string(natOut) if !strings.Contains(natStr, rdrAnchorRef) { @@ -1363,13 +1478,11 @@ func (p *prog) ensurePFAnchorActive() bool { } if !needsRestore { - filterOut, err := exec.Command("pfctl", "-sr").CombinedOutput() + filterOut, err := runPFAnchorCheckCommand("-sr") if err != nil { - if p.pfBackoffResourceExhaustion(err, filterOut, "dump filter rules") { - return false - } + p.pfBackoffResourceExhaustion(err, filterOut, "dump filter rules") mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump filter rules") - return false + return pfAnchorCheckFailed } if !strings.Contains(string(filterOut), anchorRef) { mainLog.Load().Warn().Msg("DNS intercept watchdog: anchor reference missing from running filter rules") @@ -1383,98 +1496,88 @@ func (p *prog) ensurePFAnchorActive() bool { // Without rdr, route-to sends packets to lo0 but they never get redirected to 127.0.0.1:53, // causing an infinite packet loop on lo0 and complete DNS failure. if !needsRestore { - anchorFilter, err := exec.Command("pfctl", "-a", pfAnchorName, "-sr").CombinedOutput() - if err != nil || len(strings.TrimSpace(string(anchorFilter))) == 0 { - if p.pfBackoffResourceExhaustion(err, anchorFilter, "dump anchor filter rules") { - return false - } + anchorFilter, err := runPFAnchorCheckCommand("-a", pfAnchorName, "-sr") + if err != nil { + p.pfBackoffResourceExhaustion(err, anchorFilter, "dump anchor filter rules") + mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump anchor filter rules") + return pfAnchorCheckFailed + } + if pfRulesetEmpty(string(anchorFilter)) { mainLog.Load().Warn().Msg("DNS intercept watchdog: anchor has no filter rules — content was flushed") needsRestore = true } } if !needsRestore { - anchorNat, err := exec.Command("pfctl", "-a", pfAnchorName, "-sn").CombinedOutput() - if err != nil || len(strings.TrimSpace(string(anchorNat))) == 0 { - if p.pfBackoffResourceExhaustion(err, anchorNat, "dump anchor NAT rules") { - return false - } + anchorNat, err := runPFAnchorCheckCommand("-a", pfAnchorName, "-sn") + if err != nil { + p.pfBackoffResourceExhaustion(err, anchorNat, "dump anchor NAT rules") + mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump anchor NAT rules") + return pfAnchorCheckFailed + } + if pfRulesetEmpty(string(anchorNat)) { mainLog.Load().Warn().Msg("DNS intercept watchdog: anchor has no rdr rules — translation was flushed (will cause packet loop on lo0)") needsRestore = true } } - // Check 3: "set skip on lo0" — VPN apps (e.g., Windscribe) load a complete pf.conf - // with "set skip on { lo0 }" which disables ALL pf processing on loopback. - // Our entire interception mechanism (route-to lo0 + rdr on lo0) depends on lo0 being - // processed by pf. This check detects the skip and triggers a restore that removes it. - if !needsRestore { - optOut, err := exec.Command("pfctl", "-sO").CombinedOutput() - if err == nil { - optStr := string(optOut) - // Check if lo0 appears in any "set skip on" directive. - for _, line := range strings.Split(optStr, "\n") { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "set skip on") && strings.Contains(line, "lo0") { - mainLog.Load().Warn().Msg("DNS intercept watchdog: 'set skip on lo0' detected — loopback bypass breaks our rdr rules") - needsRestore = true - break - } - } + // There is deliberately no "set skip on lo0" check here - see the note in + // verifyPFState for why macOS cannot be asked for skip state. + // + // It matters more here than a missing check might suggest: anything that cannot + // conclude has to return pfAnchorCheckFailed, and the caller only runs + // probePFIntercept() and the forced-reload recovery on an intact result. A check + // that can never conclude would therefore keep this function from ever reporting + // intact, disabling the functional probe and the self-heal it drives. + // + // So the skip case is left to probePFIntercept(). An explicit check can return via + // "pfctl -s Interfaces -v" once its output shape is confirmed on a host that has a + // skip configured. + + // Only classify a recent event as a repeated wipe after the live ruleset + // actually proves that a reference or anchor rule is missing. Previously this + // timestamp check ran first, so every healthy callback for ten seconds after + // stabilization was mislabeled as another wipe. + if lastRestore := p.pfLastRestoreTime.Load(); lastRestore > 0 { + elapsed := time.Since(time.UnixMilli(lastRestore)) + if needsRestore && allowRecentWipeDeferral && elapsed < 10*time.Second { + p.pfBackoffMultiplier.Add(1) + mainLog.Load().Warn().Msgf("DNS intercept: rules wiped %s after restore — entering stabilization (backoff multiplier: %d)", + elapsed, p.pfBackoffMultiplier.Load()) + p.pfStartStabilization() + return pfAnchorCheckDeferred + } + if elapsed >= 10*time.Second && p.pfBackoffMultiplier.Load() > 0 { + p.pfBackoffMultiplier.Store(0) } } - if !needsRestore { + if !needsRestore && !forceRebuild { mainLog.Load().Debug().Msg("DNS intercept watchdog: pf anchor intact") - return false + return pfAnchorCheckIntact } - // Restore: re-inject anchor references into the main ruleset. - mainLog.Load().Info().Msg("DNS intercept watchdog: restoring pf anchor references") - if err := p.ensurePFAnchorReference(); err != nil { - p.pfBackoffResourceExhaustion(err, nil, "restore anchor references") - mainLog.Load().Error().Err(err).Msg("DNS intercept watchdog: failed to restore anchor references") - return true + reason := "missing pf anchor state" + if forceRebuild && !needsRestore { + reason = "post-stabilization rebuild" } + return restorePFAnchorForReconcile(p, reason) +} - // Restore: always rebuild anchor rules from scratch to ensure tunnel interface - // rules are up-to-date (VPN interfaces may have appeared/disappeared since the - // anchor file was last written). - mainLog.Load().Info().Msg("DNS intercept watchdog: rebuilding anchor rules with current network state") - var vpnExemptions []vpnDNSExemption - if p.vpnDNS != nil { - vpnExemptions = p.vpnDNS.CurrentExemptions() +// dnsInterceptIgnoredChangeReconcileDue reports whether an ignored macOS +// network delta should run the expensive leading-edge pf/VPN-DNS reconciliation. +// Tunnel changes bypass this decision in the caller. The periodic watchdog and +// coalesced delayed checks remain independent safety nets. +func (p *prog) dnsInterceptIgnoredChangeReconcileDue(now time.Time) bool { + nowMillis := now.UnixMilli() + for { + last := p.pfIgnoredChangeLastReconcile.Load() + if last > 0 && now.Sub(time.UnixMilli(last)) < pfIgnoredChangeReconcileInterval { + return false + } + if p.pfIgnoredChangeLastReconcile.CompareAndSwap(last, nowMillis) { + return true + } } - rulesStr := p.buildPFAnchorRules(vpnExemptions) - if err := os.WriteFile(pfAnchorFile, []byte(rulesStr), 0644); err != nil { - mainLog.Load().Error().Err(err).Msg("DNS intercept watchdog: failed to write anchor file") - } else if out, err := exec.Command("pfctl", "-a", pfAnchorName, "-f", pfAnchorFile).CombinedOutput(); err != nil { - p.pfBackoffResourceExhaustion(err, out, "load rebuilt anchor") - mainLog.Load().Error().Err(err).Msgf("DNS intercept watchdog: failed to load rebuilt anchor (output: %s)", strings.TrimSpace(string(out))) - } else { - flushPFStates() - mainLog.Load().Info().Msg("DNS intercept watchdog: rebuilt and loaded anchor rules") - } - - // Update tracked tunnel interfaces after rebuild so checkTunnelInterfaceChanges() - // has an accurate baseline for subsequent comparisons. - p.mu.Lock() - p.lastTunnelIfaces = discoverTunnelInterfaces() - p.mu.Unlock() - - // Verify the restoration worked. - p.verifyPFState() - - // Proactively reset upstream transports. When another program replaces the pf - // ruleset with "pfctl -f", it flushes the entire state table — killing all - // existing TCP connections including our DoH connections to upstream DNS servers. - // Without this reset, Go's http.Transport keeps trying dead connections until - // the 5s context deadline, causing a DNS blackout. Re-bootstrapping forces fresh - // TLS handshakes on the next query (~200ms vs ~5s recovery). - p.resetUpstreamTransports() - - p.pfLastRestoreTime.Store(time.Now().UnixMilli()) - mainLog.Load().Info().Msg("DNS intercept watchdog: pf anchor restored successfully") - return true } func (p *prog) scheduleDNSAfterVPNSettleRefresh(reason string, delay time.Duration) { @@ -1591,20 +1694,22 @@ func (p *prog) pfWatchdog() { return } - restored := p.ensurePFAnchorActive() - if !restored { - // Rules are intact in text form — also probe actual interception. - // This catches cases where rules survive but pf's internal translation - // state is corrupted (e.g., after a hypervisor reloads pf.conf). - if !p.pfStabilizing.Load() && !p.pfMonitorRunning.Load() { - if !p.probePFIntercept() { - mainLog.Load().Warn().Msg("DNS intercept watchdog: rules intact but probe FAILED — forcing full reload") - p.forceReloadPFMainRuleset() - restored = true // treat as a restore for logging + result := p.ensurePFAnchorActive() + if result == pfAnchorCheckIntact { + // Only an authoritative intact result may trigger the functional probe. + // Skipped/backoff/failed checks must not be treated as healthy text state. + if !p.pfMonitorRunning.Load() && !p.probePFIntercept() { + mainLog.Load().Warn().Msg("DNS intercept watchdog: rules intact but probe FAILED — forcing full reload") + if p.forceReloadPFMainRuleset() { + result = pfAnchorCheckRestored + } else { + result = pfAnchorCheckFailed } } + } - // Check if backoff should be reset. + if result == pfAnchorCheckIntact { + // Check if backoff should be reset only after an authoritative healthy check. if p.pfBackoffMultiplier.Load() > 0 && p.pfLastRestoreTime.Load() > 0 { elapsed := time.Since(time.UnixMilli(p.pfLastRestoreTime.Load())) if elapsed > 60*time.Second { @@ -1613,17 +1718,37 @@ func (p *prog) pfWatchdog() { } } } - if restored { + + switch result { + case pfAnchorCheckRestored: misses := consecutiveMisses.Add(1) if misses >= pfConsecutiveMissThreshold { mainLog.Load().Error().Msgf("DNS intercept watchdog: pf anchor has been missing for %d consecutive checks — something is persistently overwriting pf rules", misses) } else { mainLog.Load().Warn().Msgf("DNS intercept watchdog: pf anchor was missing and restored (consecutive misses: %d)", misses) } - } else { + case pfAnchorCheckIntact: if old := consecutiveMisses.Swap(0); old > 0 { mainLog.Load().Info().Msgf("DNS intercept watchdog: pf anchor stable again after %d consecutive restores", old) } + case pfAnchorCheckSkipped, pfAnchorCheckDeferred, pfAnchorCheckFailed: + // Preserve the previous authoritative state. Do not probe, count a + // repair, or reset misses from an indeterminate/deferred result. + } + + // Tunnel transitions and failed PF/WFP exemption updates are normally + // event-driven, but the final delayed attempt must remain retryable even + // if macOS emits no further event. + tunnelChanged := p.checkTunnelInterfaceChanges() + pendingExemptions := p.vpnDNS != nil && p.vpnDNS.interceptExemptionsPending() + // Stabilization owns pf while a VPN's ruleset is still settling, and a + // refresh rebuilds and reloads the anchor - exactly the mutual overwriting + // that stabilization exists to stop. Nothing is lost by deferring: the + // pending tunnel state and pending exemptions both survive, and the next + // tick retries once stabilization has finished. + if (tunnelChanged || pendingExemptions) && p.vpnDNS != nil && + !p.pfExecBackoffActive() && !p.pfStabilizing.Load() { + p.vpnDNS.Refresh(true) } } } @@ -1638,8 +1763,33 @@ func (p *prog) exemptVPNDNSServers(exemptions []vpnDNSExemption) error { if p.dnsInterceptState == nil { return fmt.Errorf("pf state not available") } + if p.pfExecBackoffActive() { + return fmt.Errorf("pf exemption update deferred during exec backoff") + } + // Defence in depth for the callers guarded at their own call sites: this function + // rewrites and reloads the whole anchor, which is the mutual overwriting that + // stabilization suppresses while a VPN's ruleset settles. + // + // This does defer handleRecovery's DHCP-nameserver exemption, which has no retry of + // its own - the trade is deliberate. Those exemptions are ad hoc: they are never + // registered with the vpnDNSManager, so the post-stabilization reconcile rebuilds + // the anchor from CurrentExemptions and drops them regardless. Letting them through + // mid-settle would buy a few seconds of captive-portal access at the cost of the + // collision stabilization exists to prevent. The error text surfaces in recovery's + // warning so the reason is visible rather than silent. + // + // Manager-driven callers lose nothing: appliedExemptions only advances on success, + // so interceptExemptionsPending stays true and the next tick retries. + if p.pfStabilizing.Load() { + return fmt.Errorf("pf exemption update deferred during stabilization") + } + if !p.pfEnsureRunning.CompareAndSwap(false, true) { + return fmt.Errorf("pf reconciliation already running") + } + defer p.pfEnsureRunning.Store(false) - rulesStr := p.buildPFAnchorRules(exemptions) + tunnelIfaces := append([]string(nil), discoverTunnelInterfacesForReconcile()...) + rulesStr := p.buildPFAnchorRulesForTunnels(exemptions, tunnelIfaces) if err := os.WriteFile(pfAnchorFile, []byte(rulesStr), 0644); err != nil { return fmt.Errorf("dns intercept: failed to rewrite pf anchor: %w", err) @@ -1647,6 +1797,7 @@ func (p *prog) exemptVPNDNSServers(exemptions []vpnDNSExemption) error { out, err := exec.Command("pfctl", "-a", pfAnchorName, "-f", pfAnchorFile).CombinedOutput() if err != nil { + p.pfBackoffResourceExhaustion(err, out, "load VPN DNS anchor") return fmt.Errorf("dns intercept: failed to reload pf anchor: %w (output: %s)", err, strings.TrimSpace(string(out))) } @@ -1658,8 +1809,12 @@ func (p *prog) exemptVPNDNSServers(exemptions []vpnDNSExemption) error { // Ensure the anchor reference still exists in the main ruleset. // Another program may have replaced the ruleset since we last checked. if err := p.ensurePFAnchorReference(); err != nil { - mainLog.Load().Warn().Err(err).Msg("DNS intercept: failed to verify anchor reference during VPN DNS update") + p.pfBackoffResourceExhaustion(err, nil, "restore VPN DNS anchor reference") + p.resetUpstreamTransports() + return fmt.Errorf("dns intercept: failed to restore anchor reference during VPN DNS update: %w", err) } + p.resetUpstreamTransports() + p.commitPFReconcileState(tunnelIfaces) // Count unique excluded interfaces for logging. excludedIfaces := make(map[string]bool) @@ -1830,7 +1985,15 @@ func (p *prog) pfInterceptMonitor() { // // The reload is safe for VPN interop because it reassembles from the current running // ruleset (pfctl -sr/-sn), preserving all existing anchors and rules. -func (p *prog) forceReloadPFMainRuleset() { +func (p *prog) forceReloadPFMainRuleset() bool { + if p.dnsInterceptState == nil || p.pfExecBackoffActive() { + return false + } + if !p.pfEnsureRunning.CompareAndSwap(false, true) { + return false + } + defer p.pfEnsureRunning.Store(false) + rdrAnchorRef := fmt.Sprintf("rdr-anchor \"%s\"", pfAnchorName) anchorRef := fmt.Sprintf("anchor \"%s\"", pfAnchorName) @@ -1838,13 +2001,13 @@ func (p *prog) forceReloadPFMainRuleset() { natOut, err := exec.Command("pfctl", "-sn").CombinedOutput() if err != nil { mainLog.Load().Error().Err(err).Msg("DNS intercept: force reload — failed to dump NAT rules") - return + return false } filterOut, err := exec.Command("pfctl", "-sr").CombinedOutput() if err != nil { mainLog.Load().Error().Err(err).Msg("DNS intercept: force reload — failed to dump filter rules") - return + return false } natLines := pfFilterRuleLines(string(natOut)) @@ -1868,14 +2031,10 @@ func (p *prog) forceReloadPFMainRuleset() { pureFilterLines = append([]string{anchorRef}, pureFilterLines...) } - // Clean pf options (remove "set skip on lo0" if present). - cleanedOptions, _ := pfGetCleanedOptions() - - // Reassemble in pf's required order: options → scrub → translation → filtering. + // Reassemble in pf's required order: scrub → translation → filtering. As in + // ensurePFAnchorReference, no options section is emitted; see the KNOWN GAP note + // there. var combined strings.Builder - if cleanedOptions != "" { - combined.WriteString(cleanedOptions) - } for _, line := range scrubLines { combined.WriteString(line + "\n") } @@ -1891,32 +2050,19 @@ func (p *prog) forceReloadPFMainRuleset() { out, err := cmd.CombinedOutput() if err != nil { mainLog.Load().Error().Err(err).Msgf("DNS intercept: force reload — pfctl -f - failed (output: %s)", strings.TrimSpace(string(out))) - return + return false } - - // Also reload the anchor rules to ensure they're fresh. - var vpnExemptions []vpnDNSExemption - if p.vpnDNS != nil { - vpnExemptions = p.vpnDNS.CurrentExemptions() - } - rulesStr := p.buildPFAnchorRules(vpnExemptions) - if err := os.WriteFile(pfAnchorFile, []byte(rulesStr), 0644); err != nil { - mainLog.Load().Error().Err(err).Msg("DNS intercept: force reload — failed to write anchor file") - } else if out, err := exec.Command("pfctl", "-a", pfAnchorName, "-f", pfAnchorFile).CombinedOutput(); err != nil { - mainLog.Load().Error().Err(err).Msgf("DNS intercept: force reload — failed to load anchor (output: %s)", strings.TrimSpace(string(out))) - } - - // Flush stale rdr/reply states after the forced ruleset + anchor reload. - // Without this, macOS can keep using pre-reload state and try to send - // redirected DNS replies directly from loopback to tunnel client addresses - // (for example, 127.0.0.1: -> 100.64.0.0/10), which fails with - // "sendmsg: can't assign requested address". - flushPFStates() - - // Reset upstream transports — pf reload/state flush kills existing DoH connections. + // A successful main ruleset reload invalidates PF state even if the + // subsequent anchor rebuild fails, so retire stale DoH transports now. p.resetUpstreamTransports() + if result := p.restorePFAnchorWithTransportReset("forced main ruleset reload", false); result != pfAnchorCheckRestored { + mainLog.Load().Error().Msg("DNS intercept: force reload — anchor rebuild failed") + return false + } + mainLog.Load().Info().Msg("DNS intercept: force reload — pf ruleset and anchor reloaded successfully") + return true } // osHealthcheckSuppressed always returns false on darwin — WFP loopback diff --git a/cmd/cli/dns_intercept_darwin_test.go b/cmd/cli/dns_intercept_darwin_test.go index 3ab810a..e375d08 100644 --- a/cmd/cli/dns_intercept_darwin_test.go +++ b/cmd/cli/dns_intercept_darwin_test.go @@ -3,9 +3,16 @@ package cli import ( + "context" "errors" + "fmt" + "os" + "path/filepath" "strings" "testing" + "time" + + "tailscale.com/net/netmon" "github.com/Control-D-Inc/ctrld" ) @@ -215,3 +222,609 @@ func TestIsResourceExhaustion(t *testing.T) { }) } } + +func stubPFAnchorCheckCommand(t *testing.T, outputs map[string]string) { + t.Helper() + original := runPFAnchorCheckCommand + runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { + key := strings.Join(args, " ") + output, ok := outputs[key] + if !ok { + t.Fatalf("unexpected pf anchor check command: pfctl %s", key) + } + return []byte(output), nil + } + t.Cleanup(func() { + runPFAnchorCheckCommand = original + }) +} + +func TestEnsurePFAnchorActiveRecentRestoreWithIntactRulesDoesNotStabilize(t *testing.T) { + stubPFAnchorCheckCommand(t, map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", + }) + + p := &prog{ + dnsInterceptState: &pfState{}, + stopCh: make(chan struct{}), + } + restoredAt := time.Now().Add(-time.Second).UnixMilli() + p.pfLastRestoreTime.Store(restoredAt) + + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckIntact { + t.Fatalf("intact rules result = %v, want intact", result) + } + if p.pfBackoffMultiplier.Load() != 0 { + t.Fatalf("intact rules incremented backoff to %d", p.pfBackoffMultiplier.Load()) + } + if p.pfStabilizing.Load() { + t.Fatal("intact rules must not enter stabilization") + } + if got := p.pfLastRestoreTime.Load(); got != restoredAt { + t.Fatalf("intact check changed restore timestamp: got %d, want %d", got, restoredAt) + } +} + +func TestEnsurePFAnchorActiveCheckFailureIsNotIntact(t *testing.T) { + original := runPFAnchorCheckCommand + runPFAnchorCheckCommand = func(...string) ([]byte, error) { + return nil, errors.New("pfctl unavailable") + } + t.Cleanup(func() { runPFAnchorCheckCommand = original }) + + p := &prog{dnsInterceptState: &pfState{}} + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckFailed { + t.Fatalf("failed PF inspection result = %v, want failed", result) + } +} + +func TestEnsurePFAnchorActiveRecentActualWipeStartsStabilization(t *testing.T) { + stubPFAnchorCheckCommand(t, map[string]string{ + "-sn": "", + }) + + stopCh := make(chan struct{}) + close(stopCh) + p := &prog{ + dnsInterceptState: &pfState{}, + stopCh: stopCh, + } + restoredAt := time.Now().Add(-time.Second).UnixMilli() + p.pfLastRestoreTime.Store(restoredAt) + + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckDeferred { + t.Fatalf("recent repeated wipe result = %v, want deferred", result) + } + if got := p.pfBackoffMultiplier.Load(); got != 1 { + t.Fatalf("recent repeated wipe backoff = %d, want 1", got) + } + if got := p.pfLastRestoreTime.Load(); got != restoredAt { + t.Fatalf("deferred restore changed restore timestamp: got %d, want %d", got, restoredAt) + } + deadline := time.Now().Add(time.Second) + for p.pfStabilizing.Load() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if p.pfStabilizing.Load() { + t.Fatal("stabilization goroutine did not observe closed stop channel") + } +} + +func TestDNSInterceptIgnoredChangeReconcileDue(t *testing.T) { + p := &prog{} + start := time.Unix(1_000_000, 0) + + if !p.dnsInterceptIgnoredChangeReconcileDue(start) { + t.Fatal("first ignored change must reconcile immediately") + } + if p.dnsInterceptIgnoredChangeReconcileDue(start.Add(pfIgnoredChangeReconcileInterval - time.Millisecond)) { + t.Fatal("ignored changes inside the interval must be coalesced") + } + if !p.dnsInterceptIgnoredChangeReconcileDue(start.Add(pfIgnoredChangeReconcileInterval)) { + t.Fatal("continuous ignored changes must reconcile again at the interval boundary") + } +} + +func TestIgnoredNetworkChangeCallbackBoundsWorkWithoutBurningStabilizedSlot(t *testing.T) { + outputs := map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", + } + originalCheck := runPFAnchorCheckCommand + pfChecks := 0 + runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { + key := strings.Join(args, " ") + output, ok := outputs[key] + if !ok { + t.Fatalf("unexpected pf anchor check command: pfctl %s", key) + } + if key == "-sn" { + pfChecks++ + } + return []byte(output), nil + } + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return nil } + t.Cleanup(func() { + runPFAnchorCheckCommand = originalCheck + discoverTunnelInterfacesForReconcile = originalDiscover + }) + + refreshes := 0 + vpnDNS := newVPNDNSManager(nil) + vpnDNS.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + refreshes++ + return nil + } + p := &prog{dnsInterceptState: &pfState{}, vpnDNS: vpnDNS} + t.Cleanup(func() { + p.pfDelayedRecheckMu.Lock() + defer p.pfDelayedRecheckMu.Unlock() + for _, timer := range p.pfDelayedRecheckTimers { + if timer != nil { + timer.Stop() + } + } + }) + + delta := &netmon.ChangeDelta{ + Old: &netmon.State{Interface: map[string]netmon.Interface{}}, + New: &netmon.State{Interface: map[string]netmon.Interface{}}, + } + start := time.Unix(1_000_000, 0) + p.handleDNSInterceptIgnoredNetworkChange(delta, start) + if pfChecks != 1 || refreshes != 1 { + t.Fatalf("first ignored delta work: pf checks=%d refreshes=%d, want 1 each", pfChecks, refreshes) + } + + p.pfStabilizing.Store(true) + p.handleDNSInterceptIgnoredNetworkChange(delta, start.Add(pfIgnoredChangeReconcileInterval)) + if pfChecks != 1 || refreshes != 1 { + t.Fatalf("stabilized delta ran leading reconciliation: pf checks=%d refreshes=%d", pfChecks, refreshes) + } + + p.pfStabilizing.Store(false) + resumeAt := start.Add(pfIgnoredChangeReconcileInterval + time.Millisecond) + p.handleDNSInterceptIgnoredNetworkChange(delta, resumeAt) + if pfChecks != 2 || refreshes != 2 { + t.Fatalf("first post-stabilization delta did not reconcile immediately: pf checks=%d refreshes=%d", pfChecks, refreshes) + } + + for i := 1; i <= 8; i++ { + p.handleDNSInterceptIgnoredNetworkChange(delta, resumeAt.Add(time.Duration(i)*100*time.Millisecond)) + } + if pfChecks != 2 || refreshes != 2 { + t.Fatalf("ignored delta burst was not coalesced: pf checks=%d refreshes=%d", pfChecks, refreshes) + } + + p.handleDNSInterceptIgnoredNetworkChange(delta, resumeAt.Add(pfIgnoredChangeReconcileInterval)) + if pfChecks != 3 || refreshes != 3 { + t.Fatalf("interval boundary did not reconcile: pf checks=%d refreshes=%d, want 3 each", pfChecks, refreshes) + } +} + +func TestRestorePFAnchorFailureIsNotReportedOrTimestamped(t *testing.T) { + originalReference := ensurePFAnchorReferenceForRestore + originalRebuild := rebuildPFAnchorRulesForReconcile + ensurePFAnchorReferenceForRestore = func(*prog) error { return nil } + rebuildPFAnchorRulesForReconcile = func(*prog, []vpnDNSExemption) ([]string, error) { + return nil, errors.New("pf load failed") + } + t.Cleanup(func() { + ensurePFAnchorReferenceForRestore = originalReference + rebuildPFAnchorRulesForReconcile = originalRebuild + }) + + p := &prog{dnsInterceptState: &pfState{}} + if result := p.restorePFAnchor("test"); result != pfAnchorCheckFailed { + t.Fatalf("failed restore result = %v, want failed", result) + } + if got := p.pfLastRestoreTime.Load(); got != 0 { + t.Fatalf("failed restore changed timestamp to %d", got) + } + if len(p.lastTunnelIfaces) != 0 { + t.Fatalf("failed restore committed tunnel state: %v", p.lastTunnelIfaces) + } +} + +func TestPFStabilizationTimeoutReturnsOwnershipToDelayedRecovery(t *testing.T) { + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + p.pfStabilizationLoopWithMaxWait(t.Context(), time.Hour, 25*time.Millisecond) + + if p.pfStabilizing.Load() { + t.Fatal("stabilization retained ownership after the maximum wait") + } + p.pfDelayedRecheckMu.Lock() + timers := append([]*time.Timer(nil), p.pfDelayedRecheckTimers...) + p.pfDelayedRecheckTimers = nil + p.pfDelayedRecheckMu.Unlock() + if len(timers) != 2 { + t.Fatalf("expected bounded timeout to schedule delayed recovery, got %d timers", len(timers)) + } + for _, timer := range timers { + timer.Stop() + } +} + +func TestStopDNSInterceptWaitsForInFlightPFMutation(t *testing.T) { + binDir := t.TempDir() + pfctlPath := filepath.Join(binDir, "pfctl") + if err := os.WriteFile(pfctlPath, []byte("#!/bin/sh\nexit 0\n"), 0755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+":"+os.Getenv("PATH")) + + anchorFile := filepath.Join(t.TempDir(), "anchor") + if err := os.WriteFile(anchorFile, []byte("rules"), 0600); err != nil { + t.Fatal(err) + } + p := &prog{dnsInterceptState: &pfState{anchorName: pfAnchorName, anchorFile: anchorFile}} + p.pfEnsureRunning.Store(true) + + revoked := make(chan struct{}) + originalRevokedHook := pfShutdownStateRevokedForTest + pfShutdownStateRevokedForTest = func() { close(revoked) } + t.Cleanup(func() { pfShutdownStateRevokedForTest = originalRevokedHook }) + + done := make(chan error, 1) + go func() { done <- p.stopDNSIntercept() }() + + select { + case <-revoked: + case <-time.After(time.Second): + t.Fatal("shutdown did not revoke PF lifecycle state before waiting") + } + select { + case err := <-done: + t.Fatalf("shutdown completed before in-flight PF owner released: %v", err) + case <-time.After(25 * time.Millisecond): + } + + p.pfEnsureRunning.Store(false) + if err := <-done; err != nil { + t.Fatalf("stopDNSIntercept() error: %v", err) + } + if _, err := os.Stat(anchorFile); !os.IsNotExist(err) { + t.Fatalf("anchor file remained after serialized shutdown: %v", err) + } +} + +func TestPostStabilizationReconcileRetainsOwnershipAndForcesRebuild(t *testing.T) { + stubPFAnchorCheckCommand(t, map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", + }) + originalRestore := restorePFAnchorForReconcile + calls := 0 + restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult { + calls++ + return pfAnchorCheckRestored + } + t.Cleanup(func() { restorePFAnchorForReconcile = originalRestore }) + + p := &prog{ + dnsInterceptState: &pfState{}, + pendingTunnelIfaces: []string{"utun9"}, + hasPendingTunnelIfaces: true, + } + p.pfStabilizing.Store(true) + if result := p.reconcilePFAnchorAfterStabilization(); result != pfAnchorCheckRestored { + t.Fatalf("post-stabilization result = %v, want restored", result) + } + if calls != 1 { + t.Fatalf("post-stabilization restore calls = %d, want 1", calls) + } + if !p.pfStabilizing.Load() { + t.Fatal("post-stabilization reconcile released loop ownership") + } + if p.pfBackoffMultiplier.Load() != 0 { + t.Fatalf("post-stabilization reconcile changed backoff to %d", p.pfBackoffMultiplier.Load()) + } +} + +func TestPostStabilizationIntactWithoutPendingAvoidsRebuild(t *testing.T) { + stubPFAnchorCheckCommand(t, map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", + }) + originalRestore := restorePFAnchorForReconcile + calls := 0 + restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult { + calls++ + return pfAnchorCheckRestored + } + t.Cleanup(func() { restorePFAnchorForReconcile = originalRestore }) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + if result := p.reconcilePFAnchorAfterStabilization(); result != pfAnchorCheckIntact { + t.Fatalf("post-stabilization result = %v, want intact", result) + } + if calls != 0 { + t.Fatalf("intact post-stabilization anchor rebuilt %d times", calls) + } + if !p.pfStabilizing.Load() { + t.Fatal("intact post-stabilization reconcile released loop ownership") + } +} + +func TestTunnelRemovalFailureRetriesBeforeCommittingBaseline(t *testing.T) { + originalDiscover := discoverTunnelInterfacesForReconcile + originalRestore := restorePFAnchorForReconcile + current := []string{} + discoverTunnelInterfacesForReconcile = func() []string { + return append([]string(nil), current...) + } + calls := 0 + restorePFAnchorForReconcile = func(p *prog, _ string) pfAnchorCheckResult { + calls++ + if calls == 1 { + return pfAnchorCheckFailed + } + p.commitPFReconcileState(current) + return pfAnchorCheckRestored + } + t.Cleanup(func() { + discoverTunnelInterfacesForReconcile = originalDiscover + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{ + dnsInterceptState: &pfState{}, + lastTunnelIfaces: []string{"utun7"}, + } + if !p.checkTunnelInterfaceChanges() { + t.Fatal("first tunnel removal was not detected") + } + if !stringSlicesEqual(p.lastTunnelIfaces, []string{"utun7"}) { + t.Fatalf("failed removal committed baseline: %v", p.lastTunnelIfaces) + } + if !p.hasPendingTunnelReconcile() { + t.Fatal("failed removal did not retain desired tunnel state for retry") + } + if !p.checkTunnelInterfaceChanges() { + t.Fatal("failed tunnel removal was not retried") + } + if len(p.lastTunnelIfaces) != 0 { + t.Fatalf("successful retry did not commit empty tunnel baseline: %v", p.lastTunnelIfaces) + } + if calls != 2 { + t.Fatalf("restore calls = %d, want 2", calls) + } +} + +func TestPendingTunnelStateRetriesAfterStabilization(t *testing.T) { + originalDiscover := discoverTunnelInterfacesForReconcile + originalRestore := restorePFAnchorForReconcile + current := []string{} + discoverTunnelInterfacesForReconcile = func() []string { return nil } + calls := 0 + restorePFAnchorForReconcile = func(p *prog, _ string) pfAnchorCheckResult { + calls++ + p.commitPFReconcileState(current) + return pfAnchorCheckRestored + } + t.Cleanup(func() { + discoverTunnelInterfacesForReconcile = originalDiscover + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{ + dnsInterceptState: &pfState{}, + lastTunnelIfaces: []string{"utun7"}, + pendingTunnelIfaces: current, + hasPendingTunnelIfaces: true, + } + if !p.checkTunnelInterfaceChanges() { + t.Fatal("pending tunnel removal was not retried after stabilization") + } + if calls != 1 || len(p.lastTunnelIfaces) != 0 || p.hasPendingTunnelReconcile() { + t.Fatalf("pending retry result: calls=%d baseline=%v pending=%v", calls, p.lastTunnelIfaces, p.hasPendingTunnelReconcile()) + } +} + +func TestTunnelReconcileHonorsPFExecBackoff(t *testing.T) { + originalDiscover := discoverTunnelInterfacesForReconcile + originalRestore := restorePFAnchorForReconcile + current := []string{} + discoverTunnelInterfacesForReconcile = func() []string { return nil } + calls := 0 + restorePFAnchorForReconcile = func(p *prog, _ string) pfAnchorCheckResult { + calls++ + p.commitPFReconcileState(current) + return pfAnchorCheckRestored + } + t.Cleanup(func() { + discoverTunnelInterfacesForReconcile = originalDiscover + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{dnsInterceptState: &pfState{}, lastTunnelIfaces: []string{"utun7"}} + p.pfExecBackoffUntil.Store(time.Now().Add(time.Minute).UnixMilli()) + if !p.checkTunnelInterfaceChanges() { + t.Fatal("tunnel removal was not detected during PF exec backoff") + } + if calls != 0 || !stringSlicesEqual(p.lastTunnelIfaces, []string{"utun7"}) { + t.Fatalf("PF restore ran during exec backoff: calls=%d baseline=%v", calls, p.lastTunnelIfaces) + } + if p.checkTunnelInterfaceChanges() { + t.Fatal("identical deferred tunnel retry bypassed the ignored-event limiter") + } + p.pfExecBackoffUntil.Store(0) + if !p.checkTunnelInterfaceChanges() || calls != 1 || len(p.lastTunnelIfaces) != 0 { + t.Fatalf("tunnel removal did not retry after backoff: calls=%d baseline=%v", calls, p.lastTunnelIfaces) + } +} + +func TestTunnelRapidReversalClearsUnappliedPendingState(t *testing.T) { + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return nil } + t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover }) + + p := &prog{ + dnsInterceptState: &pfState{}, + pendingTunnelIfaces: []string{"utun9"}, + hasPendingTunnelIfaces: true, + } + p.pfStabilizing.Store(true) + if !p.checkTunnelInterfaceChanges() { + t.Fatal("rapid tunnel reversal was not observed") + } + if p.hasPendingTunnelReconcile() || len(p.lastTunnelIfaces) != 0 { + t.Fatalf("rapid reversal left unapplied tunnel state: baseline=%v pending=%v", p.lastTunnelIfaces, p.hasPendingTunnelReconcile()) + } +} + +func TestTunnelAdditionIsCoalescedUntilSuccessfulRebuild(t *testing.T) { + originalDiscover := discoverTunnelInterfacesForReconcile + current := []string{"utun9"} + discoverTunnelInterfacesForReconcile = func() []string { + return append([]string(nil), current...) + } + t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover }) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + if !p.checkTunnelInterfaceChanges() { + t.Fatal("new tunnel was not detected") + } + if p.checkTunnelInterfaceChanges() { + t.Fatal("identical pending tunnel state was not coalesced") + } + if len(p.lastTunnelIfaces) != 0 { + t.Fatalf("pending tunnel was committed before PF rebuild: %v", p.lastTunnelIfaces) + } + if !p.hasPendingTunnelReconcile() { + t.Fatal("new tunnel was not retained as pending") + } + + p.commitPFReconcileState(current) + if !stringSlicesEqual(p.lastTunnelIfaces, current) { + t.Fatalf("successful rebuild baseline = %v, want %v", p.lastTunnelIfaces, current) + } + if p.hasPendingTunnelReconcile() { + t.Fatal("successful rebuild did not clear pending tunnel state") + } +} + +// TestVPNDNSRefreshDeferredWhileStabilizing covers the ignored network-change path, +// which can trigger a VPN DNS refresh from outside stabilization. +// +// A refresh rebuilds and reloads the pf anchor. Stabilization owns pf while a VPN's +// ruleset is still settling, so refreshing then is the mutual-overwrite collision +// stabilization exists to prevent - and these deltas arrive exactly when a VPN is +// coming up. Deferring is safe: checkTunnelInterfaceChanges keeps the observation +// pending, so the transition is retried afterwards. +// +// The watchdog tick carries the same guard for the same reason; it is not driven here +// because that would mean running its 30s loop. +func TestVPNDNSRefreshDeferredWhileStabilizing(t *testing.T) { + newProg := func(t *testing.T, refreshes *int, tunnels []string) *prog { + t.Helper() + outputs := map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", + } + originalCheck := runPFAnchorCheckCommand + runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { + output, ok := outputs[strings.Join(args, " ")] + if !ok { + return nil, fmt.Errorf("unexpected pf anchor check command") + } + return []byte(output), nil + } + // Discovery reports no tunnels. With a seeded baseline that is a removal, which + // checkTunnelInterfaceChanges reports as a change without touching pf while + // stabilizing - so this fixture never reaches a real pfctl write. + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return nil } + t.Cleanup(func() { + runPFAnchorCheckCommand = originalCheck + discoverTunnelInterfacesForReconcile = originalDiscover + }) + + vpnDNS := newVPNDNSManager(nil) + vpnDNS.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + *refreshes++ + return nil + } + p := &prog{dnsInterceptState: &pfState{}, vpnDNS: vpnDNS, lastTunnelIfaces: tunnels} + t.Cleanup(func() { + p.pfDelayedRecheckMu.Lock() + defer p.pfDelayedRecheckMu.Unlock() + for _, timer := range p.pfDelayedRecheckTimers { + if timer != nil { + timer.Stop() + } + } + }) + return p + } + delta := func() *netmon.ChangeDelta { + return &netmon.ChangeDelta{ + Old: &netmon.State{Interface: map[string]netmon.Interface{}}, + New: &netmon.State{Interface: map[string]netmon.Interface{}}, + } + } + + t.Run("tunnel change during stabilization does not refresh", func(t *testing.T) { + refreshes := 0 + // Seeded baseline plus empty discovery = a tunnel transition to report, so the + // refresh is eligible on everything except the stabilization guard. + p := newProg(t, &refreshes, []string{"utun9"}) + p.pfStabilizing.Store(true) + + p.handleDNSInterceptIgnoredNetworkChange(delta(), time.Unix(1_000_000, 0)) + + if refreshes != 0 { + t.Errorf("refreshed %d time(s) while stabilizing — that rebuilds the anchor under a settling VPN ruleset", refreshes) + } + }) + + t.Run("refresh still happens outside stabilization", func(t *testing.T) { + refreshes := 0 + p := newProg(t, &refreshes, nil) + + p.handleDNSInterceptIgnoredNetworkChange(delta(), time.Unix(1_000_000, 0)) + + if refreshes == 0 { + t.Error("no refresh outside stabilization — the guard must defer, not disable") + } + }) +} + +// TestExemptVPNDNSServersDeferredWhileStabilizing checks the mutation point itself, +// not just the call sites: any future caller reaching it during stabilization is +// refused before the anchor is rewritten. +// +// It returns before pfEnsureRunning is taken and before any pfctl work, so this drives +// the real function without touching the host's pf state. +func TestExemptVPNDNSServersDeferredWhileStabilizing(t *testing.T) { + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + + err := p.exemptVPNDNSServers([]vpnDNSExemption{{Server: "192.168.1.1"}}) + if err == nil { + t.Fatal("exemption applied while stabilizing — that rewrites the anchor under a settling VPN ruleset") + } + if !strings.Contains(err.Error(), "stabilization") { + t.Errorf("error does not name the reason: %v", err) + } + // The refusal must happen before the reconcile latch is claimed, or a deferral + // would lock out the reconcile that runs once stabilization ends. + if p.pfEnsureRunning.Load() { + t.Error("pfEnsureRunning was left held by a deferred exemption") + } +} diff --git a/cmd/cli/dns_intercept_ignored_change_windows_test.go b/cmd/cli/dns_intercept_ignored_change_windows_test.go new file mode 100644 index 0000000..8ea5d5e --- /dev/null +++ b/cmd/cli/dns_intercept_ignored_change_windows_test.go @@ -0,0 +1,20 @@ +//go:build windows + +package cli + +import ( + "testing" + "time" +) + +func TestDNSInterceptIgnoredChangeReconcileDueWindowsPreservesImmediateBehavior(t *testing.T) { + p := &prog{} + now := time.Now() + + if !p.dnsInterceptIgnoredChangeReconcileDue(now) { + t.Fatal("first ignored Windows change must reconcile immediately") + } + if !p.dnsInterceptIgnoredChangeReconcileDue(now) { + t.Fatal("Windows ignored changes must not inherit the macOS pf rate limit") + } +} diff --git a/cmd/cli/dns_intercept_others.go b/cmd/cli/dns_intercept_others.go index 50c7fd0..cf7d107 100644 --- a/cmd/cli/dns_intercept_others.go +++ b/cmd/cli/dns_intercept_others.go @@ -4,6 +4,7 @@ package cli import ( "fmt" + "time" ) // startDNSIntercept is not supported on this platform. @@ -23,8 +24,8 @@ func (p *prog) exemptVPNDNSServers(exemptions []vpnDNSExemption) error { } // ensurePFAnchorActive is a no-op on unsupported platforms. -func (p *prog) ensurePFAnchorActive() bool { - return false +func (p *prog) ensurePFAnchorActive() pfAnchorCheckResult { + return pfAnchorCheckSkipped } // checkTunnelInterfaceChanges is a no-op on unsupported platforms. @@ -32,6 +33,10 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return false } +func (p *prog) dnsInterceptIgnoredChangeReconcileDue(time.Time) bool { + return false +} + // scheduleDelayedRechecks is a no-op on unsupported platforms. func (p *prog) scheduleDelayedRechecks() {} diff --git a/cmd/cli/dns_intercept_settle.go b/cmd/cli/dns_intercept_settle.go index bc8a0d7..972a2ab 100644 --- a/cmd/cli/dns_intercept_settle.go +++ b/cmd/cli/dns_intercept_settle.go @@ -14,21 +14,9 @@ func (p *prog) refreshDNSAfterVPNSettle(reason string) (routes, domainlessServer return 0, 0, 0 } - beforeExemptions := p.vpnDNS.CurrentExemptions() routes, domainlessServers, exemptions = p.vpnDNS.RefreshRoutesOnly() - afterExemptions := p.vpnDNS.CurrentExemptions() - - if vpnDNSExemptionsEqual(beforeExemptions, afterExemptions) { - mainLog.Load().Info().Msgf("DNS intercept: post-settle VPN DNS route refresh completed — %d routes, %d domainless servers, %d exemptions (pf unchanged)", - routes, domainlessServers, exemptions) - return routes, domainlessServers, exemptions - } - - if err := p.exemptVPNDNSServers(afterExemptions); err != nil { - mainLog.Load().Warn().Err(err).Msg("DNS intercept: post-settle VPN DNS exemption update failed") - } else { - mainLog.Load().Info().Msgf("DNS intercept: post-settle VPN DNS exemptions changed — updated pf/WFP with %d exemptions", len(afterExemptions)) - } + mainLog.Load().Info().Msgf("DNS intercept: post-settle VPN DNS route refresh completed — %d routes, %d domainless servers, %d exemptions", + routes, domainlessServers, exemptions) return routes, domainlessServers, exemptions } diff --git a/cmd/cli/dns_intercept_settle_test.go b/cmd/cli/dns_intercept_settle_test.go index 925806c..36cc26f 100644 --- a/cmd/cli/dns_intercept_settle_test.go +++ b/cmd/cli/dns_intercept_settle_test.go @@ -43,7 +43,12 @@ func TestRefreshDNSAfterVPNSettleRefreshesOSResolverAndVPNRoutes(t *testing.T) { if got := p.vpnDNS.UpstreamForDomain("jira.cc.bmwgroup.net."); len(got) != 1 || got[0] != "10.102.26.10" { t.Fatalf("expected refreshed VPN DNS route, got %v", got) } - if len(exemptionUpdates) != 0 { - t.Fatalf("expected route-only refresh to avoid pf exemption updates, got %+v", exemptionUpdates) + if len(exemptionUpdates) != 1 || len(exemptionUpdates[0]) != 1 || exemptionUpdates[0][0].Server != "10.102.26.10" { + t.Fatalf("expected one serialized pf exemption update for the late VPN DNS server, got %+v", exemptionUpdates) + } + + p.refreshDNSAfterVPNSettle("test-repeat") + if len(exemptionUpdates) != 1 { + t.Fatalf("unchanged post-settle VPN DNS state rewrote pf: %+v", exemptionUpdates) } } diff --git a/cmd/cli/dns_intercept_windows.go b/cmd/cli/dns_intercept_windows.go index 5c3ecb3..9f99db4 100644 --- a/cmd/cli/dns_intercept_windows.go +++ b/cmd/cli/dns_intercept_windows.go @@ -1526,8 +1526,8 @@ func parseIPv4AsUint32(ipStr string) uint32 { } // ensurePFAnchorActive is a no-op on Windows (WFP handles intercept differently). -func (p *prog) ensurePFAnchorActive() bool { - return false +func (p *prog) ensurePFAnchorActive() pfAnchorCheckResult { + return pfAnchorCheckSkipped } // checkTunnelInterfaceChanges is a no-op on Windows (WFP handles intercept differently). @@ -1535,6 +1535,12 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return false } +// Windows preserves the existing immediate reconciliation behavior. NRPT/WFP +// and adapter DNS settling have different lifecycle requirements from macOS pf. +func (p *prog) dnsInterceptIgnoredChangeReconcileDue(time.Time) bool { + return true +} + // pfAnchorRecheckDelay is the delay for deferred pf anchor re-checks. // Defined here as a stub for Windows (referenced from dns_proxy.go). const pfAnchorRecheckDelay = 2 * time.Second diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index 9992906..6a64882 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -1541,64 +1541,13 @@ func (p *prog) monitorNetworkChanges() error { mainLog.Load().Debug().Msg("Ignoring interface change - no valid interfaces affected") // check if the default IPs are still on an interface that is up ValidateDefaultLocalIPsFromDelta(delta.New) - // Even minor interface changes can trigger macOS pf reloads — verify anchor. - // We check immediately AND schedule delayed re-checks (2s + 4s) to catch - // programs like Windscribe that modify pf rules and DNS settings - // asynchronously after the network change event fires. + // Minor interface changes can still accompany pf/WFP or VPN DNS changes. + // On macOS, bound the immediate full reconciliation so link-local-only + // notification storms do not run pfctl/scutil work for every event. + // Windows keeps the existing immediate behavior. Tunnel changes always + // bypass the macOS limit, and delayed checks provide a trailing refresh. if dnsIntercept && p.dnsInterceptState != nil { - if !p.pfStabilizing.Load() { - p.ensurePFAnchorActive() - } - // Check tunnel interfaces unconditionally — it decides internally - // whether to enter stabilization or rebuild immediately. - p.checkTunnelInterfaceChanges() - // Schedule delayed re-checks to catch async VPN teardown changes. - // These also refresh the OS resolver and VPN DNS routes. - p.scheduleDelayedRechecks() - - // Detect interface appearance/disappearance — hypervisors (Parallels, - // VMware, VirtualBox) reload pf when creating/destroying virtual network - // interfaces, which can corrupt pf's internal translation state. The rdr - // rules survive in text form (watchdog says "intact") but stop evaluating. - // Spawn an async monitor that probes pf interception with backoff and - // forces a full pf reload if broken. - if delta.Old != nil { - interfaceChanged := false - var changedIface string - for ifaceName := range delta.Old.Interface { - if ifaceName == "lo0" { - continue - } - if _, exists := delta.New.Interface[ifaceName]; !exists { - interfaceChanged = true - changedIface = ifaceName - break - } - } - if !interfaceChanged { - for ifaceName := range delta.New.Interface { - if ifaceName == "lo0" { - continue - } - if _, exists := delta.Old.Interface[ifaceName]; !exists { - interfaceChanged = true - changedIface = ifaceName - break - } - } - } - if interfaceChanged { - mainLog.Load().Info().Str("interface", changedIface). - Msg("DNS intercept: interface appeared/disappeared — starting interception probe monitor") - go p.pfInterceptMonitor() - } - } - } - // Refresh VPN DNS on tunnel interface changes (e.g., Tailscale connect/disconnect) - // even though the physical interface didn't change. Runs after tunnel checks - // so the pf anchor rebuild includes current VPN DNS exemptions. - if dnsIntercept && p.vpnDNS != nil { - p.vpnDNS.Refresh(true) + p.handleDNSInterceptIgnoredNetworkChange(delta, time.Now()) } return } @@ -1700,6 +1649,76 @@ func (p *prog) monitorNetworkChanges() error { return nil } +// handleDNSInterceptIgnoredNetworkChange runs the DNS-intercept work for a +// network delta that did not affect a usable interface. Keeping this path in a +// method lets tests exercise the callback wiring with synthetic deltas. +func (p *prog) handleDNSInterceptIgnoredNetworkChange(delta *netmon.ChangeDelta, now time.Time) { + reconcileNow := false + // Stabilization owns PF repair. Do not consume the next leading-edge slot + // until an ignored delta can actually perform the corresponding PF check. + if !p.pfStabilizing.Load() { + reconcileNow = p.dnsInterceptIgnoredChangeReconcileDue(now) + if reconcileNow { + p.ensurePFAnchorActive() + } + } + + // Check tunnel interfaces unconditionally — it decides internally whether + // to enter stabilization or rebuild immediately. + tunnelChanged := p.checkTunnelInterfaceChanges() + // Schedule delayed re-checks to catch async VPN teardown changes. These also + // refresh the OS resolver and VPN DNS routes. + p.scheduleDelayedRechecks() + + // Detect interface appearance/disappearance — hypervisors (Parallels, + // VMware, VirtualBox) reload pf when creating/destroying virtual network + // interfaces, which can corrupt pf's internal translation state. The rdr + // rules survive in text form (watchdog says "intact") but stop evaluating. + // Spawn an async monitor that probes pf interception with backoff and forces + // a full pf reload if broken. + if delta.Old != nil { + interfaceChanged := false + var changedIface string + for ifaceName := range delta.Old.Interface { + if ifaceName == "lo0" { + continue + } + if _, exists := delta.New.Interface[ifaceName]; !exists { + interfaceChanged = true + changedIface = ifaceName + break + } + } + if !interfaceChanged { + for ifaceName := range delta.New.Interface { + if ifaceName == "lo0" { + continue + } + if _, exists := delta.Old.Interface[ifaceName]; !exists { + interfaceChanged = true + changedIface = ifaceName + break + } + } + } + if interfaceChanged { + mainLog.Load().Info().Str("interface", changedIface). + Msg("DNS intercept: interface appeared/disappeared — starting interception probe monitor") + go p.pfInterceptMonitor() + } + } + + // Refresh VPN DNS immediately for real tunnel changes even when the periodic + // ignored-change reconciliation is currently rate-limited - but not while + // stabilization owns pf. A refresh rebuilds the anchor, and these deltas arrive + // exactly when a VPN is bringing its own ruleset up, which is the collision + // stabilization is there to prevent. checkTunnelInterfaceChanges keeps the + // observation pending, so the transition is retried rather than dropped. + if p.vpnDNS != nil && (reconcileNow || tunnelChanged) && !p.pfStabilizing.Load() { + p.vpnDNS.Refresh(true) + } +} + // interfaceStatesEqual compares two interface states func interfaceStatesEqual(a, b *netmon.Interface) bool { if a == nil || b == nil { diff --git a/cmd/cli/pf_ruleset.go b/cmd/cli/pf_ruleset.go new file mode 100644 index 0000000..73d5df6 --- /dev/null +++ b/cmd/cli/pf_ruleset.go @@ -0,0 +1,79 @@ +package cli + +import ( + "fmt" + "strings" +) + +// pfNoRulesMarker is what pfctl prints for a ruleset that contains nothing. +const pfNoRulesMarker = "(no rules)" + +// pfFilterRuleLines reduces pfctl output to the lines that are actually pf rules. +// +// It exists because every pfctl reader here uses CombinedOutput, and pfctl on macOS +// writes "No ALTQ support in kernel" and "ALTQ related functions disabled" to stderr on +// essentially every show command, so raw output is never a clean rule list. An empty +// ruleset can also report "(no rules)", which is a status line rather than a rule. +// +// Two consequences follow from getting this wrong, and both have bitten this file: +// callers that test the output for emptiness can never see empty, and callers that feed +// the lines back into "pfctl -f -" would splice non-rule text into a ruleset and have +// the reload rejected. +// +// Registry access and platform specifics stay elsewhere; this is pure string handling +// so it can be tested on any host. +func pfFilterRuleLines(output string) []string { + var rules []string + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + // pfctl stderr warnings, merged in by CombinedOutput. + if strings.Contains(line, "ALTQ") { + continue + } + // Status line for an empty ruleset, not a rule. + if line == pfNoRulesMarker { + continue + } + rules = append(rules, line) + } + return rules +} + +// pfRulesetEmpty reports whether pfctl output describes a ruleset with no rules. +// +// Use this rather than testing the raw output for emptiness: the merged stderr warnings +// described above mean a raw test is always false, so the condition it guards - an +// anchor whose contents were flushed - would never be detected. +func pfRulesetEmpty(output string) bool { + return len(pfFilterRuleLines(output)) == 0 +} + +// pfContainsRule checks if any line in the slice contains the given rule string. +// Uses substring matching because pfctl may append extra tokens like " all" to rules +// (e.g., `rdr-anchor "com.controld.ctrld" all`), which would fail exact matching. +func pfContainsRule(lines []string, rule string) bool { + for _, line := range lines { + if strings.Contains(line, rule) { + return true + } + } + return false +} + +// pfAnchorReferencesPresent reports whether ctrld's anchor references appear in the +// running ruleset, given the output of "pfctl -sn" and "pfctl -sr". +// +// Removing the references means reloading the entire main ruleset, and that reload +// carries no options section - so it resets system-wide pf options, including any +// third-party "set skip" directives. Doing that when there is nothing of ours to +// remove is pure collateral damage, which is what a startup rollback would otherwise +// cause after failing before the references were ever added. +func pfAnchorReferencesPresent(natOutput, filterOutput, anchorName string) bool { + rdrAnchorRef := fmt.Sprintf("rdr-anchor %q", anchorName) + anchorRef := fmt.Sprintf("anchor %q", anchorName) + return pfContainsRule(pfFilterRuleLines(natOutput), rdrAnchorRef) || + pfContainsRule(pfFilterRuleLines(filterOutput), anchorRef) +} diff --git a/cmd/cli/pf_ruleset_test.go b/cmd/cli/pf_ruleset_test.go new file mode 100644 index 0000000..d84252d --- /dev/null +++ b/cmd/cli/pf_ruleset_test.go @@ -0,0 +1,157 @@ +package cli + +import "testing" + +// altqNoise is what macOS pfctl writes to stderr on show commands. Because every +// pfctl reader here uses CombinedOutput, it lands in the middle of the data being +// parsed — which is why these helpers exist. +const altqNoise = "No ALTQ support in kernel\nALTQ related functions disabled\n" + +// TestPFRulesetEmpty is the regression guard for a flushed anchor being undetectable. +// +// The anchor-content checks in verifyPFState and ensurePFAnchorActive decide whether pf +// still has ctrld's rules. Testing the raw pfctl output for emptiness can never be true +// on macOS, because the merged ALTQ warnings are always present — so a genuinely flushed +// anchor reads as healthy and neither the startup gate nor the watchdog restore fires. +func TestPFRulesetEmpty(t *testing.T) { + tests := []struct { + name string + output string + want bool + }{ + { + // The case that was broken: nothing but merged stderr. + name: "only ALTQ warnings", + output: altqNoise, + want: true, + }, + { + // As captured on macOS 26.6 from "pfctl -sn -a com.controld.ctrld". + name: "ALTQ warnings plus the empty-ruleset marker", + output: altqNoise + "(no rules)\n", + want: true, + }, + { + name: "empty output", + output: "", + want: true, + }, + { + name: "whitespace only", + output: "\n \n\t\n", + want: true, + }, + { + name: "a real rdr rule behind the warnings", + output: altqNoise + "rdr on lo0 inet proto udp from any to ! 127.0.0.1 port = 53 -> 127.0.0.1 port 5354\n", + want: false, + }, + { + name: "a real filter rule behind the warnings", + output: altqNoise + "pass in quick on lo0 reply-to lo0 inet proto udp from any to 127.0.0.1 port = 5354\n", + want: false, + }, + { + name: "rule with no warnings at all", + output: "anchor \"com.controld.ctrld\" all\n", + want: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := pfRulesetEmpty(tc.output); got != tc.want { + t.Errorf("pfRulesetEmpty() = %v, want %v\noutput:\n%s", got, tc.want, tc.output) + } + }) + } +} + +// TestPFFilterRuleLines checks what survives filtering, since these lines are fed back +// into "pfctl -f -" by the ruleset-rebuild paths. Splicing a warning or the +// empty-ruleset marker into a ruleset would have the reload rejected outright. +func TestPFFilterRuleLines(t *testing.T) { + got := pfFilterRuleLines(altqNoise + "(no rules)\nrdr-anchor \"com.controld.ctrld\" all\n\nanchor \"com.controld.ctrld\" all\n") + want := []string{ + `rdr-anchor "com.controld.ctrld" all`, + `anchor "com.controld.ctrld" all`, + } + if len(got) != len(want) { + t.Fatalf("got %d lines %q, want %d %q", len(got), got, len(want), want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("line %d = %q, want %q", i, got[i], want[i]) + } + } + + if lines := pfFilterRuleLines(altqNoise); lines != nil { + t.Errorf("warnings alone must yield no rule lines, got %q", lines) + } +} + +// TestPFAnchorReferencesPresent guards when the main ruleset may be rewritten. +// +// Removing our anchor references means reloading the whole main ruleset, and that +// reload carries no options section — so it resets system-wide pf options, including +// third-party "set skip" directives. Startup rollback runs after failures that happen +// before the references were ever added, so without this check it would reset another +// application's pf options while removing nothing of ours. +func TestPFAnchorReferencesPresent(t *testing.T) { + const anchor = "com.controld.ctrld" + const otherAppRules = "scrub-anchor \"com.apple/*\" all fragment reassemble\nanchor \"com.vendor.vpn\" all\n" + + tests := []struct { + name string + nat string + filter string + want bool + }{ + { + name: "both references present", + nat: altqNoise + "rdr-anchor \"com.controld.ctrld\" all\n", + filter: altqNoise + "anchor \"com.controld.ctrld\" all\n", + want: true, + }, + { + // pfctl appends tokens like " all", so matching is substring-based. + name: "rdr reference only", + nat: altqNoise + "rdr-anchor \"com.controld.ctrld\" all\n", + filter: altqNoise + otherAppRules, + want: true, + }, + { + name: "filter reference only", + nat: altqNoise, + filter: altqNoise + "anchor \"com.controld.ctrld\"\n", + want: true, + }, + { + // The rollback case: we failed before adding anything, and another + // application owns the ruleset. Rewriting it would be pure collateral. + name: "someone else's ruleset, none of ours", + nat: altqNoise, + filter: altqNoise + otherAppRules, + want: false, + }, + { + name: "empty ruleset", + nat: altqNoise + "(no rules)\n", + filter: altqNoise + "(no rules)\n", + want: false, + }, + { + // A different anchor whose name merely contains ours must not count. + name: "another anchor with a similar name", + nat: altqNoise, + filter: altqNoise + "anchor \"com.vendor.controld-shim\" all\n", + want: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := pfAnchorReferencesPresent(tc.nat, tc.filter, anchor); got != tc.want { + t.Errorf("pfAnchorReferencesPresent() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 6272934..04b7ead 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -92,6 +92,16 @@ var svcConfig = &service.Config{ var useSystemdResolved = false +type pfAnchorCheckResult uint8 + +const ( + pfAnchorCheckSkipped pfAnchorCheckResult = iota + pfAnchorCheckIntact + pfAnchorCheckRestored + pfAnchorCheckDeferred + pfAnchorCheckFailed +) + type prog struct { mu sync.Mutex waitCh chan struct{} @@ -162,11 +172,12 @@ type prog struct { // On Windows: *wfpState, on macOS: *pfState, nil on other platforms. dnsInterceptState any - // lastTunnelIfaces tracks the set of active VPN/tunnel interfaces (utun*, ipsec*, etc.) - // discovered during the last pf anchor rule build. When the set changes (e.g., a VPN - // connects and creates utun420), we rebuild the pf anchor to add interface-specific - // intercept rules for the new interface. Protected by mu. - lastTunnelIfaces []string //lint:ignore U1000 used on darwin + // lastTunnelIfaces tracks the tunnel set included in the last successfully loaded + // pf anchor. Pending tunnel state is kept separately so failed PF work is retried + // instead of being mistaken for an applied update. Protected by mu. + lastTunnelIfaces []string //lint:ignore U1000 used on darwin + pendingTunnelIfaces []string //lint:ignore U1000 used on darwin + hasPendingTunnelIfaces bool //lint:ignore U1000 used on darwin // pfStabilizing is true while we're waiting for a VPN's pf ruleset to settle. // While true, the watchdog and network change callbacks do NOT restore our rules. @@ -189,10 +200,10 @@ type prog struct { // interception with exponential backoff and auto-heals if broken. pfMonitorRunning atomic.Bool //lint:ignore U1000 used on darwin - // pfEnsureRunning ensures only one pf anchor validation/restoration runs at a time. - // Network-change callbacks, delayed rechecks, and the periodic watchdog can all - // converge during macOS interface churn; concurrent pfctl/scutil exec storms can - // exhaust process/file limits and make the outage worse. + // pfEnsureRunning ensures only one pf validation or mutation runs at a time. + // Network callbacks, VPN exemption updates, delayed rechecks, probes, and the + // watchdog can converge during macOS churn; concurrent pfctl/scutil work can + // exhaust process/file limits or interleave anchor snapshots. pfEnsureRunning atomic.Bool //lint:ignore U1000 used on darwin // pfExecBackoffUntil suppresses pf anchor validation after pfctl/scutil execs @@ -204,6 +215,11 @@ type prog struct { pfDelayedRecheckMu sync.Mutex //lint:ignore U1000 used on darwin pfDelayedRecheckTimers []*time.Timer //lint:ignore U1000 used on darwin + // pfIgnoredChangeLastReconcile bounds immediate pf/VPN-DNS work for noisy + // ignored macOS network deltas. Tunnel changes bypass this limit, and the + // existing delayed checks provide a trailing reconciliation after churn. + pfIgnoredChangeLastReconcile atomic.Int64 //lint:ignore U1000 used on darwin + // pfProbeExpected holds the domain name of a pending pf interception probe. // When non-empty, the DNS handler checks incoming queries against this value // and signals pfProbeCh if matched. The probe verifies that pf's rdr rules @@ -839,6 +855,45 @@ func (p *prog) deAllocateIP() error { return nil } +// Seams for the intercept-start failure lifecycle. Choosing between the interface-DNS +// fallback and refusing it has side effects - restoring the host's DNS, then +// terminating - which a test has to observe without reconfiguring the host or exiting +// the test binary. The intercept start itself is indirected for the same reason: it is +// the real platform interceptor, which on macOS mutates pf and on Windows installs an +// NRPT rule, so a test of what happens *after* it fails must not be the thing that +// runs it. +var ( + localResolverIPFn = router.LocalResolverIP + startDNSInterceptFn = (*prog).startDNSIntercept + setDnsForRunningIfaceFn = (*prog).setDnsForRunningIface + resetDNSFn = (*prog).resetDNS + refuseFallbackFatal = func(format string, v ...any) { + mainLog.Load().Fatal().Msgf(format, v...) + } +) + +// interfaceDNSFallbackViable reports whether the interface-DNS fallback can actually +// direct queries to ctrld's listener. +// +// Interface DNS names a resolver by IP and has no port field - true of macOS interface +// settings and of Windows NRPT rules - so pointing the system straight at a listener +// that did not bind :53 sends queries to whatever owns :53 instead, and that resolver's +// upstream is ctrld's address: a loop, not a fallback. +// +// A nil or portless listener is treated as viable: the port is resolved elsewhere and +// defaults to 53, so there is nothing to refuse yet. +// +// A non-53 listener is still viable where a local resolver owns :53 and forwards to +// ctrld's port. That is the arrangement on the router platforms with a dnsmasq of their +// own: ctrld writes "server=#", so the forward follows +// whatever port ctrld actually bound. setDNS then points the interface at that resolver +// rather than at the listener - see the lc.Port != 53 case there, which this mirrors. +// Refusing on port alone would turn a working configuration into a startup failure on +// those routers. +func interfaceDNSFallbackViable(lc *ctrld.ListenerConfig, localResolverIP string) bool { + return lc == nil || lc.Port == 0 || lc.Port == 53 || localResolverIP != "" +} + func (p *prog) setDNS() { setDnsOK := false defer func() { @@ -871,7 +926,30 @@ func (p *prog) setDNS() { // modifying interface DNS settings. This eliminates race conditions with VPN // software that also manages DNS. See issue #489. if dnsIntercept { - if err := p.startDNSIntercept(); err != nil { + if err := startDNSInterceptFn(p); err != nil { + // Interface DNS cannot express a port: macOS interface settings and Windows + // NRPT rules both name a resolver by IP alone. So it is only a usable + // fallback when the listener actually bound :53. When something else owns + // :53 - mDNSResponder on macOS, which is the whole reason the :5354 fallback + // exists - pointing the system at 127.0.0.1 hands queries to that other + // resolver, whose own upstream is now ctrld's address. That is a resolution + // loop, not degraded operation: a healthy ctrld listener nothing on the host + // can reach, no working DNS, and no recovery short of stopping the service. + // + // Refuse instead, after putting the host's own DNS back. A visible startup + // failure beats DNS that is broken by design, and it stops a fallback that + // cannot work from quietly undoing the fail-closed verification above. + if lc := cfg.FirstListener(); !interfaceDNSFallbackViable(lc, localResolverIPFn()) { + mainLog.Load().Error().Err(err).Msgf("DNS intercept mode failed with the listener on port %d", lc.Port) + // Leave the host resolvable: restore static settings or DHCP rather than + // exiting with an interface still pointed at a ctrld that is not serving. + resetDNSFn(p, false, true) + refuseFallbackFatal("Refusing to fall back to interface DNS: it cannot direct queries to %s:%d, which would leave this host with no working resolver. Free port 53 for ctrld, or resolve the intercept failure, then start again.", lc.IP, lc.Port) + // Unreachable in production - the line above exits - but returning + // explicitly keeps the refusal from depending on that, so nothing can + // fall through to installing the fallback this just rejected. + return + } mainLog.Load().Error().Err(err).Msg("DNS intercept mode failed — falling back to interface DNS settings") // Fall through to traditional setDNS behavior. } else { @@ -907,7 +985,7 @@ func (p *prog) setDNS() { ns = "127.0.0.1" case lc.Port != 53: ns = "127.0.0.1" - if resolver := router.LocalResolverIP(); resolver != "" { + if resolver := localResolverIPFn(); resolver != "" { ns = resolver } default: @@ -926,7 +1004,7 @@ func (p *prog) setDNS() { slices.Sort(nameservers) netIfaceName := "" - netIface := p.setDnsForRunningIface(nameservers) + netIface := setDnsForRunningIfaceFn(p, nameservers) if netIface != nil { netIfaceName = netIface.Name } diff --git a/cmd/cli/prog_intercept_fallback_test.go b/cmd/cli/prog_intercept_fallback_test.go new file mode 100644 index 0000000..2c5078c --- /dev/null +++ b/cmd/cli/prog_intercept_fallback_test.go @@ -0,0 +1,221 @@ +package cli + +import ( + "errors" + "fmt" + "net" + "slices" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +// TestInterfaceDNSFallbackViable covers when the interface-DNS fallback may be used +// after DNS intercept fails to start. +// +// The fallback names a resolver by IP with no port, so it can only reach a listener on +// :53. Taking it with the listener on a redirect-dependent port produced a total DNS +// outage on macOS: the interface points at 127.0.0.1, mDNSResponder answers there, and +// its upstream is ctrld's own address - a resolution loop with a healthy ctrld listener +// nothing can reach. Intercept startup refuses the fallback in that case rather than +// creating it. +func TestInterfaceDNSFallbackViable(t *testing.T) { + tests := []struct { + name string + lc *ctrld.ListenerConfig + localResolver string + want bool + }{ + { + name: "listener on 53 can be reached by interface DNS", + lc: &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 53}, + want: true, + }, + { + // The reported outage: no local resolver, so the :5354 fallback port + // cannot be expressed by interface DNS. + name: "listener on the fallback port cannot", + lc: &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 5354}, + want: false, + }, + { + name: "any other non-53 port cannot", + lc: &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 5300}, + want: false, + }, + { + // Router platforms with their own dnsmasq: it owns :53 and forwards to + // ctrld's port, so interface DNS reaches the listener through it. + // Refusing here would break a working EdgeOS/Firewalla setup. + name: "non-53 listener behind a forwarding local resolver", + lc: &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 5354}, + localResolver: "192.168.1.1", + want: true, + }, + { + // Port is resolved elsewhere and defaults to 53; nothing to refuse yet. + name: "unset port is not refused", + lc: &ctrld.ListenerConfig{IP: "127.0.0.1"}, + want: true, + }, + { + name: "no listener is not refused", + lc: nil, + want: true, + }, + { + // A non-loopback listener on 53 is still reachable by IP. + name: "non-loopback listener on 53", + lc: &ctrld.ListenerConfig{IP: "192.168.1.10", Port: 53}, + want: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := interfaceDNSFallbackViable(tc.lc, tc.localResolver); got != tc.want { + t.Errorf("interfaceDNSFallbackViable() = %v, want %v", got, tc.want) + } + }) + } +} + +// interceptFallbackHarness drives setDNS() through the intercept-start failure path and +// records the side effects that decide whether the host ends up with a working +// resolver. +// +// Every host-touching step is stubbed, including the intercept start itself: this test +// runs untagged on Linux, macOS and Windows runners, where the real startDNSIntercept +// would set up pf or install an NRPT rule on the machine running the tests. Stubbing it +// also makes the precondition deterministic - the failure under test is injected rather +// than depending on the runner denying a privileged operation. +type interceptFallbackHarness struct { + interceptCalls int + installedNameservers []string + installCalls int + resetCalls int + refusals []string +} + +func newInterceptFallbackHarness(t *testing.T, lc *ctrld.ListenerConfig) *interceptFallbackHarness { + t.Helper() + h := &interceptFallbackHarness{} + + origStart, origInstall := startDNSInterceptFn, setDnsForRunningIfaceFn + origReset, origFatal := resetDNSFn, refuseFallbackFatal + origResolver := localResolverIPFn + origCfg, origMode, origIntercept, origHard := cfg, interceptMode, dnsIntercept, hardIntercept + t.Cleanup(func() { + startDNSInterceptFn, setDnsForRunningIfaceFn = origStart, origInstall + resetDNSFn, refuseFallbackFatal = origReset, origFatal + localResolverIPFn = origResolver + cfg, interceptMode, dnsIntercept, hardIntercept = origCfg, origMode, origIntercept, origHard + }) + + // Default to no local resolver: the desktop case. Router cases set it per test. + localResolverIPFn = func() string { return "" } + + // Never reach the real interceptor: it would configure pf on macOS and NRPT on + // Windows, on the machine running the tests. + startDNSInterceptFn = func(_ *prog) error { + h.interceptCalls++ + return errors.New("dns intercept: injected start failure") + } + setDnsForRunningIfaceFn = func(_ *prog, nameservers []string) *net.Interface { + h.installCalls++ + h.installedNameservers = nameservers + return nil + } + resetDNSFn = func(_ *prog, _ bool, _ bool) { h.resetCalls++ } + refuseFallbackFatal = func(format string, v ...any) { + h.refusals = append(h.refusals, fmt.Sprintf(format, v...)) + } + + cfg = ctrld.Config{} + cfg.Service.InterceptMode = "dns" + cfg.Listener = map[string]*ctrld.ListenerConfig{"0": lc} + watchdogOff := false + cfg.Service.DnsWatchdogEnabled = &watchdogOff + interceptMode, dnsIntercept, hardIntercept = "dns", false, false + return h +} + +func (h *interceptFallbackHarness) run(t *testing.T) { + t.Helper() + p := &prog{cfg: &cfg} + p.setDNS() +} + +// TestSetDNSRefusesUnreachableFallback is the behaviour test for the reported outage: it +// drives the real setDNS() lifecycle rather than the classification helper alone. +// +// Deleting or bypassing the guard in setDNS makes the first case fail, because interface +// DNS then gets installed pointing at a listener that cannot answer on :53 - which is +// the resolution loop this refuses to create. +func TestSetDNSRefusesUnreachableFallback(t *testing.T) { + t.Run("non-53 listener refuses the fallback and restores DNS", func(t *testing.T) { + h := newInterceptFallbackHarness(t, &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 5354}) + h.run(t) + + if h.interceptCalls != 1 { + t.Fatalf("intercept start called %d time(s) through the seam, want 1 — the real platform interceptor must never run here", h.interceptCalls) + } + if h.installCalls != 0 { + t.Errorf("interface DNS was installed %d time(s) for a listener on :5354 — that is the resolver loop", h.installCalls) + } + if h.resetCalls == 0 { + t.Error("host DNS was not restored before refusing, leaving the interface pointed at a ctrld that is not serving") + } + if len(h.refusals) == 0 { + t.Fatal("refusal was not surfaced: startup must fail loudly rather than silently skip the fallback") + } + if !strings.Contains(h.refusals[0], "5354") { + t.Errorf("refusal does not name the unreachable port: %q", h.refusals[0]) + } + }) + + t.Run("non-53 listener behind a local resolver still falls back", func(t *testing.T) { + // EdgeOS/Firewalla: dnsmasq owns :53 and forwards to ctrld's port, so the + // fallback works and must not be refused. setDNS points the interface at the + // resolver rather than at the listener. + h := newInterceptFallbackHarness(t, &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 5354}) + localResolverIPFn = func() string { return "192.168.1.1" } + h.run(t) + + if h.installCalls != 1 { + t.Errorf("interface DNS installed %d time(s), want 1: a forwarding local resolver makes the fallback usable", h.installCalls) + } + if len(h.refusals) != 0 { + t.Errorf("refused a fallback that a local resolver can serve: %v", h.refusals) + } + // Assert on membership, not on the exact set: setDNS appends platform-dependent + // entries beside the chosen nameserver - "::1" on Windows for the local IPv6 + // listener, the RFC1918 addresses where those listeners are needed. What matters + // is that the interface points at the resolver and not at the listener IP, whose + // port the interface cannot express. + if !slices.Contains(h.installedNameservers, "192.168.1.1") { + t.Errorf("nameservers = %v, want the local resolver among them so queries reach ctrld through it", h.installedNameservers) + } + if slices.Contains(h.installedNameservers, "127.0.0.1") { + t.Errorf("nameservers = %v, must not name the listener IP: interface DNS cannot reach it on :5354", h.installedNameservers) + } + }) + + t.Run("listener on 53 still reaches the interface-DNS fallback", func(t *testing.T) { + h := newInterceptFallbackHarness(t, &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 53}) + h.run(t) + + if h.interceptCalls != 1 { + t.Fatalf("intercept start called %d time(s) through the seam, want 1", h.interceptCalls) + } + if h.installCalls != 1 { + t.Errorf("interface DNS installed %d time(s), want 1: a listener on :53 is reachable, so the fallback must still apply", h.installCalls) + } + if len(h.refusals) != 0 { + t.Errorf("unexpected refusal for a reachable listener: %v", h.refusals) + } + if len(h.installedNameservers) == 0 { + t.Error("fallback installed no nameservers") + } + }) +} diff --git a/cmd/cli/vpn_dns.go b/cmd/cli/vpn_dns.go index 115ed80..691cf17 100644 --- a/cmd/cli/vpn_dns.go +++ b/cmd/cli/vpn_dns.go @@ -6,7 +6,6 @@ import ( "runtime" "strings" "sync" - "sync/atomic" "github.com/rs/zerolog" "tailscale.com/net/netmon" @@ -43,6 +42,9 @@ type vpnDNSManager struct { // as additional nameservers for queries that match split-DNS rules // (from ctrld config, AD domain, or VPN suffix config). domainlessServers []string + // appliedExemptions advances only after the platform PF/WFP callback succeeds. + // Keeping it separate from discovered configs makes failed rule updates retryable. + appliedExemptions []vpnDNSExemption // retainedAfterEmptyDiscovery means Windows reported an empty VPN DNS // snapshot once while previous VPN DNS state existed. We keep that last-known // state for one guarded refresh cycle because Windows can briefly report an @@ -51,9 +53,13 @@ type vpnDNSManager struct { // discoverVPNDNS is injected for tests so Refresh does not depend on the // runner host's real VPN/virtual adapter state. discoverVPNDNS func(context.Context) []ctrld.VPNDNSConfig - // refreshRunning keeps noisy network-change storms from running overlapping - // scutil/networksetup VPN DNS discovery work. - refreshRunning atomic.Bool + // refreshStateMu keeps noisy network-change storms from running overlapping + // full VPN DNS refreshes and retains one trailing refresh when an event arrives + // during discovery so the newest OS state is not lost. + refreshStateMu sync.Mutex + refreshRunning bool + refreshPending bool + discoveryMu sync.Mutex // Called when VPN DNS server list changes, to update intercept exemptions. onServersChanged vpnDNSExemptFunc } @@ -70,14 +76,39 @@ func newVPNDNSManager(exemptFunc vpnDNSExemptFunc) *vpnDNSManager { } // Refresh re-discovers VPN DNS configs from the OS. -// Called on network change events. +// Called on network change events. Overlapping calls are coalesced into one +// trailing refresh so a newer OS snapshot is never silently discarded. func (m *vpnDNSManager) Refresh(guardAgainstNoNameservers bool) { - logger := mainLog.Load() - if !m.refreshRunning.CompareAndSwap(false, true) { - logger.Debug().Msg("VPN DNS refresh already running, skipping duplicate") + m.refreshStateMu.Lock() + if m.refreshRunning { + m.refreshPending = true + m.refreshStateMu.Unlock() + mainLog.Load().Debug().Msg("VPN DNS refresh already running, coalescing trailing refresh") return } - defer m.refreshRunning.Store(false) + m.refreshRunning = true + m.refreshStateMu.Unlock() + + for { + m.refreshOnce(guardAgainstNoNameservers) + + m.refreshStateMu.Lock() + if m.refreshPending { + m.refreshPending = false + m.refreshStateMu.Unlock() + guardAgainstNoNameservers = true + continue + } + m.refreshRunning = false + m.refreshStateMu.Unlock() + return + } +} + +func (m *vpnDNSManager) refreshOnce(guardAgainstNoNameservers bool) { + logger := mainLog.Load() + m.discoveryMu.Lock() + defer m.discoveryMu.Unlock() logger.Debug().Msg("Refreshing VPN DNS configurations") discoverVPNDNS := m.discoverVPNDNS @@ -104,8 +135,6 @@ func (m *vpnDNSManager) Refresh(guardAgainstNoNameservers bool) { m.mu.Lock() defer m.mu.Unlock() - previousExemptions := m.currentExemptionsLocked() - if vpnDNSSettlingEnabled && len(configs) == 0 && guardAgainstNoNameservers && m.hasVPNDNSStateLocked() { if !m.retainedAfterEmptyDiscovery { exemptions := m.currentExemptionsLocked() @@ -116,6 +145,8 @@ func (m *vpnDNSManager) Refresh(guardAgainstNoNameservers bool) { if m.onServersChanged != nil { if err := m.onServersChanged(exemptions); err != nil { logger.Error().Err(err).Msg("Failed to re-apply retained VPN DNS exemptions") + } else { + m.appliedExemptions = append([]vpnDNSExemption(nil), exemptions...) } } return @@ -192,35 +223,37 @@ func (m *vpnDNSManager) Refresh(guardAgainstNoNameservers bool) { logger.Debug().Msgf("VPN DNS refresh completed: %d configs, %d routes, %d domainless servers, %d unique exemptions", len(m.configs), len(m.routes), len(m.domainlessServers), len(exemptions)) - // Update intercept rules to permit VPN DNS traffic only when the exemption set - // actually changes. Network-change events can fire repeatedly while macOS/VPN - // state is otherwise identical; rewriting pf for identical exemptions can feed - // a self-triggering network-change loop. Empty exemptions are still applied - // when they differ from the previous set, so stale VPN exemptions are cleared - // on disconnect. - m.updateInterceptExemptionsIfChanged(logger, previousExemptions, exemptions, "VPN DNS") + // Update intercept rules only when desired exemptions differ from the last + // successfully applied set. Failed PF/WFP callbacks remain retryable on the + // next refresh even when discovery returns the same VPN DNS state. + m.updateInterceptExemptionsIfChanged(logger, exemptions, "VPN DNS") } -func (m *vpnDNSManager) updateInterceptExemptionsIfChanged(logger *zerolog.Logger, before, after []vpnDNSExemption, reason string) { +func (m *vpnDNSManager) updateInterceptExemptionsIfChanged(logger *zerolog.Logger, desired []vpnDNSExemption, reason string) { if m.onServersChanged == nil { return } - if vpnDNSExemptionsEqual(before, after) { + if vpnDNSExemptionsEqual(m.appliedExemptions, desired) { logger.Debug().Msgf("VPN DNS exemptions unchanged after %s refresh; skipping intercept rule update", reason) return } - if err := m.onServersChanged(after); err != nil { + if err := m.onServersChanged(desired); err != nil { logger.Error().Err(err).Msg("Failed to update intercept exemptions for VPN DNS servers") + return } + m.appliedExemptions = append([]vpnDNSExemption(nil), desired...) } -// RefreshRoutesOnly re-discovers VPN DNS configs and updates only ctrld's -// in-memory split-DNS routes. It intentionally does not call onServersChanged, -// so it does not rewrite/reload pf/WFP rules. Use this for post-settle discovery -// checks where we only need to learn late-published VPN search domains. +// RefreshRoutesOnly re-discovers VPN DNS configs and updates ctrld's +// in-memory split-DNS routes. It applies intercept exemptions only when that set +// changes, while holding the shared discovery lane so a concurrent full refresh +// cannot commit a newer snapshot and then be overwritten by this one. func (m *vpnDNSManager) RefreshRoutesOnly() (routes, domainlessServers, exemptions int) { logger := mainLog.Load() + m.discoveryMu.Lock() + defer m.discoveryMu.Unlock() + logger.Debug().Msg("Refreshing VPN DNS route state only") discoverVPNDNS := m.discoverVPNDNS if discoverVPNDNS == nil { @@ -267,10 +300,26 @@ func (m *vpnDNSManager) RefreshRoutesOnly() (routes, domainlessServers, exemptio } } m.domainlessServers = domainless + currentExemptions := m.currentExemptionsLocked() logger.Debug().Msgf("VPN DNS route-only refresh completed: %d configs, %d routes, %d domainless servers, %d exemptions", - len(m.configs), len(m.routes), len(m.domainlessServers), len(m.currentExemptionsLocked())) - return len(m.routes), len(m.domainlessServers), len(m.currentExemptionsLocked()) + len(m.configs), len(m.routes), len(m.domainlessServers), len(currentExemptions)) + m.updateInterceptExemptionsIfChanged(logger, currentExemptions, "route-only VPN DNS") + return len(m.routes), len(m.domainlessServers), len(currentExemptions) +} + +func (m *vpnDNSManager) markInterceptExemptionsApplied(applied []vpnDNSExemption) { + m.mu.Lock() + defer m.mu.Unlock() + if vpnDNSExemptionsEqual(m.currentExemptionsLocked(), applied) { + m.appliedExemptions = append([]vpnDNSExemption(nil), applied...) + } +} + +func (m *vpnDNSManager) interceptExemptionsPending() bool { + m.mu.RLock() + defer m.mu.RUnlock() + return !vpnDNSExemptionsEqual(m.appliedExemptions, m.currentExemptionsLocked()) } func (m *vpnDNSManager) hasVPNDNSStateLocked() bool { diff --git a/cmd/cli/vpn_dns_test.go b/cmd/cli/vpn_dns_test.go index 846dc8e..94cf9bf 100644 --- a/cmd/cli/vpn_dns_test.go +++ b/cmd/cli/vpn_dns_test.go @@ -2,9 +2,11 @@ package cli import ( "context" + "errors" "sync" "sync/atomic" "testing" + "time" "github.com/Control-D-Inc/ctrld" ) @@ -16,7 +18,7 @@ func withVPNDNSSettlingEnabled(t *testing.T) { t.Cleanup(func() { vpnDNSSettlingEnabled = old }) } -func TestVPNDNSRefreshSkipsConcurrentDuplicate(t *testing.T) { +func TestVPNDNSRefreshCoalescesConcurrentTrailingRefresh(t *testing.T) { m := newVPNDNSManager(nil) started := make(chan struct{}) release := make(chan struct{}) @@ -25,9 +27,16 @@ func TestVPNDNSRefreshSkipsConcurrentDuplicate(t *testing.T) { var calls atomic.Int32 m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { - calls.Add(1) + call := calls.Add(1) once.Do(func() { close(started) }) <-release + if call == 2 { + return []ctrld.VPNDNSConfig{{ + InterfaceName: "utun-latest", + Servers: []string{"10.0.0.2"}, + Domains: []string{"latest.internal"}, + }} + } return nil } @@ -41,8 +50,11 @@ func TestVPNDNSRefreshSkipsConcurrentDuplicate(t *testing.T) { close(release) <-done - if calls.Load() != 1 { - t.Fatalf("expected overlapping refresh to be skipped, got %d discovery calls", calls.Load()) + if calls.Load() != 2 { + t.Fatalf("expected one active and one trailing discovery call, got %d", calls.Load()) + } + if got := m.Routes()["latest.internal"]; len(got) != 1 || got[0] != "10.0.0.2" { + t.Fatalf("trailing refresh did not publish latest OS snapshot: %v", got) } } @@ -76,7 +88,9 @@ func TestVPNDNSRefreshRetainsStateForOneGuardedEmptyDiscovery(t *testing.T) { func TestVPNDNSRefreshClearsOnSecondGuardedEmptyDiscovery(t *testing.T) { withVPNDNSSettlingEnabled(t) var gotExemptions []vpnDNSExemption + updates := 0 m := newVPNDNSManager(func(exemptions []vpnDNSExemption) error { + updates++ gotExemptions = exemptions return nil }) @@ -86,6 +100,7 @@ func TestVPNDNSRefreshClearsOnSecondGuardedEmptyDiscovery(t *testing.T) { Servers: []string{"10.25.37.21"}, }} m.domainlessServers = []string{"10.25.37.21"} + m.appliedExemptions = []vpnDNSExemption{{Server: "10.25.37.21", Interface: "Ethernet 6"}} m.retainedAfterEmptyDiscovery = true m.Refresh(true) @@ -93,8 +108,8 @@ func TestVPNDNSRefreshClearsOnSecondGuardedEmptyDiscovery(t *testing.T) { if got := m.DomainlessServers(); len(got) != 0 { t.Fatalf("expected domainless servers to be cleared on second empty discovery, got %v", got) } - if len(gotExemptions) != 0 { - t.Fatalf("expected empty exemptions after clearing stale state, got %v", gotExemptions) + if updates != 1 || len(gotExemptions) != 0 { + t.Fatalf("expected one empty exemption update after clearing stale state, calls=%d exemptions=%v", updates, gotExemptions) } if m.retainedAfterEmptyDiscovery { t.Fatal("expected retained empty-discovery marker to be cleared with stale state") @@ -126,6 +141,56 @@ func TestVPNDNSRefreshSkipsUnchangedInterceptExemptions(t *testing.T) { } } +func TestVPNDNSRefreshRetriesFailedInterceptExemptionUpdate(t *testing.T) { + attempts := 0 + m := newVPNDNSManager(func([]vpnDNSExemption) error { + attempts++ + if attempts == 1 { + return errors.New("pf update failed") + } + return nil + }) + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + return []ctrld.VPNDNSConfig{{ + InterfaceName: "utun-test", + Servers: []string{"10.102.26.10"}, + Domains: []string{"internal.test"}, + }} + } + + m.Refresh(true) + if !m.interceptExemptionsPending() { + t.Fatal("failed intercept exemption update was not retained for retry") + } + m.Refresh(true) + if m.interceptExemptionsPending() { + t.Fatal("successful intercept exemption retry did not advance applied state") + } + m.Refresh(true) + + if attempts != 2 { + t.Fatalf("intercept exemption update attempts = %d, want failed attempt plus one retry", attempts) + } + if len(m.appliedExemptions) != 1 || m.appliedExemptions[0].Server != "10.102.26.10" { + t.Fatalf("applied exemptions = %+v, want successful retry state", m.appliedExemptions) + } +} + +func TestVPNDNSMarkAppliedExemptionsRejectsStaleSnapshot(t *testing.T) { + m := newVPNDNSManager(nil) + m.configs = []ctrld.VPNDNSConfig{{InterfaceName: "utun-new", Servers: []string{"10.0.0.2"}}} + + m.markInterceptExemptionsApplied([]vpnDNSExemption{{Server: "10.0.0.1", Interface: "utun-old"}}) + if !m.interceptExemptionsPending() { + t.Fatal("stale PF snapshot incorrectly advanced applied exemptions") + } + + m.markInterceptExemptionsApplied([]vpnDNSExemption{{Server: "10.0.0.2", Interface: "utun-new"}}) + if m.interceptExemptionsPending() { + t.Fatal("current PF snapshot did not advance applied exemptions") + } +} + func TestVPNDNSTransportFailureSuppressesFallbackOnlyWhileRetainingState(t *testing.T) { withVPNDNSSettlingEnabled(t) m := newVPNDNSManager(nil) @@ -145,3 +210,89 @@ func TestVPNDNSTransportFailureSuppressesFallbackOnlyWhileRetainingState(t *test t.Fatal("expected reachable DNS response to clear retained empty-discovery state") } } + +func TestVPNDNSFullAndRouteOnlyDiscoveryAreSerialized(t *testing.T) { + var updateMu sync.Mutex + var exemptionUpdates []string + m := newVPNDNSManager(func(exemptions []vpnDNSExemption) error { + updateMu.Lock() + defer updateMu.Unlock() + if len(exemptions) == 0 { + exemptionUpdates = append(exemptionUpdates, "") + } else { + exemptionUpdates = append(exemptionUpdates, exemptions[0].Server) + } + return nil + }) + firstStarted := make(chan struct{}) + releaseFirst := make(chan struct{}) + secondStarted := make(chan struct{}) + var calls atomic.Int32 + + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + switch calls.Add(1) { + case 1: + close(firstStarted) + <-releaseFirst + return []ctrld.VPNDNSConfig{{ + InterfaceName: "utun-old", + Servers: []string{"10.0.0.1"}, + Domains: []string{"old.internal"}, + }} + case 2: + close(secondStarted) + return []ctrld.VPNDNSConfig{{ + InterfaceName: "utun-new", + Servers: []string{"10.0.0.2"}, + Domains: []string{"new.internal"}, + }} + default: + t.Fatalf("unexpected discovery call %d", calls.Load()) + return nil + } + } + + routesDone := make(chan struct{}) + go func() { + defer close(routesDone) + m.RefreshRoutesOnly() + }() + <-firstStarted + + fullDone := make(chan struct{}) + go func() { + defer close(fullDone) + m.Refresh(false) + }() + + select { + case <-secondStarted: + t.Fatal("full and route-only VPN DNS discovery overlapped") + case <-time.After(50 * time.Millisecond): + } + close(releaseFirst) + + select { + case <-routesDone: + case <-time.After(time.Second): + t.Fatal("route-only refresh did not finish") + } + select { + case <-fullDone: + case <-time.After(time.Second): + t.Fatal("full refresh did not finish") + } + + routes := m.Routes() + if _, ok := routes["old.internal"]; ok { + t.Fatalf("older route-only snapshot overwrote newer full refresh: %v", routes) + } + if got := routes["new.internal"]; len(got) != 1 || got[0] != "10.0.0.2" { + t.Fatalf("final VPN DNS routes = %v, want new.internal -> 10.0.0.2", routes) + } + updateMu.Lock() + defer updateMu.Unlock() + if len(exemptionUpdates) != 2 || exemptionUpdates[0] != "10.0.0.1" || exemptionUpdates[1] != "10.0.0.2" { + t.Fatalf("serialized exemption updates = %v, want old then new", exemptionUpdates) + } +} diff --git a/docs/pf-dns-intercept.md b/docs/pf-dns-intercept.md index f8cbb42..1055460 100644 --- a/docs/pf-dns-intercept.md +++ b/docs/pf-dns-intercept.md @@ -298,11 +298,22 @@ The full pf reload is VPN-safe: it reassembles from `pfctl -sr` + `pfctl -sn` ### What about `set skip on lo0`? Some pf.conf files include `set skip on lo0` which tells pf to skip ALL processing on loopback. **This would break our approach** since both the `rdr on lo0` and `pass in on lo0` rules would be skipped. -**Mitigation:** When injecting anchor references via `ensurePFAnchorReference()`, -we strip `lo0` from any `set skip on` directives before reloading. The watchdog -also checks for `set skip on lo0` and triggers a restore if detected. The -interception probe provides an additional safety net — if `set skip on lo0` gets -re-applied by another program, the probe will fail and trigger a full reload. +**Mitigation:** the interception probe. `probePFIntercept()` sends a real query from +outside the `_ctrld` group and confirms the listener received the redirect, which cannot +succeed while pf is bypassing loopback — so a skip on `lo0` shows up as a probe failure +and triggers a full reload. + +**Not implemented, contrary to earlier versions of this document:** ctrld does *not* +strip `lo0` from `set skip on` directives, and the watchdog does *not* inspect skip +state. Apple's `pfctl` offers no way to read it — `pfctl(8)` accepts `-s` nat, queue, +rules, Anchors, states, Sources, info, References, labels, timeouts, memory, Tables, +osfp, Interfaces, all, with no options or skip modifier — so text-based detection is not +available on macOS. + +Adding an explicit check is tracked as follow-up: `pfctl(8)` documents +`-s Interfaces -v` as additionally listing which interfaces have skip rules activated, +which is the query to build on once its output shape is confirmed on a host that has a +skip configured. ## Cleanup From 4d026d836c21084ffbfb2db0e12ac3b88e2c66ac Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Mon, 10 Aug 2026 07:48:17 +0000 Subject: [PATCH 02/16] windows: adopt GP-managed NRPT catch-all --- cmd/cli/dns_intercept_darwin.go | 15 +- .../dns_intercept_lifecycle_windows_test.go | 319 +++ cmd/cli/dns_intercept_others.go | 3 + cmd/cli/dns_intercept_windows.go | 1729 +++++++++++++++-- cmd/cli/dns_proxy.go | 8 +- cmd/cli/intercept_probe.go | 68 + cmd/cli/nrpt_external_gp.go | 63 + cmd/cli/nrpt_external_gp_test.go | 129 ++ cmd/cli/nrpt_external_gp_windows_test.go | 20 + cmd/cli/nrpt_handback_windows_test.go | 1309 +++++++++++++ cmd/cli/prog.go | 85 +- docs/dns-intercept-mode.md | 67 +- docs/wfp-dns-intercept.md | 106 +- 13 files changed, 3627 insertions(+), 294 deletions(-) create mode 100644 cmd/cli/dns_intercept_lifecycle_windows_test.go create mode 100644 cmd/cli/intercept_probe.go create mode 100644 cmd/cli/nrpt_external_gp.go create mode 100644 cmd/cli/nrpt_external_gp_test.go create mode 100644 cmd/cli/nrpt_external_gp_windows_test.go create mode 100644 cmd/cli/nrpt_handback_windows_test.go diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go index 76863b9..6b30941 100644 --- a/cmd/cli/dns_intercept_darwin.go +++ b/cmd/cli/dns_intercept_darwin.go @@ -19,6 +19,10 @@ import ( "github.com/Control-D-Inc/ctrld" ) +// skipInitialDNSReset is Windows-only; macOS keeps the normal adapter cleanup +// before installing its pf redirect. +func (p *prog) skipInitialDNSReset() bool { return false } + const ( // pfWatchdogInterval is how often the periodic pf watchdog checks // that our anchor references are still present in the running ruleset. @@ -1855,14 +1859,9 @@ func (p *prog) probePFIntercept() bool { // Generate unique probe domain probeID := fmt.Sprintf("_pf-probe-%x.%s", time.Now().UnixNano()&0xFFFFFFFF, pfProbeDomain) - // Register probe so DNS handler can detect and signal it - probeCh := make(chan struct{}, 1) - p.pfProbeExpected.Store(probeID) - p.pfProbeCh.Store(&probeCh) - defer func() { - p.pfProbeExpected.Store("") - p.pfProbeCh.Store((*chan struct{})(nil)) - }() + // Register this attempt's own domain: overlapping probes must not cancel each other. + probeCh, deregister := p.registerInterceptProbe(probeID) + defer deregister() // Build a minimal DNS query packet for the probe domain. // We use exec.Command to send from a subprocess with GID=0 (wheel), diff --git a/cmd/cli/dns_intercept_lifecycle_windows_test.go b/cmd/cli/dns_intercept_lifecycle_windows_test.go new file mode 100644 index 0000000..9abad5d --- /dev/null +++ b/cmd/cli/dns_intercept_lifecycle_windows_test.go @@ -0,0 +1,319 @@ +//go:build windows + +package cli + +import ( + "runtime" + "testing" + "time" +) + +// newInterceptTestProg returns a prog with a published intercept state, fake NRPT +// operations already installed, and no WFP engine (engineHandle 0). +// +// The fake is installed here, before anything can inspect registry state, and it is the +// safety boundary - not the empty wfpState. A zero-valued state has owner None, and +// shutdown's None branch sweeps orphaned ctrld rules, so an unfaked stopDNSIntercept would +// reach the production nrptCatchAllRuleExists / removeNRPTCatchAllRule / signalNRPTChange. +// On a host that has ctrld's deterministic key - a developer box, or a CI runner where +// ctrld is installed - that deletes live policy and forces a Group Policy refresh, a +// Dnscache paramchange and a cache flush. A green run on a clean runner proves nothing +// about that. +func newInterceptTestProg(t *testing.T) (*prog, *wfpState, *fakeNRPTOps) { + t.Helper() + f := fakeNRPTOpsForTest(t) + // Prove the fake is in effect before anything can inspect registry state. Asserting + // zero side effects afterwards cannot do that: an uninstalled fake reports zero + // whether it was consulted or bypassed. + requireFakeNRPTOpsInstalled(t, f) + state := &wfpState{stopCh: make(chan struct{}), listenerIP: "127.0.0.1"} + p := &prog{} + p.dnsInterceptState = state + return p, state, f +} + +// assertNoNRPTSideEffects fails when a lifecycle path wrote NRPT policy or signalled the +// DNS Client. Every test in this file exercises a guard that is supposed to stand down, so +// any registry write or signal here means the guard did not hold - and, without the fake, +// would have hit the host's real policy. +func assertNoNRPTSideEffects(t *testing.T, f *fakeNRPTOps) { + t.Helper() + add, remove, signal, _ := f.counts() + if add != 0 || remove != 0 || signal != 0 { + t.Errorf("addRule = %d, removeRule = %d, signal = %d, want 0/0/0: this path must not write NRPT policy", + add, remove, signal) + } + if flush := f.flushCount(); flush != 0 { + t.Errorf("flush calls = %d, want 0: this path must not flush the resolver cache", flush) + } +} + +// TestStopDNSInterceptRevokesBeforeTeardown pins the ordering the shutdown/monitor race +// depends on. Teardown deletes our WFP sublayer, and a missing sublayer is precisely what +// the health monitor treats as "our filters were wiped, rebuild everything". Were the +// state revoked only after teardown, a monitor tick inside that window would rebuild the +// intercept during shutdown. +func TestStopDNSInterceptRevokesBeforeTeardown(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + + if p.interceptStateRevoked(state) { + t.Fatal("a freshly published intercept state must not read as retired") + } + if err := p.stopDNSIntercept(); err != nil { + t.Fatalf("stopDNSIntercept() = %v", err) + } + if !p.interceptStateRevoked(state) { + t.Error("state still reads live after shutdown: the monitor and heal flows would keep writing host DNS state") + } + if p.dnsInterceptState != nil { + t.Error("dnsInterceptState survived shutdown") + } + if p.dnsInterceptStopRequested.Load() { + t.Error("stop-requested flag was left set; a later start would see a phantom shutdown") + } +} + +// TestRebuildDNSInterceptRefusedAfterShutdown is the regression test for the reported +// race: SCM stop runs resetDNS -> stopDNSIntercept while the health monitor is mid-tick, +// and the monitor then reaches the rebuild path before the process exits. The rebuild +// must refuse - completing it would re-add the NRPT catch-all and the WFP filters moments +// before ctrld disappears, leaving Windows resolving through a listener that is gone. +// +// That refusal is also what keeps this test safe on a real Windows host: a rebuild that +// did not refuse would run startDNSIntercept and write NRPT policy to the machine +// running the tests. +func TestRebuildDNSInterceptRefusedAfterShutdown(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + if err := p.stopDNSIntercept(); err != nil { + t.Fatalf("stopDNSIntercept() = %v", err) + } + + if got := p.rebuildDNSIntercept(state, "WFP sublayer missing during health check"); got != interceptRebuildRetired { + t.Fatalf("rebuildDNSIntercept() = %v, want interceptRebuildRetired - a post-shutdown rebuild resurrects DNS interception", got) + } + if p.dnsInterceptState != nil { + t.Error("rebuild published new intercept state after shutdown") + } +} + +// TestRebuildDNSInterceptRefusedForReplacedState covers the other stale-owner case: an +// earlier rebuild already replaced the state, so a goroutine still holding the old one +// must not tear down its successor. +func TestRebuildDNSInterceptRefusedForReplacedState(t *testing.T) { + p, old, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + current := &wfpState{stopCh: make(chan struct{}), listenerIP: "127.0.0.1"} + p.dnsInterceptState = current + + if got := p.rebuildDNSIntercept(old, "WFP sublayer missing during health check"); got != interceptRebuildRetired { + t.Fatalf("rebuildDNSIntercept() = %v, want interceptRebuildRetired for a superseded state", got) + } + if p.dnsInterceptState != any(current) { + t.Error("a superseded state's rebuild replaced the live intercept") + } + if p.interceptStateRevoked(current) { + t.Error("the live state was revoked by a superseded rebuild") + } +} + +// TestRepairMissingWFPStandsDownAfterShutdown checks the monitor's entry point. It must +// not even query WFP for a retired state - the sublayer it looks for is what teardown +// just deleted - and it must tell the monitor goroutine to exit. +func TestRepairMissingWFPStandsDownAfterShutdown(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + if err := p.stopDNSIntercept(); err != nil { + t.Fatalf("stopDNSIntercept() = %v", err) + } + // Set the handle only after teardown. A fake handle proves the revocation check + // comes first, but must never reach the real WFP calls in cleanupWFPFilters. + state.engineHandle = 1 + + if !p.repairMissingWFP(state) { + t.Error("repairMissingWFP() = false after shutdown; the health monitor would keep running for a dead intercept") + } + if p.dnsInterceptState != nil { + t.Error("repairMissingWFP rebuilt the intercept after shutdown") + } +} + +// TestPendingStopSignalsRevocation covers how a stop avoids waiting: while it is blocked +// on the lifecycle lock it must already read as revoked, so an in-flight NRPT heal +// abandons its probe backoff instead of making the service stop wait it out. A stop that +// waits too long is killed by the Service Control Manager, which cleans up nothing. +func TestPendingStopSignalsRevocation(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + + p.dnsInterceptMu.Lock() + stopped := make(chan struct{}) + go func() { + defer close(stopped) + _ = p.stopDNSIntercept() + }() + + // Wait for the stop to announce itself while it is blocked on the lock. + deadline := time.Now().Add(5 * time.Second) + for !p.dnsInterceptStopRequested.Load() { + if time.Now().After(deadline) { + p.dnsInterceptMu.Unlock() + <-stopped + t.Fatal("stop never announced itself before waiting for the lifecycle lock") + } + runtime.Gosched() + } + if !p.interceptStateRevoked(state) { + t.Error("a pending stop does not read as revoked; the heal flows would keep it waiting") + } + p.dnsInterceptMu.Unlock() + <-stopped + + if p.dnsInterceptState != nil { + t.Error("the pending stop did not tear down the intercept once it acquired the lock") + } +} + +// TestInterceptWaitAbandonsPromptlyOnPendingStop is the bound on how long a stop can be +// delayed by a recovery flow: the heal sequence's waits add up to tens of seconds, and +// each one must end as soon as a stop is pending. +func TestInterceptWaitAbandonsPromptlyOnPendingStop(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + p.dnsInterceptStopRequested.Store(true) + + start := time.Now() + if p.interceptWait(state, 30*time.Second) { + t.Fatal("interceptWait() = true with a stop pending; the caller would carry on writing host DNS state") + } + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Errorf("interceptWait took %v to notice a pending stop; shutdown would inherit that delay", elapsed) + } +} + +// TestInterceptWaitRunsToCompletionWhileLive guards the other direction: the cancellable +// wait must still actually wait, or the recovery flows lose their backoff. +func TestInterceptWaitRunsToCompletionWhileLive(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + + start := time.Now() + if !p.interceptWait(state, 250*time.Millisecond) { + t.Fatal("interceptWait() = false for a live intercept") + } + if elapsed := time.Since(start); elapsed < 250*time.Millisecond { + t.Errorf("interceptWait returned after %v, want at least 250ms", elapsed) + } +} + +// TestNRPTNeedsCtrldActivation covers the recovery gap that left a machine unfiltered +// until restart: a failed NRPT write clears ownership, and an owner-None tick used to do +// nothing at all, so nothing ever retried the write. +func TestNRPTNeedsCtrldActivation(t *testing.T) { + tests := []struct { + name string + owner nrptRuleOwner + ruleExists bool + want bool + }{ + { + // The reported hole: activation failed, ownership was cleared, and no + // other path re-arms it. In hard mode WFP keeps blocking DNS meanwhile. + name: "no owner retries the failed write", + owner: nrptRuleOwnerNone, + want: true, + }, + { + name: "no owner retries even if a rule is somehow present", + owner: nrptRuleOwnerNone, + ruleExists: true, + want: true, + }, + { + name: "ctrld-owned rule removed externally is re-added", + owner: nrptRuleOwnerCtrld, + want: true, + }, + { + name: "healthy ctrld-owned rule is left alone", + owner: nrptRuleOwnerCtrld, + ruleExists: true, + want: false, + }, + { + // Writing beside external policy would be ambiguous policy, not recovery. + name: "external policy is never overwritten", + owner: nrptRuleOwnerGroupPolicy, + want: false, + }, + { + name: "external policy is never overwritten even with a ctrld rule present", + owner: nrptRuleOwnerGroupPolicy, + ruleExists: true, + want: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := nrptNeedsCtrldActivation(tc.owner, tc.ruleExists); got != tc.want { + t.Errorf("nrptNeedsCtrldActivation(%v, %v) = %v, want %v", tc.owner, tc.ruleExists, got, tc.want) + } + }) + } +} + +// TestActivateCtrldNRPTFallbackRefusedAfterShutdown guards the worst leftover. A +// catch-all re-added after shutdown points every DNS query on the machine at a listener +// that no longer exists, so nothing resolves at all. Refusing early also keeps this test +// from writing NRPT policy on the machine running it. +func TestActivateCtrldNRPTFallbackRefusedAfterShutdown(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + if err := p.stopDNSIntercept(); err != nil { + t.Fatalf("stopDNSIntercept() = %v", err) + } + + if p.activateCtrldNRPTFallback(state, "ctrld-owned rule missing during health check") { + t.Error("activateCtrldNRPTFallback() = true after shutdown: the catch-all would outlive ctrld") + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerNone { + t.Errorf("NRPT owner = %v after a refused fallback, want nrptRuleOwnerNone", owner) + } +} + +// TestAllowHandbackAttemptRateLimits covers the throttle on testing an external +// catch-all. Each attempt takes ctrld's rule out of the way for a probe, so a rule that +// never routes would cost a brief DNS outage on every 30s health tick without this - in +// hard mode a window where WFP blocks DNS and nothing redirects it. +func TestHandbackThrottleIsPerRule(t *testing.T) { + state := &wfpState{stopCh: make(chan struct{})} + now := time.Now() + + if !state.handbackAllowed(now, "{GP-RULE}", nrptHandbackRetryInterval) { + t.Fatal("first handback attempt must be allowed") + } + // Checking alone must not spend the budget: a pre-probe can still abort the attempt + // without disturbing NRPT, and that must not cost the rule its next window. + if !state.handbackAllowed(now, "{GP-RULE}", nrptHandbackRetryInterval) { + t.Error("handbackAllowed must not consume the budget by itself") + } + + state.recordHandbackAttempt(now, "{GP-RULE}", nrptHandbackRetryInterval) + if state.handbackAllowed(now.Add(nrptHandbackRetryInterval-time.Second), "{GP-RULE}", nrptHandbackRetryInterval) { + t.Error("re-testing the same rule inside the interval must be suppressed") + } + // Group Policy alternating between two names must not erase either one's memory: + // with a single slot every swap costs another removal of the live rule. + if !state.handbackAllowed(now.Add(time.Second), "{OTHER-RULE}", nrptHandbackRetryInterval) { + t.Error("a different rule name means the administrator changed policy: test it now") + } + state.recordHandbackAttempt(now.Add(time.Second), "{OTHER-RULE}", nrptHandbackRetryInterval) + if state.handbackAllowed(now.Add(2*time.Second), "{GP-RULE}", nrptHandbackRetryInterval) { + t.Error("testing another rule must not clear the first rule's throttle") + } + + if !state.handbackAllowed(now.Add(2*nrptHandbackRetryInterval), "{GP-RULE}", nrptHandbackRetryInterval) { + t.Error("the same rule must be testable again after the interval") + } +} diff --git a/cmd/cli/dns_intercept_others.go b/cmd/cli/dns_intercept_others.go index cf7d107..6729f01 100644 --- a/cmd/cli/dns_intercept_others.go +++ b/cmd/cli/dns_intercept_others.go @@ -18,6 +18,9 @@ func (p *prog) stopDNSIntercept() error { return nil } +// skipInitialDNSReset is Windows-only; other platforms keep the normal reset. +func (p *prog) skipInitialDNSReset() bool { return false } + // exemptVPNDNSServers is a no-op on unsupported platforms. func (p *prog) exemptVPNDNSServers(exemptions []vpnDNSExemption) error { return nil diff --git a/cmd/cli/dns_intercept_windows.go b/cmd/cli/dns_intercept_windows.go index 9f99db4..7776303 100644 --- a/cmd/cli/dns_intercept_windows.go +++ b/cmd/cli/dns_intercept_windows.go @@ -9,6 +9,7 @@ import ( "net" "os/exec" "runtime" + "strings" "sync" "sync/atomic" "time" @@ -235,6 +236,14 @@ type fwpmAction0 struct { filterType windows.GUID // union: filterType or calloutKey } +type nrptRuleOwner uint8 + +const ( + nrptRuleOwnerNone nrptRuleOwner = iota + nrptRuleOwnerCtrld + nrptRuleOwnerGroupPolicy +) + // wfpState holds the state of the WFP DNS interception filters. // It tracks the engine handle and all filter IDs for cleanup on shutdown. // All filter IDs are stored so we can remove them individually without @@ -258,18 +267,20 @@ type wfpState struct { // Static permit filter IDs for RFC1918/CGNAT subnet ranges. // These allow VPN DNS servers on private IPs to work without dynamic exemptions. subnetPermitFilterIDs []uint64 - // nrptActive tracks whether the NRPT catch-all rule was successfully added. - // Used by stopDNSIntercept to know whether cleanup is needed. - nrptActive bool + // nrptOwner distinguishes a ctrld-created rule from a GP rule that ctrld is + // only observing. Shutdown and recovery must never delete the latter. + nrptOwner nrptRuleOwner + // externalGPRuleName is the GP child key currently routing the catch-all to + // listenerIP. It is diagnostic identity, not an ownership claim. + externalGPRuleName string // listenerIP is the actual IP address ctrld is listening on (e.g., "127.0.0.1" // or "127.0.0.2" on AD DC). Used by NRPT rule creation and health monitor to // ensure NRPT points to the correct address. listenerIP string // stopCh is used to shut down the NRPT health monitor goroutine. stopCh chan struct{} - // mu protects loopbackProtectActive, loopbackPermitIDs, and engineHandle - // from concurrent access between nrptProbeAndHeal (goroutine) and - // stopDNSIntercept / cleanupWFPFilters (main goroutine). + // mu protects NRPT ownership, externalGPRuleName, loopbackProtectActive, + // loopbackPermitIDs, and engineHandle from concurrent monitor/recovery/stop use. mu sync.Mutex // loopbackProtectActive is true when DNS mode has activated a minimal WFP // session to permit loopback DNS. This counters third-party WFP block filters @@ -281,6 +292,59 @@ type wfpState struct { // nrptRecoveryLimiter prevents repeated Windows policy/Dnscache signaling // when another agent keeps putting NRPT back into a broken state. nrptRecoveryLimiter nrptRecoveryLimiter + // handbackAttempts records, per external rule name, when ctrld last removed its own + // catch-all to test whether that rule routes on its own. Protected by mu. + // + // It is a map rather than one (rule, time) pair because Group Policy can alternate + // between two rule names: with a single slot each swap erases the memory of the other + // one, and every swap costs another removal of the live rule. + handbackAttempts map[string]time.Time +} + +// handbackAllowed reports whether ctrld may test external rule ruleName again, without +// recording anything. +// +// Each attempt takes ctrld's rule out of the way for a probe, so a rule that never routes +// would otherwise cost a brief DNS outage on every health tick - in hard mode a window +// where WFP blocks DNS with nothing redirecting it. A different rule name means the +// administrator changed policy, which is worth testing immediately; the same name is held +// off for minInterval. +func (s *wfpState) handbackAllowed(now time.Time, ruleName string, minInterval time.Duration) bool { + s.mu.Lock() + defer s.mu.Unlock() + last, ok := s.handbackAttempts[ruleName] + return !ok || now.Sub(last) >= minInterval +} + +// recordHandbackAttempt spends ruleName's budget. Callers record only once an attempt is +// actually about to disturb NRPT, so a cheap abort - a pre-probe that shows nothing is +// routing - does not cost the rule its next 15 minutes. +func (s *wfpState) recordHandbackAttempt(now time.Time, ruleName string, minInterval time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + if s.handbackAttempts == nil { + s.handbackAttempts = make(map[string]time.Time, 2) + } + // Drop entries whose window has passed, so a churning GP store cannot grow this map. + for rule, at := range s.handbackAttempts { + if now.Sub(at) >= minInterval { + delete(s.handbackAttempts, rule) + } + } + s.handbackAttempts[ruleName] = now +} + +func (s *wfpState) nrptPolicyOwner() (nrptRuleOwner, string) { + s.mu.Lock() + defer s.mu.Unlock() + return s.nrptOwner, s.externalGPRuleName +} + +func (s *wfpState) setNRPTPolicyOwner(owner nrptRuleOwner, externalGPRuleName string) { + s.mu.Lock() + defer s.mu.Unlock() + s.nrptOwner = owner + s.externalGPRuleName = externalGPRuleName } // Lazy-loaded WFP DLL procedures. @@ -328,8 +392,6 @@ const ( // from both locations, but on some machines (including stock Win11) it only // honors the direct path. This is the same path Add-DnsClientNrptRule uses. nrptDirectKey = `SYSTEM\CurrentControlSet\Services\Dnscache\Parameters\DnsPolicyConfig` - // nrptRuleName is the name of our specific rule key under the GP path. - nrptRuleName = `CtrldCatchAll` // nrptDirectRuleName is the key name for the direct service store path. // The DNS Client requires direct-path rules to use GUID-in-braces format. // Using a plain name like "CtrldCatchAll" makes the rule visible in @@ -339,6 +401,107 @@ const ( nrptDirectRuleName = `{B2E9A3C1-7F4D-4A8E-9D6B-5C1E0F3A2B8D}` ) +func (p *prog) nrptListenerIP() string { + listenerIP := "127.0.0.1" + if lc := p.cfg.FirstListener(); lc != nil && lc.IP != "" && lc.IP != "0.0.0.0" && lc.IP != "::" { + listenerIP = lc.IP + } + return listenerIP +} + +// skipInitialDNSReset is the first half of GP-rule adoption. postRun normally +// resets adapter DNS before setDNS starts intercept mode; doing that first would +// violate externally managed policy even if startDNSIntercept adopted the GP rule +// a few milliseconds later. This is only a read-only candidate check. The rule is +// not trusted until a DNS Client probe reaches the listener and the same child is +// re-read by startDNSIntercept. +func (p *prog) skipInitialDNSReset() bool { + mode := p.configuredInterceptMode() + if mode != "dns" && mode != "hard" { + return false + } + if ruleName := findMatchingGPNRPTRule(p.nrptListenerIP()); ruleName != "" { + mainLog.Load().Info().Str("rule", ruleName). + Msg("DNS intercept: matching GP-managed NRPT candidate found - preserving adapter DNS until functional verification") + return true + } + return false +} + +// findMatchingGPNRPTRule returns the first non-ctrld GP child that is exactly a +// catch-all for listenerIP. Multiple namespaces or nameservers are deliberately +// rejected: ctrld must not infer exclusive routing from a broader policy shape. +func findMatchingGPNRPTRule(listenerIP string) string { + parent, err := registry.OpenKey(registry.LOCAL_MACHINE, nrptBaseKey, registry.ENUMERATE_SUB_KEYS) + if err != nil { + return "" + } + names, err := parent.ReadSubKeyNames(-1) + parent.Close() + if err != nil { + return "" + } + for _, name := range names { + if gpNRPTRuleMatches(name, listenerIP) { + return name + } + } + return "" +} + +func gpNRPTRuleMatches(ruleName, listenerIP string) bool { + namespaces, dnsServers, ok := readGPNRPTRule(ruleName) + return ok && isMatchingGPNRPTRule(ruleName, namespaces, dnsServers, listenerIP) +} + +func readGPNRPTRule(ruleName string) ([]string, string, bool) { + if ruleName == "" || strings.EqualFold(ruleName, nrptRuleName) { + return nil, "", false + } + key, err := registry.OpenKey(registry.LOCAL_MACHINE, nrptBaseKey+`\`+ruleName, registry.QUERY_VALUE) + if err != nil { + return nil, "", false + } + defer key.Close() + namespaces, _, err := key.GetStringsValue("Name") + if err != nil { + return nil, "", false + } + dnsServers, _, err := key.GetStringValue("GenericDNSServers") + if err != nil { + // A malformed external catch-all is still authoritative enough to block + // ctrld from creating a second catch-all; it simply cannot be adopted. + dnsServers = "" + } + return namespaces, dnsServers, true +} + +// findConflictingGPCatchAll reports an administrator-owned catch-all that no +// longer targets ctrld. Adding another GP catch-all beside it would create the +// same ambiguous policy class as a competing local-store rule, so callers leave +// policy untouched and wait for the administrator to restore/remove it. +func findConflictingGPCatchAll(listenerIP string) (string, string) { + parent, err := registry.OpenKey(registry.LOCAL_MACHINE, nrptBaseKey, registry.ENUMERATE_SUB_KEYS) + if err != nil { + return "", "" + } + names, err := parent.ReadSubKeyNames(-1) + parent.Close() + if err != nil { + return "", "" + } + for _, name := range names { + namespaces, dnsServers, ok := readGPNRPTRule(name) + if !ok || !isExternalGPCatchAll(name, namespaces) { + continue + } + if !isMatchingGPNRPTRule(name, namespaces, dnsServers, listenerIP) { + return name, dnsServers + } + } + return "", "" +} + // addNRPTCatchAllRule creates an NRPT catch-all rule that directs all DNS queries // to the specified listener IP. // @@ -520,29 +683,55 @@ func nrptCatchAllRuleExists() bool { // // Uses RefreshPolicyEx(bMachine=TRUE, dwOptions=RP_FORCE=1) from userenv.dll. // See: https://learn.microsoft.com/en-us/windows/win32/api/userenv/nf-userenv-refreshpolicyex +// nrptSignalExecTimeout bounds every helper process the NRPT signalling path shells out +// to. These run while a transition holds nrptTransitionMu, and a service stop takes that +// same lock: "gpupdate /force" against a slow or unreachable domain controller can take +// tens of seconds, which is exactly the Service Control Manager timeout the locking exists +// to avoid. A signal that cannot finish in this window has already failed as a nudge. +const nrptSignalExecTimeout = 10 * time.Second + +// runBoundedNRPTExec runs one signalling helper with a hard deadline and reports its +// combined output. +func runBoundedNRPTExec(name string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), nrptSignalExecTimeout) + defer cancel() + out, err := exec.CommandContext(ctx, name, args...).CombinedOutput() + if ctx.Err() != nil { + mainLog.Load().Warn().Str("command", name).Dur("timeout", nrptSignalExecTimeout). + Msg("DNS intercept: NRPT signalling helper timed out and was killed") + } + return out, err +} + +func runGPUpdate() { + if out, err := runBoundedNRPTExec("gpupdate", "/target:computer", "/force"); err != nil { + mainLog.Load().Debug().Msgf("DNS intercept: gpupdate failed: %v: %s", err, string(out)) + } else { + mainLog.Load().Debug().Msg("DNS intercept: triggered GP refresh via gpupdate") + } +} + func refreshNRPTPolicy() { if err := userenvDLL.Load(); err != nil { mainLog.Load().Debug().Err(err).Msg("DNS intercept: userenv.dll not available, falling back to gpupdate") - if out, err := exec.Command("gpupdate", "/target:computer", "/force").CombinedOutput(); err != nil { - mainLog.Load().Debug().Msgf("DNS intercept: gpupdate failed: %v: %s", err, string(out)) - } else { - mainLog.Load().Debug().Msg("DNS intercept: triggered GP refresh via gpupdate") - } + runGPUpdate() return } if err := procRefreshPolicyEx.Find(); err != nil { mainLog.Load().Debug().Err(err).Msg("DNS intercept: RefreshPolicyEx not found, falling back to gpupdate") - exec.Command("gpupdate", "/target:computer", "/force").Run() + runGPUpdate() return } // RefreshPolicyEx(BOOL bMachine, DWORD dwOptions) - // bMachine=1 (TRUE) = refresh computer policy, dwOptions=1 (RP_FORCE) = force refresh + // bMachine=1 (TRUE) = refresh computer policy, dwOptions=1 (RP_FORCE) = force refresh. + // This one only asks the policy engine to refresh and returns; it does not wait for a + // domain controller, which is why it is preferred over gpupdate. ret, _, _ := procRefreshPolicyEx.Call(1, 1) if ret != 0 { mainLog.Load().Debug().Msg("DNS intercept: triggered machine GP refresh via RefreshPolicyEx") } else { mainLog.Load().Debug().Msg("DNS intercept: RefreshPolicyEx returned FALSE, falling back to gpupdate") - exec.Command("gpupdate", "/target:computer", "/force").Run() + runGPUpdate() } } @@ -564,7 +753,7 @@ func flushDNSCacheOnly() { } } } - if out, err := exec.Command("ipconfig", "/flushdns").CombinedOutput(); err != nil { + if out, err := runBoundedNRPTExec("ipconfig", "/flushdns"); err != nil { mainLog.Load().Debug().Msgf("DNS intercept: ipconfig /flushdns failed: %v: %s", err, string(out)) } else { mainLog.Load().Debug().Msg("DNS intercept: flushed DNS resolver cache via ipconfig /flushdns") @@ -591,13 +780,23 @@ func signalNRPTChange() { // the OS cannot reach those servers on port 53 — queries fail and fall back // to ctrld via the loopback address. func (p *prog) startDNSIntercept() error { - // Resolve the actual listener IP. On AD DC / Windows Server with a local DNS - // server, ctrld may have fallen back to 127.0.0.x:53 instead of 127.0.0.1:53. - // NRPT must point to whichever address ctrld is actually listening on. - listenerIP := "127.0.0.1" - if lc := p.cfg.FirstListener(); lc != nil && lc.IP != "" && lc.IP != "0.0.0.0" && lc.IP != "::" { - listenerIP = lc.IP - } else if lc != nil && (lc.IP == "0.0.0.0" || lc.IP == "::") { + p.dnsInterceptMu.Lock() + defer p.dnsInterceptMu.Unlock() + return p.startDNSInterceptLocked() +} + +// startDNSInterceptLocked is startDNSIntercept with p.dnsInterceptMu already held, so +// the rebuild path can make teardown and re-create one atomic transition. +// +// Nothing it calls may take p.dnsInterceptMu. It runs nrptProbeAndHeal synchronously, +// and that whole family - nrptProbeAndHeal, activateCtrldNRPTFallback, +// tryAdoptMatchingGPNRPT - stays lock-free on purpose and uses interceptStateRevoked +// instead: those flows wait seconds between probes, and holding the lifecycle lock +// across them would make a service stop wait just as long. +func (p *prog) startDNSInterceptLocked() error { + ops := p.nrptOps() + listenerIP := p.nrptListenerIP() + if lc := p.cfg.FirstListener(); lc != nil && (lc.IP == "0.0.0.0" || lc.IP == "::") { mainLog.Load().Warn().Str("configured_ip", lc.IP). Msg("DNS intercept: listener configured with wildcard IP, using 127.0.0.1 for NRPT rules") } @@ -606,35 +805,131 @@ func (p *prog) startDNSIntercept() error { stopCh: make(chan struct{}), listenerIP: listenerIP, } + // The probe and heal flows take state as an argument rather than reading the + // published field, so nothing is published until startup has fully succeeded. + // Publishing a provisional state would expose a half-built wfpState - no engine + // handle yet, filter IDs still being assigned - to exemptVPNDNSServers, which the + // VPN DNS manager can call at any time. - // Step 1: Add NRPT catch-all rule (both dns and hard modes). - // NRPT must succeed before proceeding with WFP in hard mode. mainLog.Load().Info().Msgf("DNS intercept: initializing (mode: %s)", interceptMode) logNRPTParentKeyState("pre-write") - // Empty parent key recovery: if the GP DnsPolicyConfig key exists but is - // empty, DNS Client enters GP mode and hides local rules. Delete empty - // parents first, then send one change signal so DNS Client drops stale state. - if cleanEmptyNRPTParent() { - signalNRPTChange() + // GP adoption is a two-part proof. Registry shape establishes ownership; the + // DNS Client probe establishes that the policy actually routes to this listener. + // Re-reading the same child after the probe prevents adopting a rule replaced + // during a concurrent Group Policy refresh. + externalProbeOK := false + if ruleName := ops.findGPRule(listenerIP); ruleName != "" { + // Adoption goes through the same handback transition the running service uses. + // A rule left behind by an earlier unclean exit points at this very listener, so + // a probe taken with it still installed proves nothing about the GP rule; the + // transition removes ctrld's keys first, and puts them back if the GP rule + // cannot carry DNS on its own. + switch p.nrptHandbackToExternal(state, ruleName, "startup GP-managed catch-all candidate") { + case nrptHandbackVerified: + externalProbeOK = true + mainLog.Load().Info().Str("rule", ruleName).Str("listener", listenerIP). + Msg("DNS intercept: adopted working GP-managed NRPT catch-all; ctrld will not modify NRPT policy") + case nrptHandbackUnverified: + mainLog.Load().Warn().Str("rule", ruleName).Str("listener", listenerIP). + Msg("DNS intercept: GP-managed NRPT catch-all is present but the probe did not reach ctrld; leaving external policy untouched") + case nrptHandbackKeptCtrld: + mainLog.Load().Warn().Str("rule", ruleName).Str("listener", listenerIP). + Msg("DNS intercept: GP-managed NRPT catch-all could not carry DNS alone; keeping the ctrld-owned rule from the previous run") + case nrptHandbackConflict: + // The candidate turned out to target another resolver. Startup then refuses to + // write a competing rule below, which is a hard startup failure by design. + mainLog.Load().Error().Str("rule", ruleName).Str("listener", listenerIP). + Msg("DNS intercept: GP catch-all does not target ctrld; refusing to write a competing NRPT rule") + case nrptHandbackAborted: + // Undecided, not decided: nothing was proved about the candidate and no + // ownership was recorded. Say so, because the fall-through below writes + // ctrld's rule, and an unlogged fall-through here is indistinguishable from + // "no external policy exists". + mainLog.Load().Warn().Str("rule", ruleName).Str("listener", listenerIP). + Msg("DNS intercept: GP-managed NRPT candidate could not be tested at startup; continuing without external ownership") + } } - if err := addNRPTCatchAllRule(listenerIP); err != nil { - return fmt.Errorf("dns intercept: failed to add NRPT catch-all rule: %w", err) + owner, _ := state.nrptPolicyOwner() + if owner == nrptRuleOwnerNone { + // No working external ownership contract exists. Preserve the current ctrld + // path unless another GP catch-all already owns the namespace; writing a + // second catch-all would create an ambiguous policy rather than recovery. + if gpCatchAllConflictBlocksFallback(state, "startup GP catch-all does not target ctrld") { + return fmt.Errorf("dns intercept: conflicting GP NRPT catch-all targets another resolver") + } + // A matching candidate that is still there means the handback above came back + // undecided - typically because the pre-probe ran while the DNS Client was still + // settling at boot. Give it one more pass before writing anything: the DNS Client + // has had the startup work since, and a decision here avoids writing beside an + // administrator rule that no probe has tested. + if ruleName := ops.findGPRule(listenerIP); ruleName != "" { + switch p.nrptHandbackToExternal(state, ruleName, "startup retry of an undecided GP candidate") { + case nrptHandbackVerified: + externalProbeOK = true + case nrptHandbackUnverified, nrptHandbackConflict, nrptHandbackKeptCtrld: + // Ownership is recorded by the transition (or ctrld's own rule was put + // back), so the write below is neither needed nor safe. + } + } } - logNRPTParentKeyState("post-write") - state.nrptActive = true - signalNRPTChange() - mainLog.Load().Info().Msgf("DNS intercept: NRPT catch-all rule active — all DNS queries directed to %s", listenerIP) - // Step 2: In hard mode, also set up WFP filters to block non-local DNS. + owner, _ = state.nrptPolicyOwner() + if owner == nrptRuleOwnerNone { + if ops.ruleExists() { + // A rule from an earlier run already points at this listener. Adopt it rather + // than writing a second time: with a GP child present, addNRPTCatchAllRule + // would also write ctrld's GP-path sibling. + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + mainLog.Load().Info().Str("listener", listenerIP). + Msg("DNS intercept: adopting the ctrld NRPT catch-all already present from an earlier run") + } else { + if ops.cleanParent() { + ops.signal() + } + if err := ops.addRule(listenerIP); err != nil { + return fmt.Errorf("dns intercept: failed to add NRPT catch-all rule: %w", err) + } + logNRPTParentKeyState("post-write") + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + ops.signal() + mainLog.Load().Info().Msgf("DNS intercept: NRPT catch-all rule active - all DNS queries directed to %s", listenerIP) + } + } + + // In hard mode, also set up WFP filters to block non-local DNS. if hardIntercept { - if err := p.startWFPFilters(state); err != nil { - // Roll back NRPT since WFP failed. - mainLog.Load().Error().Err(err).Msg("DNS intercept: WFP setup failed, rolling back NRPT") - _ = removeNRPTCatchAllRule() - flushDNSCache() - state.nrptActive = false + if err := ops.startWFP(state); err != nil { + owner, _ := state.nrptPolicyOwner() + if owner == nrptRuleOwnerGroupPolicy && externalProbeOK { + // A GP rule the probe proved is routing keeps DNS flowing through ctrld + // even with no WFP filters, and rewriting adapter DNS would violate that + // external policy - so this is not a fall-back-to-adapter-DNS failure. + // + // It is still a hard-mode enforcement gap: with no block filters, raw DNS + // to a public resolver, DoH clients and apps with their own resolver are + // not filtered at all. Publish the state and start the health monitor so + // repairMissingWFP keeps retrying WFP instead of the process running + // unenforced for its whole life on one error line. Ownership stays with + // Group Policy so the monitor never writes NRPT policy here. + mainLog.Load().Error().Err(err). + Msg("DNS intercept: WFP setup failed while GP-managed NRPT is verified routing - DNS resolves through ctrld but hard-mode enforcement is OFF; retrying WFP in the background") + p.dnsInterceptState = state + go p.nrptHealthMonitor(state) + // The service start is still reported as failed (setDnsOK stays false in + // setDNS): a hard-mode process with no block filters must not read as a + // healthy start, even though name resolution works. + return fmt.Errorf("dns intercept: WFP setup failed: %w: %w", err, errGPNRPTVerified) + } + if owner == nrptRuleOwnerCtrld { + mainLog.Load().Error().Err(err).Msg("DNS intercept: WFP setup failed, rolling back ctrld-owned NRPT") + _ = ops.removeRule() + ops.flush() + } else { + mainLog.Load().Error().Err(err).Msg("DNS intercept: WFP setup failed; leaving GP-managed NRPT untouched") + } + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") return fmt.Errorf("dns intercept: WFP setup failed: %w", err) } } else { @@ -645,26 +940,80 @@ func (p *prog) startDNSIntercept() error { // only) and use CLEAR_ACTION_RIGHT to override any block from other // sublayers. Adding them at startup eliminates the DNS outage window // that would otherwise occur between VPN connect and reactive activation. - if err := p.activateLoopbackWFPProtect(state); err != nil { + if err := ops.loopback(state); err != nil { // Non-fatal: loopback protect is a defense-in-depth measure. // NRPT still works when no third-party WFP blocks are present. mainLog.Load().Warn().Err(err).Msg("DNS intercept: failed to activate proactive loopback WFP protect — will retry on probe failure") } } - p.dnsInterceptState = state + owner, externalRuleName := state.nrptPolicyOwner() + if owner == nrptRuleOwnerGroupPolicy && !externalProbeOK { + // The first probe ran before loopback WFP protection existed. Verify once + // more synchronously after WFP setup so service readiness does not race an + // async proof; this path is ownership-aware and never mutates GP NRPT. + externalProbeOK = p.nrptProbeAndHeal(state) + } - // Start periodic NRPT health monitor. + // Everything the host needs is in place: publish, then start the goroutines that + // keep it that way. They receive state directly, so this ordering is only about + // when the rest of ctrld may observe intercept mode as active. + p.dnsInterceptState = state go p.nrptHealthMonitor(state) - // Verify NRPT is actually working (async — doesn't block startup). - // This catches the race condition where RefreshPolicyEx returns before - // the DNS Client service has loaded the NRPT rule from registry. - go p.nrptProbeAndHeal() + owner, _ = state.nrptPolicyOwner() + if owner == nrptRuleOwnerCtrld { + // ctrld-owned policy keeps the existing asynchronous activation/heal path. + go p.nrptProbeAndHeal(state) + } + + if owner == nrptRuleOwnerGroupPolicy && !externalProbeOK { + // External policy owns the namespace and neither synchronous proof reached ctrld. + // The recovery state and monitor stay up - the rule may start routing once the DNS + // Client settles, and only external policy may fix it - but this start is not + // ready: the DNS Client is not delivering queries to ctrld, adapter DNS was + // deliberately preserved, and no owned fallback may be written beside an + // administrator's catch-all. Reporting success here would publish readiness while + // nothing is filtering, and in hard mode WFP is simultaneously blocking every + // other resolver, which is an outage rather than degraded health. + mainLog.Load().Error().Str("rule", externalRuleName).Str("listener", listenerIP). + Msg("DNS intercept: GP-managed NRPT owns the namespace but no probe reached ctrld; leaving adapter DNS untouched and reporting a failed start until a probe succeeds") + return fmt.Errorf("dns intercept: %w (rule %q)", errGPNRPTIneffective, externalRuleName) + } return nil } +// removeOrphanedCtrldNRPTRule deletes a ctrld-owned NRPT catch-all that no running +// ctrld is backing. It is safe against external policy: the rule is found by ctrld's +// own deterministic GUID, never by shape. +// +// Without this, one unclean exit can strand a rule that later takes the whole machine +// off DNS. ctrld dies without a stop while it owns policy, so the GUID rule stays in +// the local store pointing at 127.0.0.1. The org then deploys a GP catch-all: from that +// point every start adopts the GP rule and every stop takes the GP branch, so nothing +// ever looks at the local store - including the stop during uninstall. The orphan stays +// invisible, because any rule in the GP store puts the DNS Client in GP mode where the +// local store is ignored entirely. When the admin eventually removes the GP rule - +// most plausibly while cleaning up after uninstalling ctrld - the DNS Client leaves GP +// mode, reads the local store again, and every query on the machine goes to a listener +// that has not existed for months. It is also miserable to diagnose: local-store rules +// do not appear in Get-DnsClientNrptPolicy, so the standard tooling reports no policy +// at all while nothing resolves. +func (p *prog) removeOrphanedCtrldNRPTRule(reason string) { + ops := p.nrptOps() + if !ops.ruleExists() { + return + } + mainLog.Load().Warn().Str("reason", reason). + Msg("DNS intercept: removing orphaned ctrld NRPT catch-all left by an earlier run") + if err := ops.removeRule(); err != nil { + mainLog.Load().Warn().Err(err).Msg("DNS intercept: failed to remove orphaned ctrld NRPT catch-all") + return + } + ops.signal() +} + // startWFPFilters opens the WFP engine and adds all block/permit filters. // Called only in hard intercept mode. func (p *prog) startWFPFilters(state *wfpState) error { @@ -1017,12 +1366,15 @@ func (p *prog) cleanupWFPFilters(state *wfpState) { return } - // Clean up loopback protect filters (DNS mode VPN workaround). + // Hold state.mu across the whole teardown: engineHandle and every filter ID slice + // below is shared with the VPN DNS exemption path and the recovery flows. state.mu.Lock() + defer state.mu.Unlock() + + // Clean up loopback protect filters (DNS mode VPN workaround). loopbackIDs := state.loopbackPermitIDs state.loopbackPermitIDs = nil state.loopbackProtectActive = false - state.mu.Unlock() for _, filterID := range loopbackIDs { r1, _, _ := procFwpmFilterDeleteById0.Call(state.engineHandle, uintptr(filterID)) if r1 != 0 { @@ -1264,6 +1616,27 @@ func (p *prog) addWFPHardPermitLocalhostFilter(engineHandle uintptr, name string // stopDNSIntercept removes all WFP filters and shuts down the DNS interception. func (p *prog) stopDNSIntercept() error { + // Announce the stop before waiting for the lifecycle lock. A rebuild that holds it + // runs a full start, whose NRPT verification can sit in probe backoffs for seconds; + // the flag is what lets those flows abandon their work instead of making the stop + // wait. A stop that waits too long is not a delay but a failure mode: the Service + // Control Manager kills ctrld on timeout, and then nothing is cleaned up at all. + p.dnsInterceptStopRequested.Store(true) + p.dnsInterceptMu.Lock() + defer p.dnsInterceptMu.Unlock() + defer p.dnsInterceptStopRequested.Store(false) + return p.stopDNSInterceptLocked() +} + +// stopDNSInterceptLocked is stopDNSIntercept with p.dnsInterceptMu already held. +// +// It revokes the state before removing anything. Teardown deletes the WFP sublayer, and +// a missing sublayer is exactly what the health monitor reads as "our filters were +// wiped, rebuild everything" - so a monitor tick landing inside the shutdown window +// would otherwise re-add the NRPT catch-all and the WFP filters moments before the +// process exits, leaving Windows resolving through a ctrld that is gone. Revoking first +// means every such flow sees a retired state and stands down. +func (p *prog) stopDNSInterceptLocked() error { if p.dnsInterceptState == nil { mainLog.Load().Debug().Msg("DNS intercept: no state to clean up") return nil @@ -1271,22 +1644,48 @@ func (p *prog) stopDNSIntercept() error { state := p.dnsInterceptState.(*wfpState) + // Revoke first, then remove. Both signals are what the monitor, the delayed + // rechecks and the NRPT heal flows read to decide whether they may still write + // host DNS state. + p.dnsInterceptState = nil // Stop the health monitor goroutine. if state.stopCh != nil { close(state.stopCh) } - // Remove NRPT rule BEFORE WFP cleanup — restore normal DNS resolution - // before removing the block filters that enforce it. - if state.nrptActive { - if err := removeNRPTCatchAllRule(); err != nil { - mainLog.Load().Warn().Err(err).Msg("DNS intercept: failed to remove NRPT catch-all rule") + // Remove only ctrld-owned NRPT state. A GP-owned catch-all is an external + // deployment contract and must survive service stop, restart, and uninstall. + // + // Hold the transition lock across the removal so an in-flight NRPT transition + // finishes first and any later one sees the revoked state under the same lock. The + // stop-requested flag is already set, so an in-flight transition abandons its probes + // rather than making this wait. + p.nrptTransitionMu.Lock() + defer p.nrptTransitionMu.Unlock() + + ops := p.nrptOps() + owner, externalRuleName := state.nrptPolicyOwner() + switch owner { + case nrptRuleOwnerCtrld: + if err := ops.removeRule(); err != nil { + mainLog.Load().Warn().Err(err).Msg("DNS intercept: failed to remove ctrld-owned NRPT catch-all rule") } else { - mainLog.Load().Info().Msg("DNS intercept: removed NRPT catch-all rule") + mainLog.Load().Info().Msg("DNS intercept: removed ctrld-owned NRPT catch-all rule") } - flushDNSCache() - state.nrptActive = false + ops.flush() + case nrptRuleOwnerGroupPolicy: + mainLog.Load().Info().Str("rule", externalRuleName). + Msg("DNS intercept: leaving GP-managed NRPT catch-all untouched during shutdown") + // External policy stays, but a ctrld rule from an earlier unclean exit must + // not: while GP mode hides the local store, this stop is the last chance + // anything will look there. See removeOrphanedCtrldNRPTRule. + p.removeOrphanedCtrldNRPTRule("shutdown with externally owned NRPT policy") + case nrptRuleOwnerNone: + // No ownership was ever established this run - an activation that failed, or a + // start that never got that far. A ctrld rule found here is still ours. + p.removeOrphanedCtrldNRPTRule("shutdown with no NRPT owner") } + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") // Clean up WFP if the engine was opened (hard mode or loopback protect). if state.engineHandle != 0 { @@ -1295,11 +1694,129 @@ func (p *prog) stopDNSIntercept() error { mainLog.Load().Info().Msg("DNS intercept: WFP shutdown complete") } - p.dnsInterceptState = nil mainLog.Load().Info().Msg("DNS intercept: shutdown complete") return nil } +// interceptRebuildResult reports what rebuildDNSIntercept did. +type interceptRebuildResult int + +const ( + // interceptRebuildRetired means the caller's state was no longer the live one - + // shutdown revoked it, or an earlier rebuild replaced it - so nothing was touched. + interceptRebuildRetired interceptRebuildResult = iota + // interceptRebuildDone means the intercept was torn down and re-created. + interceptRebuildDone + // interceptRebuildFailed means teardown ran but the re-create returned an error. + // dnsInterceptState is nil afterwards, except on the one path that publishes a + // partial intercept on purpose - hard mode with verified GP NRPT but no WFP - which + // leaves a health monitor running to keep retrying. + interceptRebuildFailed +) + +// rebuildDNSIntercept tears the intercept down and creates it again, for callers that +// found our filters wiped from underneath us. +// +// It refuses unless state is still the published intercept state. That check is what +// stops a health monitor tick from resurrecting DNS interception during or after a +// service stop: shutdown revokes the state under this same lock before it removes +// anything, so a monitor arriving late finds its state retired and does nothing. +// Holding the lock across teardown and create also means a stop that arrives +// mid-rebuild waits, then tears down whatever the rebuild published - never a +// half-built intercept. +// +// Whatever the result, the caller's state is dead afterwards: the monitor goroutine +// that owns it must exit. +// rebuildDNSInterceptFn is the rebuild entry point. Indirected so tests can assert which +// conditions ask for a rebuild without running a real teardown and start. Assigned in +// init because the rebuild reaches back here through the health monitor. +var rebuildDNSInterceptFn func(*prog, *wfpState, string) interceptRebuildResult + +func init() { + rebuildDNSInterceptFn = (*prog).rebuildDNSIntercept +} + +func (p *prog) rebuildDNSIntercept(state *wfpState, reason string) interceptRebuildResult { + p.dnsInterceptMu.Lock() + defer p.dnsInterceptMu.Unlock() + + if live, ok := p.dnsInterceptState.(*wfpState); !ok || live != state { + mainLog.Load().Info().Str("reason", reason). + Msg("DNS intercept: not rebuilding - this intercept was already retired by shutdown or an earlier rebuild") + return interceptRebuildRetired + } + + mainLog.Load().Warn().Str("reason", reason).Msg("DNS intercept: rebuilding interception") + _ = p.stopDNSInterceptLocked() + if err := p.startDNSInterceptLocked(); err != nil { + mainLog.Load().Error().Err(err).Str("reason", reason).Msg("DNS intercept: rebuild failed") + return interceptRebuildFailed + } + return interceptRebuildDone +} + +// interceptStateRevoked reports whether state has been retired, meaning nothing may +// write host DNS state on its behalf any more. +// +// stopDNSInterceptLocked closes stopCh before it removes the NRPT rule or the WFP +// filters, and a stop waiting for the lifecycle lock sets dnsInterceptStopRequested +// first. Together they are the cheap shutdown signal for the flows that must not take +// p.dnsInterceptMu: the NRPT probe and heal sequences, which wait seconds between +// probes and also run from inside the locked start path. +func (p *prog) interceptStateRevoked(state *wfpState) bool { + if state == nil || state.stopCh == nil { + return true + } + if p.dnsInterceptStopRequested.Load() { + return true + } + select { + case <-state.stopCh: + return true + default: + return false + } +} + +// interceptRevocationPollInterval is how quickly the recovery flows notice a stop that +// is waiting for the lifecycle lock. A pending stop can only set a flag - it cannot +// close stopCh until it owns the lock - so waits poll instead of selecting on a channel. +const interceptRevocationPollInterval = 100 * time.Millisecond + +// interceptWait waits for d, or until the intercept is retired, whichever comes first. +// It reports whether the caller may keep working. +// +// Every wait in the NRPT recovery flows goes through this. Those flows can be holding +// the lifecycle lock - startDNSInterceptLocked runs one synchronously, and a rebuild +// holds the lock across the whole start - and their backoffs add up to tens of seconds. +// A plain time.Sleep there makes a service stop wait that long for the lock, risking +// the Service Control Manager killing ctrld before it removes the NRPT rule and the WFP +// filters, which is the unclean shutdown the locking exists to prevent. Bounding each +// wait by the shutdown signal keeps a stop's wait to about one poll interval plus +// whatever uncancellable Windows call is in flight. +func (p *prog) interceptWait(state *wfpState, d time.Duration) bool { + deadline := time.Now().Add(d) + for { + if p.interceptStateRevoked(state) { + return false + } + remaining := time.Until(deadline) + if remaining <= 0 { + return true + } + if remaining > interceptRevocationPollInterval { + remaining = interceptRevocationPollInterval + } + timer := time.NewTimer(remaining) + select { + case <-state.stopCh: + timer.Stop() + return false + case <-timer.C: + } + } +} + // exemptVPNDNSServers updates the WFP filters to permit outbound DNS to the given // VPN DNS server IPs. This prevents the block filters from intercepting ctrld's own // forwarded queries to VPN DNS servers (split DNS routing). @@ -1317,6 +1834,12 @@ func (p *prog) exemptVPNDNSServers(exemptions []vpnDNSExemption) error { if !ok || state == nil { return fmt.Errorf("DNS intercept state not available") } + // engineHandle, loopbackProtectActive and vpnPermitFilterIDs are all shared with the + // monitor, the recovery flows and teardown, so this runs under state.mu like every + // other reader and writer of them. + state.mu.Lock() + defer state.mu.Unlock() + // In dns mode (no WFP) or loopback-protect-only mode, VPN DNS exemptions // are not needed — there are no ctrld block filters to exempt from. // Loopback protect only adds hard-permit filters for localhost DNS; @@ -1548,6 +2071,582 @@ const pfAnchorRecheckDelay = 2 * time.Second // pfAnchorRecheckDelayLong is the longer delayed re-check for slower VPN teardowns. const pfAnchorRecheckDelayLong = 4 * time.Second +func gpCatchAllConflictBlocksFallback(state *wfpState, reason string) bool { + ruleName, dnsServers := findConflictingGPCatchAll(state.listenerIP) + if ruleName == "" { + return false + } + mainLog.Load().Error().Str("rule", ruleName).Str("nameservers", dnsServers).Str("reason", reason). + Msg("DNS intercept: GP catch-all targets another resolver; refusing to create a competing fallback rule") + return true +} + +// nrptOps is the seam between NRPT ownership decisions and the Windows side effects +// they cause. Tests substitute it to drive a transition - probe outcomes, registry +// state, concurrency - without touching the host's registry or DNS Client. +type nrptOps struct { + probe func(state *wfpState) bool + ruleExists func() bool + addRule func(listenerIP string) error + removeRule func() error + signal func() + findGPRule func(listenerIP string) string + gpRuleMatches func(ruleName, listenerIP string) bool + gpConflicts func(state *wfpState, reason string) bool + loopback func(state *wfpState) error + wait func(state *wfpState, d time.Duration) bool + flush func() + parentEmpty func(keyPath string) bool + cleanParent func() bool + startWFP func(state *wfpState) error +} + +// nrptOpsForTest overrides the NRPT side effects. Windows tests only. +var nrptOpsForTest *nrptOps + +func (p *prog) nrptOps() nrptOps { + if nrptOpsForTest != nil { + return *nrptOpsForTest + } + return nrptOps{ + probe: p.probeNRPT, + ruleExists: nrptCatchAllRuleExists, + addRule: addNRPTCatchAllRule, + removeRule: removeNRPTCatchAllRule, + signal: signalNRPTChange, + findGPRule: findMatchingGPNRPTRule, + gpRuleMatches: gpNRPTRuleMatches, + gpConflicts: p.gpCatchAllConflictBlocksFallbackOps, + loopback: p.activateLoopbackWFPProtect, + wait: p.interceptWait, + flush: flushDNSCache, + parentEmpty: nrptParentKeyEmpty, + cleanParent: cleanEmptyNRPTParent, + startWFP: p.startWFPFilters, + } +} + +func (p *prog) gpCatchAllConflictBlocksFallbackOps(state *wfpState, reason string) bool { + return gpCatchAllConflictBlocksFallback(state, reason) +} + +// nrptHandbackResult reports how a handback attempt ended. +type nrptHandbackResult int + +const ( + // nrptHandbackVerified: external policy owns NRPT and proved, with ctrld's own keys + // gone, that it routes to this listener. + nrptHandbackVerified nrptHandbackResult = iota + // nrptHandbackUnverified: external policy owns the namespace but is not routing. + // Nothing of ctrld's was removed, so there was nothing to lose by recording it. + nrptHandbackUnverified + // nrptHandbackKeptCtrld: the proof failed with ctrld's rule removed, so the rule was + // restored and ctrld keeps ownership. + nrptHandbackKeptCtrld + // nrptHandbackAborted: shutdown landed, a registry step failed, the attempt was + // throttled, or the candidate child is no longer there. Nothing was decided. + nrptHandbackAborted + // nrptHandbackConflict: an administrator-owned catch-all is present that does not + // target ctrld. External policy owns the namespace and ctrld must not write beside it. + nrptHandbackConflict +) + +// nrptHandbackRetryInterval bounds how often ctrld will take its own rule out of the way +// to re-test the same external catch-all. Each attempt briefly removes the only working +// route, so retrying on every 30s health tick would be its own outage. A rule the +// administrator has changed is retested immediately regardless. +const nrptHandbackRetryInterval = 15 * time.Minute + +// nrptHandbackProbePasses bounds how many probes one handback spends chasing a Group +// Policy store that keeps changing under it. Each pass costs a probe timeout, and the +// budget being spent is not an excuse to guess: see externalAfterRemoval. +const nrptHandbackProbePasses = 2 + +// nrptHandbackToExternal is the only path that records external (Group Policy) +// ownership of NRPT. +// +// The proof has to be produced with ctrld's own keys gone. A probe taken while the ctrld +// fallback is still installed can be answered by that fallback, so a present-but- +// ineffective GP child would otherwise let ctrld delete the last working route and then +// declare external ownership. In hard mode that is a machine-wide DNS outage: WFP keeps +// blocking outbound DNS with nothing redirecting it to ctrld. +// +// The transition is: remove only ctrld's own keys, signal, probe again, re-read the same +// GP child - and if that second probe fails, put ctrld's rule back and keep ctrld +// ownership. Everything runs under nrptTransitionMu so a stop cannot interleave with it. +func (p *prog) nrptHandbackToExternal(state *wfpState, ruleName, reason string) nrptHandbackResult { + ops := p.nrptOps() + + p.nrptTransitionMu.Lock() + defer p.nrptTransitionMu.Unlock() + + if p.interceptStateRevoked(state) { + return nrptHandbackAborted + } + + if !ops.gpRuleMatches(ruleName, state.listenerIP) { + return nrptHandbackAborted + } + + // Nothing of ours in the way: a probe already measures external policy alone, and + // refusing would mean writing a competing catch-all beside an administrator's rule. + if !ops.ruleExists() { + child, class, routes := p.externalAfterRemoval(ops, state, ruleName, reason) + switch { + case class == gpChildSameExact && routes: + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, child) + state.nrptRecoveryLimiter.recordStableSuccess() + mainLog.Load().Info().Str("rule", child).Str("reason", reason). + Msg("DNS intercept: GP-managed catch-all verified routing to ctrld; external policy owns NRPT") + return nrptHandbackVerified + case class == gpChildSameExact: + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, child) + mainLog.Load().Warn().Str("rule", child).Str("reason", reason). + Msg("DNS intercept: GP-managed catch-all owns the namespace but is not routing; leaving external policy untouched") + return nrptHandbackUnverified + case class == gpChildConflicting: + // An administrator-owned catch-all now targets another resolver, or is + // malformed. It owns the namespace and ctrld must not write a sibling. + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") + mainLog.Load().Error().Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: GP catch-all changed to one that does not target ctrld; refusing to write a competing rule") + return nrptHandbackConflict + default: + // External policy is gone, or the store is still churning. Decide nothing: + // the caller re-reads and retries. + return nrptHandbackAborted + } + } + + // ctrld's rule is installed. Taking it out to test the external one is disruptive, so + // it is throttled: an external rule that never routes would otherwise cost a brief + // outage on every health tick. The check comes before the probe so a throttled tick + // costs nothing, and the budget is only spent below, once the attempt is real. + now := time.Now() + if !state.handbackAllowed(now, ruleName, nrptHandbackRetryInterval) { + return nrptHandbackAborted + } + + // Pre-probe: this measures whatever routes DNS today, ctrld's own rule included, so + // it can never prove anything about external policy - it only says whether there is a + // working route here to risk. If nothing is routing there is nothing to protect and + // nothing to compare against, so leave it to the heal cycle rather than start + // deleting rules. + if !ops.probe(state) { + return nrptHandbackAborted + } + state.recordHandbackAttempt(now, ruleName, nrptHandbackRetryInterval) + + if err := ops.removeRule(); err != nil { + mainLog.Load().Warn().Err(err).Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: GP catch-all found but ctrld's own rule could not be removed for the handback probe") + return nrptHandbackAborted + } + ops.signal() + if !ops.wait(state, nrptHandbackSettleDelay) { + // Shutdown landed. NRPT is clean, which is the right state to leave behind. + return nrptHandbackAborted + } + + // Post-removal probe and classification, always - not only when the probe succeeded. + // Group Policy can refresh during the probe, and what it changed into decides whether + // restoring ctrld's rule is right or would create a sibling that must never be written. + child, class, routes := p.externalAfterRemoval(ops, state, ruleName, reason) + + switch { + case class == gpChildSameExact && routes: + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, child) + state.nrptRecoveryLimiter.recordStableSuccess() + mainLog.Load().Info().Str("rule", child).Str("listener", state.listenerIP).Str("reason", reason). + Msg("DNS intercept: GP-managed catch-all carried DNS without ctrld's rule; returned NRPT ownership to Group Policy") + return nrptHandbackVerified + + case class == gpChildConflicting: + // The child changed into a catch-all that does not target ctrld. It owns the + // namespace, so ctrld's rule stays off: restoring it here is exactly the + // competing sibling beside administrator policy that is forbidden elsewhere. + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") + mainLog.Load().Error().Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: GP catch-all changed to one that does not target ctrld during the handback probe; leaving NRPT to Group Policy and not restoring the ctrld rule") + return nrptHandbackConflict + + case class == gpChildSameExact && child != ruleName: + // A different administrator catch-all took the namespace while ctrld's keys were + // off, and its own pass says it is not routing yet. ctrld's rule still must not + // come back: addNRPTCatchAllRule writes ctrld's GP catch-all whenever another GP + // rule exists, so restoring would put a sibling beside a catch-all that was not + // even there when this transition started. Record external ownership and let the + // health monitor keep watching the new child. + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, child) + mainLog.Load().Warn().Str("old_rule", ruleName).Str("rule", child).Str("reason", reason). + Msg("DNS intercept: a different GP catch-all took the namespace during the handback probe and is not routing yet; leaving NRPT to Group Policy without restoring the ctrld rule") + return nrptHandbackUnverified + + } + + // The original child is still exact but cannot carry DNS, or external policy is gone + // altogether: ctrld's route is the one that has to come back. This only restores the + // state the transition started from, so it creates no new sibling. + if err := ops.addRule(state.listenerIP); err != nil { + mainLog.Load().Error().Err(err).Str("rule", ruleName). + Msg("DNS intercept: handback probe failed and the ctrld NRPT rule could not be restored; the health monitor will retry") + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") + return nrptHandbackAborted + } + ops.signal() + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + if class == gpChildGone { + mainLog.Load().Warn().Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: GP-managed catch-all disappeared during the handback probe; restored the ctrld fallback and kept ctrld ownership") + } else { + mainLog.Load().Warn().Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: GP-managed catch-all did not carry DNS without ctrld's rule; restored the ctrld fallback and kept ctrld ownership") + } + return nrptHandbackKeptCtrld +} + +// gpChildClass is what an external NRPT catch-all looks like at the moment ctrld checks, +// which is not necessarily what it looked like when a probe was sent: Group Policy can +// refresh while the probe is in flight. +type gpChildClass int + +const ( + // gpChildSameExact: the same child still names exactly this listener. + gpChildSameExact gpChildClass = iota + // gpChildReplaced: a different child now matches this listener exactly. It has not + // proved anything yet, so it is not adopted on this pass. + gpChildReplaced + // gpChildConflicting: an administrator-owned catch-all is present that does not target + // ctrld - another resolver, or malformed. ctrld must not write a rule beside it. + gpChildConflicting + // gpChildGone: no external catch-all owns the namespace any more. + gpChildGone +) + +// externalAfterRemoval probes external policy with ctrld's keys already gone, then says +// which child the result belongs to, what state that child is in, and whether it routed. +// It writes nothing: the caller decides what the verdict means. +// +// A probe result can only ever be attributed to the child that was on disk for the whole +// probe. When Group Policy swaps the child mid-probe the old result is void, so the +// replacement gets one pass of its own here rather than inheriting a verdict it never +// earned. Two passes is the limit; a store that keeps changing is reported as still +// churning so the caller can retry instead of guessing. +func (p *prog) externalAfterRemoval(ops nrptOps, state *wfpState, candidate, reason string) (string, gpChildClass, bool) { + for pass := 0; pass < nrptHandbackProbePasses; pass++ { + routes := ops.probe(state) + class := p.classifyGPChild(ops, state, candidate, reason) + if class != gpChildReplaced { + return candidate, class, routes + } + if pass == nrptHandbackProbePasses-1 { + break + } + next := ops.findGPRule(state.listenerIP) + if next == "" { + return candidate, gpChildGone, routes + } + mainLog.Load().Warn().Str("old_rule", candidate).Str("rule", next).Str("reason", reason). + Msg("DNS intercept: a different GP catch-all appeared during the probe; testing that one instead") + candidate = next + } + + // The probe budget is spent and the store is still moving. Report the store as it is + // now, with no route proved, rather than reporting churn: "undecided" would let + // startup and owned recovery fall through to writing ctrld's rule, and + // addNRPTCatchAllRule puts that in the GP path beside whatever exact catch-all is + // there - the sibling that must never exist. Failing safe from the current store + // keeps every outcome terminal for those callers unless the namespace is genuinely + // free. + mainLog.Load().Warn().Str("rule", candidate).Str("reason", reason). + Msg("DNS intercept: GP catch-alls kept changing during the handback probe; classifying the store as it stands with no route proved") + if current := ops.findGPRule(state.listenerIP); current != "" { + return current, gpChildSameExact, false + } + if ops.gpConflicts(state, reason) { + return candidate, gpChildConflicting, false + } + return candidate, gpChildGone, false +} + +// classifyGPChild re-reads the GP store and says what state the external catch-all is in. +// Every post-probe decision goes through it, so a child that changed mid-probe can never +// be treated as the child that was measured. +func (p *prog) classifyGPChild(ops nrptOps, state *wfpState, ruleName, reason string) gpChildClass { + if ops.gpRuleMatches(ruleName, state.listenerIP) { + return gpChildSameExact + } + if other := ops.findGPRule(state.listenerIP); other != "" { + return gpChildReplaced + } + if ops.gpConflicts(state, reason) { + return gpChildConflicting + } + return gpChildGone +} + +// nrptHandbackSettleDelay gives the DNS Client a moment to drop ctrld's removed rule +// before the handback probe decides whether external policy routes on its own. +const nrptHandbackSettleDelay = 1 * time.Second + +// deferToExternalCatchAll stops ctrld-owned recovery when an administrator catch-all owns +// the namespace, and reports whether the caller must stop. +// +// It hands back where a verdict is possible. Where one is not - the handback needs a +// working route to compare against, and during a heal cycle there often is none - the +// presence of an exact external catch-all is itself the ownership signal, so ownership is +// recorded without proof and recovery stops anyway. Continuing would mean signalling the +// DNS Client, or deleting and recreating ctrld's rule, while administrator policy owns the +// namespace: the two things #576 forbids. The health monitor keeps testing the rule and +// can hand back properly once it routes. +func (p *prog) deferToExternalCatchAll(ops nrptOps, state *wfpState, reason string) bool { + ruleName := ops.findGPRule(state.listenerIP) + if ruleName == "" || !ops.gpRuleMatches(ruleName, state.listenerIP) { + return false + } + + switch result := p.nrptHandbackToExternal(state, ruleName, reason); { + case result == nrptHandbackKeptCtrld: + // The external rule was proved unable to carry DNS and ctrld's rule is back, so + // owned recovery is exactly what should continue. + return false + case nrptExternalOwns(result): + if result == nrptHandbackUnverified { + p.healBlockedLoopbackDNS(state, reason) + } + return true + default: + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, ruleName) + mainLog.Load().Warn().Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: a GP catch-all owns the namespace but could not be tested; stopping ctrld-owned recovery and leaving external policy untouched") + p.healBlockedLoopbackDNS(state, reason) + return true + } +} + +// nrptExternalOwns reports whether a handback left external policy owning the namespace. +// +// All three outcomes count, not just Verified: an administrator's catch-all owns the +// namespace whether it routes to ctrld (Verified), does not route at all (Unverified), or +// points somewhere else entirely (Conflict). Each is terminal for ctrld-owned recovery, +// because continuing would signal the DNS Client and delete and recreate ctrld's rule +// beside that catch-all - the competing, ambiguous policy #576 exists to avoid. Where the +// external rule is present but ineffective, loopback WFP protect is the only remediation +// left. +func nrptExternalOwns(result nrptHandbackResult) bool { + switch result { + case nrptHandbackVerified, nrptHandbackUnverified, nrptHandbackConflict: + return true + default: + return false + } +} + +// nrptTransition runs one complete NRPT mutation - write or delete, signal, record +// owner - with the transition lock held, and reports whether it ran. +// +// Checking interceptStateRevoked and then mutating is not enough on its own: a stop can +// land in between, revoke the state, remove NRPT and finish, after which the mutation +// would write a catch-all pointing at a listener that no longer exists. The stop takes +// this same lock around its own NRPT removal, so holding it across the whole +// observe-mutate-signal step is what makes the two mutually exclusive. Callers must keep +// probe backoffs outside fn: the lock is for a single transition, not for a heal cycle. +func (p *prog) nrptTransition(state *wfpState, fn func()) bool { + p.nrptTransitionMu.Lock() + defer p.nrptTransitionMu.Unlock() + if p.interceptStateRevoked(state) { + return false + } + fn() + return true +} + +func (p *prog) activateCtrldNRPTFallback(state *wfpState, reason string) bool { + ops := p.nrptOps() + + // Close the observation-to-write race in both directions. Group Policy can refresh + // between the monitor's missing-rule check and this call, and again between this + // check and the write - so the write path re-checks under the transition lock and + // sends us back here when a matching child has appeared. Two passes is enough: the + // second either hands back or writes with the store checked under the lock. + for attempt := 0; attempt < 2; attempt++ { + if ruleName := ops.findGPRule(state.listenerIP); ruleName != "" && + ops.gpRuleMatches(ruleName, state.listenerIP) { + // A matching child owns the namespace, so hand back rather than create a + // sibling rule. The handback runs its own transition, so it cannot be called + // with the lock held. + if p.nrptHandbackToExternal(state, ruleName, reason) == nrptHandbackKeptCtrld { + // The handback restored ctrld's rule, which is what this call wanted. + return true + } + // Every other outcome means external policy owns the namespace, or that + // nothing could be decided. Either way this must not write a rule beside it. + return false + } + wrote, gpAppeared := p.writeCtrldFallback(ops, state, reason) + if !gpAppeared { + return wrote + } + } + return false +} + +// writeCtrldFallback writes ctrld's own catch-all under the transition lock. It reports +// whether it wrote, and whether it stood down because a matching GP child appeared - in +// which case the caller must take the handback path instead. +func (p *prog) writeCtrldFallback(ops nrptOps, state *wfpState, reason string) (wrote, gpAppeared bool) { + p.nrptTransitionMu.Lock() + defer p.nrptTransitionMu.Unlock() + + // Re-check under the transition lock. A retired state must not write NRPT policy: + // shutdown has already removed the ctrld-owned catch-all, and re-adding it + // afterwards leaves the DNS Client routing every query to a listener that no longer + // exists - a machine-wide resolution outage, not a cosmetic leftover. Checking + // before the lock cannot close that gap, because a stop can land between the check + // and the write; the stop takes this same lock around its own NRPT removal. + if p.interceptStateRevoked(state) { + mainLog.Load().Debug().Str("reason", reason). + Msg("DNS intercept: skipping ctrld NRPT fallback - intercept was retired") + return false, false + } + // The same reasoning applies to the GP store, which the caller read before taking + // this lock. gpConflicts alone does not cover it: findConflictingGPCatchAll skips a + // *matching* child by design, so without this re-read a policy refresh inside that + // window would let the write land beside an administrator catch-all that no probe has + // tested. + if ruleName := ops.findGPRule(state.listenerIP); ruleName != "" && + ops.gpRuleMatches(ruleName, state.listenerIP) { + mainLog.Load().Info().Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: a matching GP catch-all appeared before the fallback write; handing back instead of writing beside it") + return false, true + } + if ops.gpConflicts(state, reason) { + return false, false + } + // Another health or delayed-recheck path may have written the rule while this one + // waited for the lock; do not duplicate the write and the signalling. + if ops.ruleExists() { + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + mainLog.Load().Debug().Str("reason", reason). + Msg("DNS intercept: ctrld NRPT rule was already restored by a concurrent transition") + return false, false + } + if err := ops.addRule(state.listenerIP); err != nil { + mainLog.Load().Error().Err(err).Str("reason", reason). + Msg("DNS intercept: failed to activate ctrld-owned NRPT fallback; the health monitor will retry") + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") + return false, false + } + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + ops.signal() + mainLog.Load().Warn().Str("reason", reason). + Msg("DNS intercept: GP-managed catch-all unavailable - activated ctrld-owned NRPT fallback") + return true, false +} + +// tryAdoptMatchingGPNRPT hands NRPT ownership back to Group Policy when a matching +// external catch-all exists. It reports whether external policy now owns NRPT. +// +// The proof lives in nrptHandbackToExternal: a probe taken while ctrld's fallback is +// still installed proves nothing about the external rule, so the decision is always +// made with ctrld's keys removed. +func (p *prog) tryAdoptMatchingGPNRPT(state *wfpState) bool { + if p.interceptStateRevoked(state) { + return false + } + ruleName := p.nrptOps().findGPRule(state.listenerIP) + if ruleName == "" { + return false + } + switch p.nrptHandbackToExternal(state, ruleName, "matching GP-managed catch-all detected") { + case nrptHandbackVerified: + return true + case nrptHandbackUnverified: + // External policy owns the namespace but is not routing. ctrld must not write a + // competing rule, so the only remediation left is loopback WFP protect. Report + // external ownership: the caller must not fall through to owned recovery. + p.healBlockedLoopbackDNS(state, "GP-managed catch-all owns the namespace but is not routing") + return true + case nrptHandbackConflict: + // External policy now points somewhere other than ctrld. It owns the namespace, + // so report external ownership: owned recovery must not write beside it. + return true + case nrptHandbackKeptCtrld: + // The external rule could not carry DNS alone. A third-party WFP block dropping + // DNS below NRPT looks exactly like this, so try the one remediation that leaves + // external policy untouched; the next handback attempt can then succeed. + p.healBlockedLoopbackDNS(state, "GP-managed catch-all did not carry DNS on its own") + return false + } + return false +} + +// nrptNeedsCtrldActivation reports whether ctrld should write its own NRPT catch-all, +// given who owns policy and whether a ctrld rule is currently present. +// +// Owner None is the interesting case: it means an earlier write failed. Nothing else +// re-arms it - nrptProbeAndHeal returns early without ctrld ownership, and the health +// monitor used to skip the owner-None tick entirely - so without retrying, the machine +// keeps no NRPT rule for the rest of the process lifetime. In hard mode that is a full +// DNS outage rather than degraded interception: WFP goes on blocking outbound DNS while +// nothing redirects it to ctrld, and only a restart recovers. +func nrptNeedsCtrldActivation(owner nrptRuleOwner, ruleExists bool) bool { + switch owner { + case nrptRuleOwnerNone: + return true + case nrptRuleOwnerCtrld: + return !ruleExists + default: + // nrptRuleOwnerGroupPolicy: external policy owns the namespace, and a competing + // ctrld catch-all beside it would be ambiguous policy, not recovery. + return false + } +} + +// loopbackProtectSettleDelay gives WFP a moment to apply the loopback permits before +// the retry probe goes out. +const loopbackProtectSettleDelay = 500 * time.Millisecond + +// healBlockedLoopbackDNS retries the NRPT probe from behind loopback WFP protection and +// reports whether the probe then succeeded. +// +// A failed probe does not always mean the NRPT rule is wrong. Third-party WFP filters - +// OpenVPN's block-outside-dns is the common one - can drop DNS below NRPT, so the +// policy is correct and the packets never arrive. Loopback protect is the one +// remediation that fixes this while leaving externally owned policy untouched. Without +// this attempt, a GP rule blocked that way is never adopted and never healed: the +// monitor keeps finding a "present but not routing" rule for the life of the process. +// It is also the only remediation ctrld may run while external policy owns NRPT. +// signalNRPTChange is not a content-neutral nudge - it forces machine Group Policy via +// RefreshPolicyEx, sends Dnscache paramchange and flushes the resolver cache - so #576's +// contract for a present-but-ineffective GP rule is a warning plus WFP-only retries, with +// no refresh/paramchange/flush loop. +func (p *prog) healBlockedLoopbackDNS(state *wfpState, reason string) bool { + ops := p.nrptOps() + if p.interceptStateRevoked(state) { + return false + } + if hardIntercept { + // Hard mode owns the whole sublayer; loopback protect deliberately does + // nothing there, so a retry probe would only burn the probe timeout. + return false + } + if err := ops.loopback(state); err != nil { + mainLog.Load().Warn().Err(err).Str("reason", reason). + Msg("DNS intercept: could not activate loopback WFP protect while retrying a failed probe") + return false + } + if !ops.wait(state, loopbackProtectSettleDelay) { + return false + } + if !ops.probe(state) { + return false + } + mainLog.Load().Info().Str("reason", reason). + Msg("DNS intercept: probe recovered behind loopback WFP protect - a third-party WFP block was dropping DNS below NRPT") + return true +} + // scheduleDelayedRechecks schedules delayed OS resolver and VPN DNS refreshes after // network change events. While WFP filters don't get wiped like pf anchors, the OS // resolver and VPN DNS state can still be stale after VPN disconnect (same issue as macOS). @@ -1564,39 +2663,84 @@ func (p *prog) scheduleDelayedRechecks() { p.vpnDNS.Refresh(true) } - // NRPT watchdog: some VPN software clears NRPT policy rules on - // connect/disconnect. Re-add our catch-all rule if it was removed. + // Delayed rechecks must respect NRPT ownership. The old path looked only + // for ctrld's deterministic key, so it recreated that key immediately + // after startup had correctly adopted a GP-managed catch-all. state, ok := p.dnsInterceptState.(*wfpState) - if ok && state.nrptActive && !nrptCatchAllRuleExists() { - mainLog.Load().Warn().Msg("DNS intercept: NRPT catch-all rule was removed externally — re-adding") - if err := addNRPTCatchAllRule(state.listenerIP); err != nil { - mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to re-add NRPT catch-all rule") - state.nrptActive = false - } else { - signalNRPTChange() - mainLog.Load().Info().Msg("DNS intercept: NRPT catch-all rule restored") + if ok && !p.interceptStateRevoked(state) { + owner, _ := state.nrptPolicyOwner() + switch owner { + case nrptRuleOwnerGroupPolicy: + if findMatchingGPNRPTRule(state.listenerIP) == "" { + if p.activateCtrldNRPTFallback(state, "matching GP rule disappeared during delayed network recheck") { + go p.nrptProbeAndHeal(state) + } + } + case nrptRuleOwnerNone, nrptRuleOwnerCtrld: + if nrptNeedsCtrldActivation(owner, nrptCatchAllRuleExists()) { + mainLog.Load().Warn().Msg("DNS intercept: no ctrld NRPT catch-all in place - re-adding") + if p.activateCtrldNRPTFallback(state, "ctrld NRPT rule missing during delayed network recheck") { + go p.nrptProbeAndHeal(state) + } + } } } // WFP watchdog: verify our sublayer still exists. If another program - // or a crash removed it, the block filters are gone too. - if ok && state.engineHandle != 0 && !wfpSublayerExists(state.engineHandle) { + // or a crash removed it, the block filters are gone too. A timer that + // fires during shutdown must not rebuild what teardown removed, so this + // goes through rebuildDNSIntercept's ownership check. + if ok && !p.interceptStateRevoked(state) && state.engineHandle != 0 && !wfpSublayerExists(state.engineHandle) { mainLog.Load().Warn().Msg("DNS intercept: WFP sublayer was removed externally — re-creating all filters") - // Full teardown + re-init. stopDNSIntercept clears state, - // then startDNSIntercept creates everything fresh. - _ = p.stopDNSIntercept() - if err := p.startDNSIntercept(); err != nil { - mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to re-create WFP filters") - } + rebuildDNSInterceptFn(p, state, "WFP sublayer removed externally (delayed network recheck)") } }) } } +// repairMissingWFP re-creates the intercept when hard mode has no WFP enforcement - +// because the sublayer disappeared, or because the engine never opened in the first place. +// It reports whether the caller's monitor goroutine must stop. +func (p *prog) repairMissingWFP(state *wfpState) bool { + // Never interrogate WFP for a retired state: shutdown deletes our sublayer, so a + // tick landing in the shutdown window would read "missing" and try to rebuild what + // stopDNSIntercept just removed. + if p.interceptStateRevoked(state) { + return true + } + + reason := "" + switch { + case state.engineHandle == 0: + // In dns mode a closed engine is the normal state: loopback protect opens one + // only when it is needed. In hard mode it means startWFPFilters never got the + // engine open, so nothing is being blocked at all. That is the state a startup + // WFP failure leaves behind, and this is what retries it - without this entry + // point the process would run unenforced for its whole life. + if !hardIntercept { + return false + } + reason = "hard mode has no WFP engine - enforcement never started" + case wfpSublayerExists(state.engineHandle): + return false + default: + reason = "WFP sublayer missing during health check" + } + + mainLog.Load().Warn().Str("reason", reason).Msg("DNS intercept: WFP health check - re-initializing all filters") + if rebuildDNSInterceptFn(p, state, reason) == interceptRebuildDone { + mainLog.Load().Info().Msg("DNS intercept: WFP filters restored by health monitor") + } + return true +} + // nrptHealthMonitor periodically checks that the NRPT catch-all rule is still // present and re-adds it if removed by VPN software or Group Policy updates. // In hard mode, it also verifies the WFP sublayer exists and re-initializes // all filters if they were removed. +// +// One monitor belongs to one intercept state, and it exits when that state is retired: +// nothing here may act on state after stopDNSIntercept or a rebuild has moved on. func (p *prog) nrptHealthMonitor(state *wfpState) { ticker := time.NewTicker(30 * time.Second) defer ticker.Stop() @@ -1605,12 +2749,67 @@ func (p *prog) nrptHealthMonitor(state *wfpState) { case <-state.stopCh: return case <-ticker.C: - if !state.nrptActive { + // A tick and a shutdown can become ready together and select picks either, + // so re-check before doing any health work for a retired intercept. + if p.interceptStateRevoked(state) { + return + } + owner, externalRuleName := state.nrptPolicyOwner() + switch owner { + case nrptRuleOwnerNone: + // No owner means an earlier NRPT write failed. Nothing else re-arms + // it - nrptProbeAndHeal returns early without ctrld ownership, and the + // missing-rule branch below used to be unreachable from here - so + // without this retry the machine keeps no NRPT rule for the rest of the + // process lifetime. In hard mode that is a full DNS outage: WFP still + // blocks outbound DNS while nothing redirects it to ctrld. Fall through + // to the activation path below. + case nrptRuleOwnerGroupPolicy: + currentRule := p.nrptOps().findGPRule(state.listenerIP) + if currentRule == "" { + if p.activateCtrldNRPTFallback(state, "matching GP rule disappeared during health check") { + go p.nrptProbeAndHeal(state) + } + continue + } + // Confirm through the handback transition so the verdict is always about + // external policy alone, never about a ctrld rule left behind by an + // earlier run that happens to answer the probe. + switch p.nrptHandbackToExternal(state, currentRule, "GP-managed NRPT health check") { + case nrptHandbackVerified: + // Ownership and stable-success are recorded by the transition. + case nrptHandbackKeptCtrld: + // External policy could not carry DNS; ctrld owns the rule again. + // The next tick continues as ctrld-owned. + case nrptHandbackConflict: + // The administrator's catch-all no longer targets ctrld. Nothing to + // remediate: ctrld may not write beside it, and the conflict is + // already logged by the transition. + default: + mainLog.Load().Warn().Str("rule", externalRuleName). + Msg("DNS intercept: GP-managed NRPT rule is present but its probe failed; leaving external policy untouched") + // The only remediation allowed over external policy: a third-party + // WFP block dropping DNS below NRPT looks exactly like an ineffective + // rule. No policy refresh, paramchange or cache flush here - see + // healBlockedLoopbackDNS. + if !p.healBlockedLoopbackDNS(state, "GP-managed NRPT rule present but not routing") { + go p.nrptProbeAndHeal(state) + } + } + if p.repairMissingWFP(state) { + return + } continue + case nrptRuleOwnerCtrld: + // Group Policy may have returned after ctrld activated its fallback. + // Prefer the externally owned rule once it proves the same route without + // ctrld's rule installed. + if p.tryAdoptMatchingGPNRPT(state) { + continue + } } - // Step 1: Check registry key exists. - if !nrptCatchAllRuleExists() { + if nrptNeedsCtrldActivation(owner, nrptCatchAllRuleExists()) { now := time.Now() if ok, wait := state.nrptRecoveryLimiter.allow(now, p.cfg); !ok { if state.nrptRecoveryLimiter.shouldLogSkip(now) { @@ -1619,39 +2818,32 @@ func (p *prog) nrptHealthMonitor(state *wfpState) { } continue } - mainLog.Load().Warn().Msg("DNS intercept: NRPT health check — catch-all rule missing, restoring") - if err := addNRPTCatchAllRule(state.listenerIP); err != nil { - mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to restore NRPT catch-all rule") - state.nrptActive = false - continue + reason := "ctrld-owned rule missing during health check" + if owner == nrptRuleOwnerNone { + reason = "no NRPT owner during health check - retrying a failed activation" + } + if p.activateCtrldNRPTFallback(state, reason) { + state.nrptRecoveryLimiter.recordRecoveryFlow(time.Now(), p.cfg) + go p.nrptProbeAndHeal(state) + } else if owner == nrptRuleOwnerNone && hardIntercept { + // Worth shouting about: hard mode keeps blocking outbound DNS + // whether or not NRPT redirects it, so a machine stuck here has no + // working resolver until activation succeeds. + mainLog.Load().Error(). + Msg("DNS intercept: hard mode is blocking DNS but no NRPT rule could be activated - DNS will not resolve until this recovers") } - signalNRPTChange() - state.nrptRecoveryLimiter.recordRecoveryFlow(time.Now(), p.cfg) - mainLog.Load().Info().Msg("DNS intercept: NRPT catch-all rule restored by health monitor") - // After restoring, verify it's actually working. - go p.nrptProbeAndHeal() continue } - // Step 2: Registry key exists — verify NRPT is actually routing - // queries to ctrld (catches the async GP refresh race). - if !p.probeNRPT() { - mainLog.Load().Warn().Msg("DNS intercept: NRPT health check — rule present but probe failed, running heal cycle") - go p.nrptProbeAndHeal() + if !p.nrptOps().probe(state) { + mainLog.Load().Warn().Msg("DNS intercept: ctrld-owned NRPT rule present but probe failed, running heal cycle") + go p.nrptProbeAndHeal(state) } else { state.nrptRecoveryLimiter.recordStableSuccess() } - // Step 3: In hard mode, also verify WFP sublayer. - if state.engineHandle != 0 && !wfpSublayerExists(state.engineHandle) { - mainLog.Load().Warn().Msg("DNS intercept: WFP health check — sublayer missing, re-initializing all filters") - _ = p.stopDNSIntercept() - if err := p.startDNSIntercept(); err != nil { - mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to re-initialize after WFP sublayer loss") - } else { - mainLog.Load().Info().Msg("DNS intercept: WFP filters restored by health monitor") - } - return // stopDNSIntercept closed our stopCh; startDNSIntercept started a new monitor + if p.repairMissingWFP(state) { + return // our state was retired: shutdown, or a rebuild that started a new monitor } } } @@ -1680,24 +2872,24 @@ var nrptProbeRunning atomic.Bool // the Windows DNS Client service (via Go's net.Resolver / GetAddrInfoW). If ctrld // receives the query on its listener, NRPT is working. // -// Returns true if NRPT is verified working, false if the probe timed out. -func (p *prog) probeNRPT() bool { - if p.dnsInterceptState == nil { +// Returns true if NRPT is verified working, false if the probe timed out or a shutdown +// arrived first. Reporting an abandoned probe as "not working" is safe: every recovery +// action a failed probe can trigger checks for revocation before it writes anything. +// +// state is passed explicitly rather than read from p.dnsInterceptState so that startup +// can probe before it publishes, and so a probe belongs to exactly one intercept. +func (p *prog) probeNRPT(state *wfpState) bool { + if state == nil { return true } // Generate unique probe domain to defeat DNS caching. probeID := fmt.Sprintf("_nrpt-probe-%x.%s", rand.Uint32(), nrptProbeDomain) - // Register probe so DNS handler can detect and signal it. - // Reuse the same mechanism as macOS pf probes (pfProbeExpected/pfProbeCh). - probeCh := make(chan struct{}, 1) - p.pfProbeExpected.Store(probeID) - p.pfProbeCh.Store(&probeCh) - defer func() { - p.pfProbeExpected.Store("") - p.pfProbeCh.Store((*chan struct{})(nil)) - }() + // Register this attempt's own domain so overlapping probes - a health tick, a + // handback and a heal cycle can each have one out - cannot cancel each other. + probeCh, deregister := p.registerInterceptProbe(probeID) + defer deregister() mainLog.Load().Debug().Str("domain", probeID).Msg("DNS intercept: sending NRPT verification probe") @@ -1713,13 +2905,24 @@ func (p *prog) probeNRPT() bool { _, _ = resolver.LookupHost(ctx, probeID) }() - select { - case <-probeCh: - mainLog.Load().Debug().Str("domain", probeID).Msg("DNS intercept: NRPT probe received — interception verified") - return true - case <-ctx.Done(): - mainLog.Load().Debug().Str("domain", probeID).Msg("DNS intercept: NRPT probe timed out — interception not working") - return false + // Poll for a pending stop so a service stop never waits out the probe timeout on + // top of the lifecycle lock. + shutdownPoll := time.NewTicker(interceptRevocationPollInterval) + defer shutdownPoll.Stop() + for { + select { + case <-probeCh: + mainLog.Load().Debug().Str("domain", probeID).Msg("DNS intercept: NRPT probe received - interception verified") + return true + case <-ctx.Done(): + mainLog.Load().Debug().Str("domain", probeID).Msg("DNS intercept: NRPT probe timed out - interception not working") + return false + case <-shutdownPoll.C: + if p.interceptStateRevoked(state) { + mainLog.Load().Debug().Str("domain", probeID).Msg("DNS intercept: abandoning NRPT probe - shutdown in progress") + return false + } + } } } @@ -1730,7 +2933,7 @@ func (p *prog) probeNRPT() bool { // restarting the Dnscache service (which always fails on modern Windows because // Dnscache is a protected shared svchost service). func sendParamChange() { - if out, err := exec.Command("sc", "control", "dnscache", "paramchange").CombinedOutput(); err != nil { + if out, err := runBoundedNRPTExec("sc", "control", "dnscache", "paramchange"); err != nil { mainLog.Load().Debug().Err(err).Str("output", string(out)).Msg("DNS intercept: sc control dnscache paramchange failed") } else { mainLog.Load().Debug().Msg("DNS intercept: sent paramchange to Dnscache service") @@ -1814,25 +3017,83 @@ func logNRPTParentKeyState(context string) { // Dnscache paramchange cannot make local rules visible while GP mode is // selected by an empty GP parent. // 3. Otherwise, signal DNS Client with increasing backoff between probes. -func (p *prog) nrptProbeAndHeal() { - state, _ := p.dnsInterceptState.(*wfpState) - if state != nil { - now := time.Now() - if ok, wait := state.nrptRecoveryLimiter.allow(now, p.cfg); !ok { - if state.nrptRecoveryLimiter.shouldLogSkip(now) { - mainLog.Load().Warn().Dur("remaining", wait). - Msg("DNS intercept: NRPT recovery suppressed after repeated failed recovery flows") - } - return - } +// +// state is passed in rather than read from p.dnsInterceptState: this runs from the +// locked start path before anything is published, and it must keep acting on the +// intercept it was launched for and no other. +// It reports whether the cycle ended with a probe that reached ctrld. Callers that need a +// readiness answer - startup's synchronous verification - use that; the asynchronous +// callers ignore it. +func (p *prog) nrptProbeAndHeal(state *wfpState) bool { + if p.interceptStateRevoked(state) { + return false } - if !nrptProbeRunning.CompareAndSwap(false, true) { mainLog.Load().Debug().Msg("DNS intercept: NRPT probe already running, skipping") - return + return false } defer nrptProbeRunning.Store(false) + ops := p.nrptOps() + owner, externalRuleName := state.nrptPolicyOwner() + if owner == nrptRuleOwnerGroupPolicy { + currentRule := ops.findGPRule(state.listenerIP) + if currentRule == "" { + if !p.activateCtrldNRPTFallback(state, "matching GP rule disappeared before verification") { + return false + } + owner = nrptRuleOwnerCtrld + } else { + switch p.nrptHandbackToExternal(state, currentRule, "GP-managed NRPT verification") { + case nrptHandbackVerified: + mainLog.Load().Info().Str("rule", currentRule). + Msg("DNS intercept: GP-managed NRPT verified working") + return true + case nrptHandbackKeptCtrld: + // External policy could not carry DNS without ctrld's rule, which the + // transition has restored. Continue as the ctrld-owned flow below. + owner = nrptRuleOwnerCtrld + case nrptHandbackConflict: + // External policy owns the namespace and points elsewhere. Owned recovery + // must not write beside it, so this heal cycle ends here. + return false + default: + if ops.findGPRule(state.listenerIP) == "" { + // The GP child went away while we were looking at it. + if !p.activateCtrldNRPTFallback(state, "matching GP rule disappeared during verification") { + return false + } + owner = nrptRuleOwnerCtrld + break + } + // A matching external rule owns the GP store, so ctrld may not rewrite + // policy or signal the DNS Client here. Loopback WFP protect is the one + // ctrld-owned remediation left: a third-party WFP block dropping DNS + // below NRPT presents exactly as an ineffective rule. + if p.healBlockedLoopbackDNS(state, "GP-managed NRPT present but not routing") { + mainLog.Load().Info().Str("rule", currentRule). + Msg("DNS intercept: GP-managed NRPT verified after loopback WFP protection") + return true + } + mainLog.Load().Error().Str("rule", externalRuleName). + Msg("DNS intercept: GP-managed NRPT remains present but ineffective; no NRPT recovery actions were taken") + return false + } + } + } + if owner != nrptRuleOwnerCtrld { + return false + } + + now := time.Now() + if ok, wait := state.nrptRecoveryLimiter.allow(now, p.cfg); !ok { + if state.nrptRecoveryLimiter.shouldLogSkip(now) { + mainLog.Load().Warn().Dur("remaining", wait). + Msg("DNS intercept: NRPT recovery suppressed after repeated failed recovery flows") + } + return false + } + remediated := false defer func() { if remediated && state != nil { @@ -1846,9 +3107,16 @@ func (p *prog) nrptProbeAndHeal() { logNRPTParentKeyState("probe-start") // Attempt 1: immediate probe - if p.probeNRPT() { + if ops.probe(state) { mainLog.Load().Info().Msg("DNS intercept: NRPT verified working") - return + return true + } + // Group Policy can appear while a ctrld-owned heal is running. Once an exact + // administrator catch-all is there, this cycle stops: everything below signals the + // DNS Client or rewrites ctrld's rule, and neither may happen beside external policy. + if p.deferToExternalCatchAll(ops, state, "GP catch-all appeared during ctrld recovery") { + mainLog.Load().Warn().Msg("DNS intercept: matching GP catch-all present during ctrld recovery; stopping NRPT mutations and deferring to Group Policy") + return false } remediated = true @@ -1856,20 +3124,22 @@ func (p *prog) nrptProbeAndHeal() { // signaling. Those retries create SIEM noise but cannot succeed because DNS // Client is still reading the empty GP store instead of the populated local // store. Delete the blocker, send one notification, then re-probe. - if nrptParentKeyEmpty(nrptBaseKey) { + if ops.parentEmpty(nrptBaseKey) { mainLog.Load().Warn().Msg("DNS intercept: NRPT probe failed with empty GP parent — cleaning before retry signaling") - if cleanEmptyNRPTParent() { - signalNRPTChange() - time.Sleep(1 * time.Second) + if ops.cleanParent() { + ops.signal() + if !ops.wait(state, nrptHandbackSettleDelay) { + return false + } logNRPTParentKeyState("empty-gp-after-clean") - if p.probeNRPT() { + if ops.probe(state) { mainLog.Load().Info().Msg("DNS intercept: NRPT verified working after empty GP parent cleanup") - return + return true } } - if nrptParentKeyEmpty(nrptBaseKey) { + if ops.parentEmpty(nrptBaseKey) { mainLog.Load().Warn().Msg("DNS intercept: empty GP NRPT parent still present after cleanup; skipping redundant policy refresh retries") - return + return false } } @@ -1877,50 +3147,134 @@ func (p *prog) nrptProbeAndHeal() { delays := []time.Duration{1 * time.Second, 2 * time.Second, 4 * time.Second} for i, delay := range delays { attempt := i + 2 + // Each round signals Windows and then waits; a stop must not have the whole + // remaining backoff added to the time it waits for the lifecycle lock. + if p.interceptStateRevoked(state) { + mainLog.Load().Debug().Int("attempt", attempt). + Msg("DNS intercept: intercept retired - abandoning NRPT probe retries") + return false + } mainLog.Load().Info().Int("attempt", attempt).Dur("delay", delay). Msg("DNS intercept: NRPT probe failed, retrying with policy refresh + paramchange") logNRPTParentKeyState(fmt.Sprintf("probe-attempt-%d", attempt)) - signalNRPTChange() - time.Sleep(delay) - if p.probeNRPT() { + ops.signal() + if !ops.wait(state, delay) { + return false + } + if ops.probe(state) { mainLog.Load().Info().Int("attempt", attempt). Msg("DNS intercept: NRPT verified working") - return + return true } } + // Re-check external ownership before the destructive two-phase recovery. A GP refresh + // can land during the bounded retry waits above, and the delete half of that recovery + // must not run once it has: it would strand the machine mid-recovery under policy + // ctrld does not own. + if p.deferToExternalCatchAll(ops, state, "GP catch-all appeared during ctrld retries") { + mainLog.Load().Warn().Msg("DNS intercept: matching GP catch-all present after the retries; skipping ctrld two-phase recovery") + return false + } + if gpCatchAllConflictBlocksFallback(state, "GP catch-all changed during ctrld recovery") { + return false + } + + // A stop can land during the retry waits above, and its own NRPT removal has + // already run: deleting and re-adding from here would leave the rule behind. + if p.interceptStateRevoked(state) { + mainLog.Load().Debug().Msg("DNS intercept: intercept retired during probe retries - skipping two-phase NRPT recovery") + return false + } + // Nuclear option: two-phase delete → re-add cycle. // DNS Client may have cached a stale "no rules" state. Delete our rule, // signal DNS Client to forget it, wait, then re-add and signal again. mainLog.Load().Warn().Msg("DNS intercept: all probes failed — attempting two-phase NRPT recovery (delete → signal → re-add)") - listenerIP := "127.0.0.1" - if state != nil { - listenerIP = state.listenerIP + listenerIP := state.listenerIP + + // Phase 1: Remove our rule and the parent key if now empty. Each phase is its own + // transition: serialized against a stop and against other NRPT writers, but the lock + // is released across the wait between them. + if !p.nrptTransition(state, func() { + _ = ops.removeRule() + // If parent key is now empty after removing our rule, delete it too. + ops.cleanParent() + ops.signal() + logNRPTParentKeyState("nuclear-after-delete") + }) { + mainLog.Load().Debug().Msg("DNS intercept: intercept retired - skipping two-phase NRPT recovery") + return false } - // Phase 1: Remove our rule and the parent key if now empty. - _ = removeNRPTCatchAllRule() - // If parent key is now empty after removing our rule, delete it too. - cleanEmptyNRPTParent() - signalNRPTChange() - logNRPTParentKeyState("nuclear-after-delete") + // Wait for DNS Client to process the deletion. Stopping here is the clean outcome: + // phase 1 left no ctrld rule behind. + if !ops.wait(state, nrptHandbackSettleDelay) { + mainLog.Load().Debug().Msg("DNS intercept: intercept retired mid-recovery - leaving NRPT removed") + return false + } - // Wait for DNS Client to process the deletion. - time.Sleep(1 * time.Second) + // Group Policy may refresh while the ctrld-owned rule is absent. Never re-create our + // catch-all beside a newly authoritative GP catch-all. With ctrld's rule already + // gone, this is the one place a probe measures external policy on its own. + if p.deferToExternalCatchAll(ops, state, "GP catch-all appeared during two-phase recovery") { + mainLog.Load().Warn().Msg("DNS intercept: matching GP catch-all appeared during two-phase recovery; skipping ctrld re-add") + return false + } + if ops.gpConflicts(state, "GP catch-all appeared during two-phase recovery") { + return false + } - // Phase 2: Re-add the rule. - if err := addNRPTCatchAllRule(listenerIP); err != nil { - mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to re-add NRPT after nuclear recovery") - return + // Phase 2: Re-add the rule. Phase 1 left NRPT clean, so if a stop arrived during the + // wait the transition refuses and the host stays clean. + readded := false + if !p.nrptTransition(state, func() { + // Re-validate the ownership picture here, inside the lock. The checks above ran + // before this transition queued for nrptTransitionMu, and another health or + // delayed path can complete a whole handback while this one waits: it may have + // given the namespace to a GP child that arrived meanwhile and recorded + // GroupPolicy. Re-adding on top of that would plant the competing catch-all this + // work exists to prevent, and then stamp ctrld ownership over the administrator's. + if owner, ruleName := state.nrptPolicyOwner(); owner == nrptRuleOwnerGroupPolicy { + mainLog.Load().Warn().Str("rule", ruleName). + Msg("DNS intercept: ownership moved to Group Policy while the two-phase re-add waited for the transition lock; leaving NRPT to external policy") + return + } + if ruleName := ops.findGPRule(state.listenerIP); ruleName != "" && + ops.gpRuleMatches(ruleName, state.listenerIP) { + mainLog.Load().Warn().Str("rule", ruleName). + Msg("DNS intercept: a matching GP catch-all appeared while the two-phase re-add waited for the transition lock; skipping the ctrld re-add") + return + } + if ops.gpConflicts(state, "two-phase re-add revalidation") { + return + } + if err := ops.addRule(listenerIP); err != nil { + mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to re-add NRPT after nuclear recovery") + return + } + ops.signal() + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + logNRPTParentKeyState("nuclear-after-readd") + readded = true + }) || !readded { + mainLog.Load().Debug().Msg("DNS intercept: two-phase recovery did not restore the ctrld NRPT rule") + return false } - signalNRPTChange() - logNRPTParentKeyState("nuclear-after-readd") // Final probe after recovery. - time.Sleep(1 * time.Second) - if p.probeNRPT() { + if !ops.wait(state, nrptHandbackSettleDelay) { + // This cycle is retired. Do not clean up from here: the deterministic ctrld key is + // process-global, and by now it can belong to a successor intercept that a rebuild + // started while this goroutine waited - deleting it would take out the successor's + // route and leave hard mode blocking DNS with nothing redirecting it. Teardown of + // the state this cycle belonged to already owns that cleanup. + mainLog.Load().Debug().Msg("DNS intercept: intercept retired after the two-phase re-add - leaving cleanup to the owning teardown") + return false + } + if ops.probe(state) { mainLog.Load().Info().Msg("DNS intercept: NRPT verified working after two-phase recovery") - return + return true } logNRPTParentKeyState("probe-failed-final") @@ -1932,32 +3286,27 @@ func (p *prog) nrptProbeAndHeal() { // loopback. A high-priority "hard permit" for localhost DNS overrides these // blocks and restores NRPT routing to ctrld's listener. // See: https://gitlab.int.windscribe.com/controld/clients/ctrld/-/issues/526 - loopbackState, ok := p.dnsInterceptState.(*wfpState) - if !ok || loopbackState == nil { - mainLog.Load().Error().Msg("DNS intercept: no state available for loopback WFP protect") - return - } - // Bail out if shutdown is in progress — avoid racing with cleanupWFPFilters. - select { - case <-loopbackState.stopCh: + if p.interceptStateRevoked(state) { mainLog.Load().Info().Msg("DNS intercept: shutdown in progress, skipping loopback WFP protect activation") - return - default: + return false } - if err := p.activateLoopbackWFPProtect(loopbackState); err != nil { + if err := p.activateLoopbackWFPProtect(state); err != nil { mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to activate loopback WFP protect — " + "DNS queries may not be routed through ctrld. A network interface toggle may be needed.") - return + return false } // Retry NRPT probe now that loopback DNS is explicitly permitted through WFP. - time.Sleep(500 * time.Millisecond) - if p.probeNRPT() { + if !ops.wait(state, loopbackProtectSettleDelay) { + return false + } + if ops.probe(state) { mainLog.Load().Info().Msg("DNS intercept: NRPT verified working after loopback WFP protect activation") - return + return true } mainLog.Load().Error().Msg("DNS intercept: NRPT probe still failing after loopback WFP protect — " + "DNS queries may not be routed through ctrld. A network interface toggle may be needed.") + return false } diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index 6a64882..b34013c 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -130,13 +130,7 @@ func (p *prog) serveDNS(listenerNum string) error { // signal the prober and respond NXDOMAIN. Used by both macOS pf probes // (_pf-probe-*) and Windows NRPT probes (_nrpt-probe-*) to verify that // DNS interception is actually routing queries to ctrld's listener. - if probeID, ok := p.pfProbeExpected.Load().(string); ok && probeID != "" && domain == probeID { - if chPtr, ok := p.pfProbeCh.Load().(*chan struct{}); ok && chPtr != nil { - select { - case *chPtr <- struct{}{}: - default: - } - } + if p.signalInterceptProbe(domain) { answer := new(dns.Msg) answer.SetRcode(m, dns.RcodeNameError) // NXDOMAIN _ = w.WriteMsg(answer) diff --git a/cmd/cli/intercept_probe.go b/cmd/cli/intercept_probe.go new file mode 100644 index 0000000..36e5707 --- /dev/null +++ b/cmd/cli/intercept_probe.go @@ -0,0 +1,68 @@ +package cli + +// Interception probe registry. +// +// A probe sends a DNS query for a unique synthetic domain through the OS resolver and +// waits for ctrld's own handler to receive it. That is the only way to tell "the rules are +// present" from "the rules are actually redirecting packets", and both the macOS pf path +// and the Windows NRPT path use it. +// +// Each attempt registers its own domain, so overlapping probes cannot cancel each other, +// and deregistration only removes the entry it owns. + +// registerInterceptProbe registers domain and returns the channel it will be signalled on +// plus the function that removes the registration. +// +//lint:ignore U1000 used on darwin (pf probes) and windows (NRPT probes) +func (p *prog) registerInterceptProbe(domain string) (<-chan struct{}, func()) { + ch := make(chan struct{}, 1) + + p.interceptProbeMu.Lock() + current, _ := p.interceptProbes.Load().(map[string]chan struct{}) + next := make(map[string]chan struct{}, len(current)+1) + for k, v := range current { + next[k] = v + } + next[domain] = ch + p.interceptProbes.Store(next) + p.interceptProbeMu.Unlock() + + return ch, func() { + p.interceptProbeMu.Lock() + defer p.interceptProbeMu.Unlock() + current, _ := p.interceptProbes.Load().(map[string]chan struct{}) + // Only drop the entry while it is still this attempt's channel. A later probe + // that reused the domain owns the slot now, and clearing it would make that one + // wait out its timeout for a query it already received. + if existing, ok := current[domain]; !ok || existing != ch { + return + } + next := make(map[string]chan struct{}, len(current)) + for k, v := range current { + if k != domain { + next[k] = v + } + } + p.interceptProbes.Store(next) + } +} + +// signalInterceptProbe reports whether domain is a pending probe, signalling its waiter +// when it is. Called from the DNS handler for every query, so the common case is a nil or +// empty map and no allocation. +func (p *prog) signalInterceptProbe(domain string) bool { + probes, _ := p.interceptProbes.Load().(map[string]chan struct{}) + if len(probes) == 0 { + return false + } + ch, ok := probes[domain] + if !ok { + return false + } + select { + case ch <- struct{}{}: + default: + // Buffered channel already holds a signal: the waiter has what it needs. + } + return true +} diff --git a/cmd/cli/nrpt_external_gp.go b/cmd/cli/nrpt_external_gp.go new file mode 100644 index 0000000..62c57f8 --- /dev/null +++ b/cmd/cli/nrpt_external_gp.go @@ -0,0 +1,63 @@ +package cli + +import ( + "errors" + "net/netip" + "strings" +) + +const nrptRuleName = `CtrldCatchAll` + +// errGPNRPTVerified marks an intercept startup failure that happened while an externally +// managed (Group Policy) NRPT catch-all was proved - by probe, not by registry shape +// alone - to be routing DNS to this listener. It is the difference between "intercept +// failed but DNS still reaches ctrld" and "intercept failed and nothing is filtering", +// which is what decides whether the interface-DNS fallback must run. +// +// Only the Windows path produces it, but setDNS is shared, so the sentinel and its +// predicate live here with the other platform-neutral NRPT helpers. +var errGPNRPTVerified = errors.New("GP-managed NRPT verified routing to ctrld") + +// errGPNRPTIneffective marks a startup that ends with externally managed NRPT owning the +// namespace while no probe has proved it routes to ctrld. DNS is not reaching ctrld, but +// adapter DNS was deliberately preserved and no ctrld rule may be written beside an +// administrator's catch-all - so this is a failed start that must not take the +// interface-DNS fallback either. +var errGPNRPTIneffective = errors.New("GP-managed NRPT owns the namespace but no probe reached ctrld") + +// interceptFailedWithVerifiedExternalDNS reports whether an intercept startup failure +// happened while externally managed DNS policy was verified to be routing to ctrld. +func interceptFailedWithVerifiedExternalDNS(err error) bool { + return errors.Is(err, errGPNRPTVerified) +} + +// interceptFailedUnderExternalDNSPolicy reports whether an intercept startup failure +// happened while externally managed DNS policy owned the namespace, whether or not it was +// proved to route. Either way the interface-DNS fallback must not run: adapter DNS was +// preserved on purpose, and rewriting it would violate the policy ctrld just deferred to. +// Only the verified case is a successful start. +func interceptFailedUnderExternalDNSPolicy(err error) bool { + return errors.Is(err, errGPNRPTVerified) || errors.Is(err, errGPNRPTIneffective) +} + +// isExternalGPCatchAll recognizes only a single catch-all namespace that is not +// ctrld's deterministic GP key. Registry access stays in the Windows file; this +// pure classifier is shared with host-runnable tests. +func isExternalGPCatchAll(ruleName string, namespaces []string) bool { + return ruleName != "" && !strings.EqualFold(ruleName, nrptRuleName) && len(namespaces) == 1 && strings.TrimSpace(namespaces[0]) == "." +} + +func isMatchingGPNRPTRule(ruleName string, namespaces []string, dnsServers, listenerIP string) bool { + if !isExternalGPCatchAll(ruleName, namespaces) { + return false + } + server, err := netip.ParseAddr(strings.TrimSpace(dnsServers)) + if err != nil { + return false + } + listener, err := netip.ParseAddr(strings.TrimSpace(listenerIP)) + if err != nil { + return false + } + return server.Unmap() == listener.Unmap() +} diff --git a/cmd/cli/nrpt_external_gp_test.go b/cmd/cli/nrpt_external_gp_test.go new file mode 100644 index 0000000..277581b --- /dev/null +++ b/cmd/cli/nrpt_external_gp_test.go @@ -0,0 +1,129 @@ +package cli + +import ( + "errors" + "fmt" + "testing" +) + +func TestIsMatchingGPNRPTRule(t *testing.T) { + tests := []struct { + name string + ruleName string + namespaces []string + servers string + listener string + want bool + }{ + { + name: "exact IPv4 catch-all", + ruleName: "{A1B2C3D4}", + namespaces: []string{"."}, + servers: "127.0.0.1", + listener: "127.0.0.1", + want: true, + }, + { + name: "normalized IPv4-mapped listener", + ruleName: "{A1B2C3D4}", + namespaces: []string{"."}, + servers: "::ffff:127.0.0.1", + listener: "127.0.0.1", + want: true, + }, + { + name: "ctrld GP key is not external", + ruleName: "ctrldcatchall", + namespaces: []string{"."}, + servers: "127.0.0.1", + listener: "127.0.0.1", + }, + { + name: "partial namespace", + ruleName: "{A1B2C3D4}", + namespaces: []string{"corp.example"}, + servers: "127.0.0.1", + listener: "127.0.0.1", + }, + { + name: "multiple namespaces", + ruleName: "{A1B2C3D4}", + namespaces: []string{".", "corp.example"}, + servers: "127.0.0.1", + listener: "127.0.0.1", + }, + { + name: "wrong listener", + ruleName: "{A1B2C3D4}", + namespaces: []string{"."}, + servers: "127.0.0.2", + listener: "127.0.0.1", + }, + { + name: "multiple nameservers", + ruleName: "{A1B2C3D4}", + namespaces: []string{"."}, + servers: "127.0.0.1;127.0.0.2", + listener: "127.0.0.1", + }, + { + name: "malformed nameserver", + ruleName: "{A1B2C3D4}", + namespaces: []string{"."}, + servers: "localhost", + listener: "127.0.0.1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isMatchingGPNRPTRule(tt.ruleName, tt.namespaces, tt.servers, tt.listener); got != tt.want { + t.Fatalf("isMatchingGPNRPTRule() = %t, want %t", got, tt.want) + } + }) + } +} + +func TestIsExternalGPCatchAll(t *testing.T) { + tests := []struct { + name string + ruleName string + namespaces []string + want bool + }{ + {name: "external catch-all", ruleName: "{GP-RULE}", namespaces: []string{"."}, want: true}, + {name: "ctrld key", ruleName: nrptRuleName, namespaces: []string{"."}}, + {name: "partial namespace", ruleName: "{GP-RULE}", namespaces: []string{"corp.example"}}, + {name: "multiple namespaces", ruleName: "{GP-RULE}", namespaces: []string{".", "corp.example"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isExternalGPCatchAll(tt.ruleName, tt.namespaces); got != tt.want { + t.Fatalf("isExternalGPCatchAll() = %t, want %t", got, tt.want) + } + }) + } +} + +// TestInterceptFailedWithVerifiedExternalDNS covers the distinction the interface-DNS +// fallback turns on. "A GP rule exists" is not enough: if it is not actually routing and +// intercept failed too, skipping the fallback leaves the machine with no NRPT, no WFP and +// no adapter DNS - that is, unfiltered. Only a probe-verified route earns the skip. +func TestInterceptFailedWithVerifiedExternalDNS(t *testing.T) { + wfpErr := errors.New("FwpmEngineOpen0 failed: HRESULT 0x5") + + verified := fmt.Errorf("dns intercept: WFP setup failed: %w: %w", wfpErr, errGPNRPTVerified) + if !interceptFailedWithVerifiedExternalDNS(verified) { + t.Error("a failure carrying errGPNRPTVerified must skip the interface-DNS fallback") + } + if !errors.Is(verified, wfpErr) { + t.Error("the underlying cause must stay inspectable for logs and callers") + } + + if interceptFailedWithVerifiedExternalDNS(fmt.Errorf("dns intercept: WFP setup failed: %w", wfpErr)) { + t.Error("an unverified failure must take the interface-DNS fallback rather than leave the machine unfiltered") + } + if interceptFailedWithVerifiedExternalDNS(nil) { + t.Error("no error must not read as a verified external route") + } +} diff --git a/cmd/cli/nrpt_external_gp_windows_test.go b/cmd/cli/nrpt_external_gp_windows_test.go new file mode 100644 index 0000000..3d402d7 --- /dev/null +++ b/cmd/cli/nrpt_external_gp_windows_test.go @@ -0,0 +1,20 @@ +//go:build windows + +package cli + +import "testing" + +func TestWFPStateNRPTPolicyOwner(t *testing.T) { + state := &wfpState{} + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, "{GP-RULE}") + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE}" { + t.Fatalf("owner = %v, rule = %q", owner, ruleName) + } + + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + owner, ruleName = state.nrptPolicyOwner() + if owner != nrptRuleOwnerCtrld || ruleName != "" { + t.Fatalf("owner = %v, rule = %q", owner, ruleName) + } +} diff --git a/cmd/cli/nrpt_handback_windows_test.go b/cmd/cli/nrpt_handback_windows_test.go new file mode 100644 index 0000000..153376a --- /dev/null +++ b/cmd/cli/nrpt_handback_windows_test.go @@ -0,0 +1,1309 @@ +//go:build windows + +package cli + +import ( + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// fakeNRPTOps records the Windows side effects a transition would cause and serves +// scripted probe results, so the ownership decisions can be driven without touching the +// registry or the DNS Client. +type fakeNRPTOps struct { + mu sync.Mutex + + // probeResults is consumed in order; the last value repeats once exhausted. + probeResults []bool + probeCalls int + flushCalls int + + gpRule string // rule name findGPRule reports, "" for none + gpConflict bool // whether an external catch-all targets another resolver + parentEmpty bool // whether the GP parent key reads as present but empty + cleanCalls int + gpConflictCalls int + ctrldRule bool // whether ctrld's own catch-all exists + existsCalls int + addCalls int + removeCalls int + signalCalls int + + addErr error + removeErr error + + // beforeAdd runs inside addRule, while the transition lock is held. + beforeAdd func() + // onProbe runs after the nth probe (1-based) is answered, so a test can change the + // GP store mid-probe the way a Group Policy refresh would. + onProbe func(call int) + // onGPConflicts runs after each gpConflicts check, which is the last ops call before + // the two-phase re-add queues for the transition lock. + onGPConflicts func(call int) + // waitHook stands in for a cancellable wait: returning false models the intercept + // being retired mid-wait. + waitHook func() bool +} + +func (f *fakeNRPTOps) ops() *nrptOps { + return &nrptOps{ + probe: func(*wfpState) bool { + f.mu.Lock() + f.probeCalls++ + call := f.probeCalls + result := false + if len(f.probeResults) > 0 { + result = f.probeResults[0] + if len(f.probeResults) > 1 { + f.probeResults = f.probeResults[1:] + } + } + onProbe := f.onProbe + f.mu.Unlock() + if onProbe != nil { + onProbe(call) + } + return result + }, + ruleExists: func() bool { + f.mu.Lock() + defer f.mu.Unlock() + f.existsCalls++ + return f.ctrldRule + }, + addRule: func(string) error { + f.mu.Lock() + beforeAdd, err := f.beforeAdd, f.addErr + f.addCalls++ + if err == nil { + f.ctrldRule = true + } + f.mu.Unlock() + if beforeAdd != nil { + beforeAdd() + } + return err + }, + removeRule: func() error { + f.mu.Lock() + defer f.mu.Unlock() + f.removeCalls++ + if f.removeErr != nil { + return f.removeErr + } + f.ctrldRule = false + return nil + }, + signal: func() { + f.mu.Lock() + defer f.mu.Unlock() + f.signalCalls++ + }, + findGPRule: func(string) string { + f.mu.Lock() + defer f.mu.Unlock() + return f.gpRule + }, + gpRuleMatches: func(ruleName, _ string) bool { + f.mu.Lock() + defer f.mu.Unlock() + return ruleName != "" && ruleName == f.gpRule + }, + gpConflicts: func(*wfpState, string) bool { + f.mu.Lock() + f.gpConflictCalls++ + call := f.gpConflictCalls + conflict, hook := f.gpConflict, f.onGPConflicts + f.mu.Unlock() + if hook != nil { + hook(call) + } + return conflict + }, + loopback: func(*wfpState) error { return nil }, + // No real waiting in tests; report "still live" unless a test says otherwise. + wait: func(*wfpState, time.Duration) bool { + f.mu.Lock() + hook := f.waitHook + f.mu.Unlock() + if hook != nil { + return hook() + } + return true + }, + startWFP: func(*wfpState) error { return nil }, + flush: func() { + f.mu.Lock() + defer f.mu.Unlock() + f.flushCalls++ + }, + parentEmpty: func(string) bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.parentEmpty + }, + cleanParent: func() bool { + f.mu.Lock() + defer f.mu.Unlock() + f.cleanCalls++ + wasEmpty := f.parentEmpty + f.parentEmpty = false + return wasEmpty + }, + } +} + +func (f *fakeNRPTOps) counts() (add, remove, signal, probe int) { + f.mu.Lock() + defer f.mu.Unlock() + return f.addCalls, f.removeCalls, f.signalCalls, f.probeCalls +} + +func (f *fakeNRPTOps) ruleExistsCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.existsCalls +} + +func (f *fakeNRPTOps) flushCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.flushCalls +} + +// setGPRule replaces what the fake GP store reports, as a policy refresh would. +func (f *fakeNRPTOps) setGPRule(ruleName string, conflicting bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.gpRule = ruleName + f.gpConflict = conflicting +} + +// hasCtrldRule reports whether ctrld's catch-all is currently installed. +func (f *fakeNRPTOps) hasCtrldRule() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.ctrldRule +} + +// installedFakeNRPTOps is the fake currently standing in for the production NRPT +// operations, so fixtures can hand it to tests and can tell whether one is installed at +// all. +var installedFakeNRPTOps *fakeNRPTOps + +// installFakeNRPTOps swaps in the fake for the duration of the test. +func installFakeNRPTOps(t *testing.T, f *fakeNRPTOps) { + t.Helper() + nrptOpsForTest = f.ops() + installedFakeNRPTOps = f + t.Cleanup(func() { + nrptOpsForTest = nil + installedFakeNRPTOps = nil + }) +} + +// fakeNRPTSeed is the starting state a test wants the fixture's fake to have. It is a +// separate type so the fake - which carries a mutex - is never copied. +type fakeNRPTSeed struct { + probeResults []bool + gpRule string + gpConflict bool + parentEmpty bool + ctrldRule bool + addErr error + removeErr error +} + +// configure re-arms the fake the fixture installed with a test's own starting state, so +// the fixture keeps ownership of installation - and therefore of the production-call guard. +func (f *fakeNRPTOps) configure(seed fakeNRPTSeed) { + f.mu.Lock() + defer f.mu.Unlock() + f.probeResults = seed.probeResults + f.gpRule = seed.gpRule + f.gpConflict = seed.gpConflict + f.parentEmpty = seed.parentEmpty + f.ctrldRule = seed.ctrldRule + f.addErr = seed.addErr + f.removeErr = seed.removeErr +} + +// requireFakeNRPTOpsInstalled proves the fake is the table production code will actually +// consult, and fails the test immediately if it is not. +// +// Identity is not enough on its own, and neither is asserting side-effect counters after +// the fact: a fixture that never installed the fake, or a seam that stopped consulting the +// override, leaves those counters at zero and every "no side effects" assertion passes +// while the real registry and signalling functions run. So this calls through +// prog.nrptOps() and requires the call to land on the fake. The probe is only made once an +// override is known to exist, so it can never reach the host itself. +func requireFakeNRPTOpsInstalled(t *testing.T, f *fakeNRPTOps) { + t.Helper() + if err := checkFakeNRPTOpsInstalled(f); err != nil { + t.Fatal(err) + } +} + +// checkFakeNRPTOpsInstalled is the precondition itself, as a predicate so it can be tested +// like any other logic - see TestFakeNRPTOpsPreconditionDetectsBypass. It returns nil only +// when f is the table prog.nrptOps() hands out. +func checkFakeNRPTOpsInstalled(f *fakeNRPTOps) error { + if nrptOpsForTest == nil { + return errors.New("fake NRPT operations are not installed: production registry and signalling functions would run") + } + if installedFakeNRPTOps != f { + return errors.New("the installed fake is not the one this fixture returned") + } + before := f.ruleExistsCount() + _ = (&prog{}).nrptOps().ruleExists() + if f.ruleExistsCount() == before { + return errors.New("prog.nrptOps() did not route to the installed fake: the seam is not in effect") + } + return nil +} + +// fakeNRPTOpsForTest returns the installed fake, installing a default one if the test has +// not already done so. +// +// Every fixture goes through this before publishing state, so no test can reach the +// production registry and signalling functions. That matters because those are not +// read-only: on a host where ctrld's deterministic key exists, one unfaked call can delete +// live NRPT policy and force a Group Policy refresh, a Dnscache paramchange and a cache +// flush. A clean runner is not a safety boundary. +func fakeNRPTOpsForTest(t *testing.T) *fakeNRPTOps { + t.Helper() + if installedFakeNRPTOps != nil { + return installedFakeNRPTOps + } + f := &fakeNRPTOps{} + installFakeNRPTOps(t, f) + return f +} + +func newHandbackTestProg(t *testing.T) (*prog, *wfpState, *fakeNRPTOps) { + t.Helper() + // Never let a test fall through to the production NRPT operations, and prove it here + // rather than trusting that installation happened. + f := fakeNRPTOpsForTest(t) + requireFakeNRPTOpsInstalled(t, f) + state := &wfpState{stopCh: make(chan struct{}), listenerIP: "127.0.0.1"} + p := &prog{} + p.dnsInterceptState = state + return p, state, f +} + +// TestHandbackRestoresFallbackWhenGPCannotRoute is the behaviour test for the handback +// contract. The first probe runs while ctrld's fallback is still installed, so that +// fallback can satisfy it; only a probe taken after ctrld's keys are gone says anything +// about the GP rule. With results [true, false] - the GP child looks fine until ctrld +// steps aside - the transition must put ctrld's rule back and keep ctrld ownership. +// +// Getting this wrong deletes the last working route and declares external ownership: in +// hard mode a machine-wide DNS outage, because WFP keeps blocking outbound DNS with +// nothing redirecting it to ctrld. +func TestHandbackRestoresFallbackWhenGPCannotRoute(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.tryAdoptMatchingGPNRPT(state); got { + t.Error("tryAdoptMatchingGPNRPT() = true for a GP rule that cannot route without ctrld's rule") + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("NRPT owner = %v, want nrptRuleOwnerCtrld: ownership must not move to a rule that failed the post-removal probe", owner) + } + add, remove, _, _ := f.counts() + if remove != 1 { + t.Errorf("removeRule calls = %d, want 1: the handback probe must run with ctrld's keys gone", remove) + } + if add != 1 { + t.Errorf("addRule calls = %d, want 1: the ctrld fallback must be restored after the probe fails", add) + } + if !f.hasCtrldRule() { + t.Error("ctrld's NRPT rule is missing after a failed handback; the host has no working route") + } +} + +// TestHandbackAcceptsGPWhenItRoutesWithoutCtrld is the other half of the contract: when +// the post-removal probe passes, external policy really is carrying DNS, so ctrld hands +// ownership over and leaves its keys off the machine. +func TestHandbackAcceptsGPWhenItRoutesWithoutCtrld(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, true}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if !p.tryAdoptMatchingGPNRPT(state) { + t.Fatal("tryAdoptMatchingGPNRPT() = false for a GP rule that routes on its own") + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{GP-RULE}", owner, ruleName) + } + add, remove, _, _ := f.counts() + if remove != 1 || add != 0 { + t.Errorf("removeRule = %d, addRule = %d, want 1/0: ctrld's rule must be removed and not restored", remove, add) + } + if f.hasCtrldRule() { + t.Error("ctrld's rule is still installed beside the adopted GP catch-all") + } +} + +// TestIneffectiveGPRuleTriggersNoSignalling holds ctrld to #576's contract for a +// present-but-ineffective external rule: warn and retry WFP-only, with no policy +// refresh, no Dnscache paramchange and no cache flush. signalNRPTChange is all three at +// once, so on a permanently dead GP rule a signalling loop would force machine Group +// Policy on a schedule. +func TestIneffectiveGPRuleTriggersNoSignalling(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false}, gpRule: "{GP-RULE}"} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, "{GP-RULE}") + + // The verification path a health tick and the heal cycle both take. + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackUnverified { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackUnverified", got) + } + if p.healBlockedLoopbackDNS(state, "test") { + t.Error("healBlockedLoopbackDNS() = true although the probe never succeeds") + } + + add, remove, signal, _ := f.counts() + if signal != 0 { + t.Errorf("signal calls = %d, want 0: ctrld must not force GP refresh, paramchange or a cache flush while external policy owns NRPT", signal) + } + if add != 0 || remove != 0 { + t.Errorf("addRule = %d, removeRule = %d, want 0/0: external policy must be left untouched", add, remove) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerGroupPolicy { + t.Errorf("owner = %v, want nrptRuleOwnerGroupPolicy: ctrld must not seize a namespace an administrator owns", owner) + } +} + +// TestConcurrentFallbackActivationWritesOnce drives two health paths - the monitor and a +// delayed recheck - into activation at the same moment. Without a transition lock they +// both observe "rule missing" and both write and signal. +func TestConcurrentFallbackActivationWritesOnce(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true}} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + // Hold both callers at the barrier until each is inside activateCtrldNRPTFallback. + start := make(chan struct{}) + var wg sync.WaitGroup + results := make([]bool, 2) + for i := range results { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + results[i] = p.activateCtrldNRPTFallback(state, "concurrent test") + }(i) + } + close(start) + wg.Wait() + + add, _, signal, _ := f.counts() + if add != 1 { + t.Errorf("addRule calls = %d, want 1: concurrent activations must not both write the catch-all", add) + } + if signal != 1 { + t.Errorf("signal calls = %d, want 1: the DNS Client must be signalled once per transition", signal) + } + if results[0] == results[1] { + t.Errorf("both callers reported %v; exactly one transition should report having written the rule", results[0]) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("owner = %v, want nrptRuleOwnerCtrld", owner) + } +} + +// waitFor polls cond until it holds, failing the test if it never does. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(time.Millisecond) + } +} + +// TestActivationLosingRaceWithStopWritesNothing is the interleaving a revocation check +// alone cannot catch: the activation passes its check, and the stop then revokes the +// state, removes NRPT and finishes before the write lands. Windows would be left routing +// every query to a listener that is gone. +// +// The test asserts the transition lock is load-bearing, in three steps. The barrier sits +// inside addRule, so the stop starts while a transition is mid-write; the stop is then +// shown to be unable to progress past its own NRPT cleanup while that transition holds +// the lock; and once the transition releases, the stop must actually have removed the +// rule before a later activation attempt is refused. Dropping the lock on either side - +// activation or stop cleanup - makes the second step fail, because the stop's removeRule +// then runs while the transition is still in flight. +func TestActivationLosingRaceWithStopWritesNothing(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true}} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + stopDone := make(chan struct{}) + f.beforeAdd = func() { + // Runs with the transition lock held, from the test goroutine. + go func() { + defer close(stopDone) + _ = p.stopDNSIntercept() + }() + + // The stop closes stopCh before it reaches the transition lock, so once that is + // closed the stop can only be waiting on this transition. Waiting on stopCh + // rather than interceptStateRevoked matters: the latter also reports the + // stop-requested flag, which is set before the stop has done anything. + waitFor(t, "the stop to revoke the intercept state", func() bool { + select { + case <-state.stopCh: + return true + default: + return false + } + }) + + // Give it a window in which it would visibly proceed if the lock were not held, + // then prove it did not: no NRPT removal, and the stop has not finished. + time.Sleep(100 * time.Millisecond) + if _, remove, _, _ := f.counts(); remove != 0 { + t.Errorf("stop removed NRPT (removeRule calls = %d) while a transition held the lock", remove) + } + select { + case <-stopDone: + t.Error("stop completed while an NRPT transition was still in flight") + default: + } + } + + if !p.activateCtrldNRPTFallback(state, "activation racing a stop") { + t.Fatal("the in-flight activation should have completed its transition") + } + <-stopDone + + // The stop ran after the transition released the lock, so its cleanup must have + // removed the rule this transition wrote. + if f.hasCtrldRule() { + t.Error("ctrld's NRPT rule survived the stop: the machine is left pointing at a listener that is gone") + } + if _, remove, _, _ := f.counts(); remove != 1 { + t.Errorf("removeRule calls = %d, want 1: the stop must clean up the rule written by the racing transition", remove) + } + if p.dnsInterceptState != nil { + t.Error("the stop did not complete after the in-flight transition released the lock") + } + + // A second attempt models the other health path arriving after the stop finished. + addBefore, _, signalBefore, _ := f.counts() + if p.activateCtrldNRPTFallback(state, "activation after the stop completed") { + t.Error("activateCtrldNRPTFallback() = true after shutdown completed") + } + addAfter, _, signalAfter, _ := f.counts() + if addAfter != addBefore || signalAfter != signalBefore { + t.Errorf("addRule %d->%d, signal %d->%d: a post-shutdown transition wrote NRPT policy for a dead listener", + addBefore, addAfter, signalBefore, signalAfter) + } + if f.hasCtrldRule() { + t.Error("a post-shutdown transition re-created ctrld's NRPT rule") + } +} + +// TestOwnedRecoveryDefersToIneffectiveExternalPolicy is the caller-level case for the +// ownership boundary: state says ctrld-owned, ctrld's key has disappeared, and an exact +// GP child is present but not routing. +// +// The handback records Group Policy ownership and reports Unverified. That is terminal: +// an administrator's catch-all owns the namespace whether or not it currently routes, so +// the owned-recovery flow must not go on to signal the DNS Client or delete and recreate +// ctrld's rule beside it. Only loopback WFP protect is allowed, and it touches no policy. +func TestOwnedRecoveryDefersToIneffectiveExternalPolicy(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false}, gpRule: "{GP-RULE}", ctrldRule: false} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + p.nrptProbeAndHeal(state) + + add, remove, signal, _ := f.counts() + if add != 0 || remove != 0 || signal != 0 { + t.Errorf("addRule = %d, removeRule = %d, signal = %d, want 0/0/0: owned recovery must stop once external policy owns the namespace", + add, remove, signal) + } + if f.hasCtrldRule() { + t.Error("owned recovery recreated ctrld's catch-all beside the administrator's rule") + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{GP-RULE}", owner, ruleName) + } +} + +// TestHandbackAbortedWhenRuleCannotBeRemoved keeps a failed registry step from being +// read as a verdict about external policy. +func TestHandbackAbortedWhenRuleCannotBeRemoved(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{ + probeResults: []bool{true, true}, + gpRule: "{GP-RULE}", + ctrldRule: true, + removeErr: errors.New("access denied"), + }) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackAborted { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackAborted", got) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("owner = %v, want nrptRuleOwnerCtrld: a failed removal is not proof about external policy", owner) + } +} + +// TestHandbackRestoresFallbackWhenGPChildDisappearsMidProbe covers a Group Policy refresh +// landing during the post-removal probe: the child ctrld was testing is simply gone. There +// is no external policy left to hand ownership to, so ctrld's rule has to come back. +func TestHandbackRestoresFallbackWhenGPChildDisappearsMidProbe(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + if call == 2 { + f.setGPRule("", false) // the administrator removed the catch-all + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackKeptCtrld { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("owner = %v, want nrptRuleOwnerCtrld", owner) + } + if !f.hasCtrldRule() { + t.Error("ctrld's rule was not restored after the GP child disappeared; the host has no route") + } + if add, remove, _, _ := f.counts(); add != 1 || remove != 1 { + t.Errorf("addRule = %d, removeRule = %d, want 1/1", add, remove) + } +} + +// TestHandbackWritesNoSiblingWhenGPChildTurnsConflicting is the case the same-child +// re-read exists for. ctrld removes its keys to test an exact catch-all, and during the +// probe Group Policy replaces it with one that targets another resolver (or a malformed +// one). Restoring ctrld's rule would create exactly the competing sibling beside +// administrator policy that this must never write, so the rule stays off and ownership +// goes to nobody. +func TestHandbackWritesNoSiblingWhenGPChildTurnsConflicting(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + if call == 2 { + // The child now names another resolver: not a match for this listener, and + // reported as a conflict by the GP classifier. + f.setGPRule("", true) + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackConflict { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackConflict", got) + } + if add, _, _, _ := f.counts(); add != 0 { + t.Errorf("addRule calls = %d, want 0: restoring here writes a competing rule beside administrator policy", add) + } + if f.hasCtrldRule() { + t.Error("ctrld's rule is installed beside a GP catch-all that targets another resolver") + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerNone { + t.Errorf("owner = %v, want nrptRuleOwnerNone: ctrld owns nothing once external policy points elsewhere", owner) + } +} + +// TestHandbackWithoutCtrldRuleClassifiesAfterFailedProbe covers the other short-circuit: +// with no ctrld rule to remove, a failed probe must still be classified before anything is +// recorded. A child that changed into a conflicting catch-all is not an ineffective ctrld +// candidate, and must not be recorded as one. +func TestHandbackWithoutCtrldRuleClassifiesAfterFailedProbe(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false}, gpRule: "{GP-RULE}"} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + if call == 1 { + f.setGPRule("", true) + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackConflict { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackConflict", got) + } + if add, remove, signal, _ := f.counts(); add != 0 || remove != 0 || signal != 0 { + t.Errorf("addRule = %d, removeRule = %d, signal = %d, want 0/0/0", add, remove, signal) + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerNone || ruleName != "" { + t.Errorf("owner = %v, rule = %q, want nrptRuleOwnerNone/\"\": a conflicting child must not be recorded as ctrld's external owner", owner, ruleName) + } +} + +// TestHandbackGivesReplacementChildItsOwnPass keeps an unproved child from inheriting a +// verdict that was measured against a different one. With no ctrld rule installed the +// replacement is simply probed on its own pass; failing that pass records it as external +// policy that is not routing, and writes nothing either way. +func TestHandbackGivesReplacementChildItsOwnPass(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false, false}, gpRule: "{GP-RULE}"} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + if call == 1 { + f.setGPRule("{NEW-GP-RULE}", false) + } + } + + p, state, _ := newHandbackTestProg(t) + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackUnverified { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackUnverified", got) + } + if add, remove, signal, _ := f.counts(); add != 0 || remove != 0 || signal != 0 { + t.Errorf("addRule = %d, removeRule = %d, signal = %d, want 0/0/0", add, remove, signal) + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{NEW-GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{NEW-GP-RULE}: the verdict must name the child it was measured against", owner, ruleName) + } +} + +// TestHandbackWritesNoSiblingWhenReplacementChildTakesOver is the case where ctrld's rule +// was removed for the probe and a *different* exact catch-all took the namespace while it +// was off. +// +// Restoring ctrld's rule here is not a return to the status quo: the replacement was not +// there when the transition started, and addNRPTCatchAllRule writes ctrld's own GP +// catch-all whenever another GP rule exists, so the restore would plant a sibling beside +// the administrator's new rule. The replacement gets its own probe instead, and a failed +// one leaves NRPT to Group Policy with nothing of ctrld's written. +func TestHandbackWritesNoSiblingWhenReplacementChildTakesOver(t *testing.T) { + // [pre-probe true, post-removal probe false, replacement's own pass false] + f := &fakeNRPTOps{probeResults: []bool{true, false, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + if call == 2 { + f.setGPRule("{NEW-GP-RULE}", false) // a policy refresh swaps the child + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackUnverified { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackUnverified", got) + } + if add, remove, _, _ := f.counts(); add != 0 || remove != 1 { + t.Errorf("addRule = %d, removeRule = %d, want 0/1: ctrld's rule must not be restored beside a replacement catch-all", add, remove) + } + if f.hasCtrldRule() { + t.Error("ctrld's rule is installed beside an administrator catch-all that took the namespace") + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{NEW-GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{NEW-GP-RULE}", owner, ruleName) + } +} + +// TestHandbackAdoptsReplacementChildThatRoutes is the same swap where the replacement does +// carry DNS on its own: it is adopted, still with no ctrld write. +func TestHandbackAdoptsReplacementChildThatRoutes(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false, true}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + if call == 2 { + f.setGPRule("{NEW-GP-RULE}", false) + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackVerified { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackVerified", got) + } + if add, _, _, _ := f.counts(); add != 0 { + t.Errorf("addRule calls = %d, want 0", add) + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{NEW-GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{NEW-GP-RULE}", owner, ruleName) + } +} + +// TestHandbackChurnFailsSafeToCurrentExternalOwner bounds the extra pass without letting +// exhaustion become a licence to write. +// +// When the probe budget runs out with the store still changing, the transition classifies +// the store as it stands and proves no route. Reporting "undecided" instead would be +// unsafe: that disposition is not terminal external ownership, so startup and owned +// recovery fall through to writing ctrld's rule, and addNRPTCatchAllRule puts it in the GP +// path beside whichever exact catch-all is there now. +func TestHandbackChurnFailsSafeToCurrentExternalOwner(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + switch call { + case 2: + f.setGPRule("{GP-RULE-2}", false) + case 3: + f.setGPRule("{GP-RULE-3}", false) + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test") + if got != nrptHandbackUnverified { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackUnverified", got) + } + if !nrptExternalOwns(got) { + t.Error("the churn disposition is not terminal external ownership; callers would fall through and write a sibling") + } + if add, _, _, _ := f.counts(); add != 0 { + t.Errorf("addRule calls = %d, want 0: no sibling write while the GP store is churning", add) + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE-3}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{GP-RULE-3}: exhaustion must record the store as it stands", owner, ruleName) + } +} + +// TestHandbackChurnFailsSafeToConflict covers exhaustion where the store has settled on a +// catch-all that does not target ctrld: still terminal, still no write. +func TestHandbackChurnFailsSafeToConflict(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + switch call { + case 2: + f.setGPRule("{GP-RULE-2}", false) + case 3: + f.setGPRule("", true) // now an administrator catch-all for another resolver + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackConflict { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackConflict", got) + } + if add, _, _, _ := f.counts(); add != 0 { + t.Errorf("addRule calls = %d, want 0", add) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerNone { + t.Errorf("owner = %v, want nrptRuleOwnerNone", owner) + } +} + +// TestHandbackChurnRestoresFallbackWhenNamespaceFreed is the third exhaustion outcome: the +// store ended up with no external catch-all at all, so the namespace is free and ctrld's +// rule is the one that belongs there. +func TestHandbackChurnRestoresFallbackWhenNamespaceFreed(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + switch call { + case 2: + f.setGPRule("{GP-RULE-2}", false) + case 3: + f.setGPRule("", false) // the administrator removed the policy entirely + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackKeptCtrld { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + if !f.hasCtrldRule() { + t.Error("ctrld's rule was not restored although no external catch-all remains") + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("owner = %v, want nrptRuleOwnerCtrld", owner) + } +} + +// TestActivationWritesNoSiblingWhileGPStoreChurns is the caller-level closure for the churn +// path. activateCtrldNRPTFallback is the function both startup fallback and owned recovery +// reach when ctrld's rule is missing, and it is where a non-terminal churn disposition +// would turn into addNRPTCatchAllRule writing a GP sibling beside the current catch-all. +func TestActivationWritesNoSiblingWhileGPStoreChurns(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false, false}, gpRule: "{GP-RULE}"} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + switch call { + case 1: + f.setGPRule("{GP-RULE-2}", false) + case 2: + f.setGPRule("{GP-RULE-3}", false) + } + } + + p, state, _ := newHandbackTestProg(t) + // Startup and the owner-None retry both arrive here with no ownership recorded. + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") + + if p.activateCtrldNRPTFallback(state, "churn test") { + t.Error("activateCtrldNRPTFallback() = true while an external catch-all owns the namespace") + } + add, remove, signal, _ := f.counts() + if add != 0 || signal != 0 { + t.Errorf("addRule = %d, signal = %d, want 0/0: ctrld must not write beside the administrator's catch-all", add, signal) + } + if remove != 0 { + t.Errorf("removeRule calls = %d, want 0: there was no ctrld rule to remove", remove) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerGroupPolicy { + t.Errorf("owner = %v, want nrptRuleOwnerGroupPolicy: the current external catch-all owns the namespace", owner) + } +} + +// TestStopLeavesGPManagedPolicyAlone is the central promise of this work: an +// administrator's catch-all is a deployment contract that must survive service stop, +// restart and uninstall. A regression that removed NRPT for every owner would delete that +// rule and take the fleet's DNS policy with it. +func TestStopLeavesGPManagedPolicyAlone(t *testing.T) { + f := &fakeNRPTOps{gpRule: "{GP-RULE}"} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, "{GP-RULE}") + + if err := p.stopDNSIntercept(); err != nil { + t.Fatalf("stopDNSIntercept() = %v", err) + } + + add, remove, signal, _ := f.counts() + if remove != 0 || signal != 0 || add != 0 { + t.Errorf("removeRule = %d, signal = %d, addRule = %d, want 0/0/0: shutdown must not touch externally owned NRPT policy", + remove, signal, add) + } + if f.flushCount() != 0 { + t.Errorf("flush calls = %d, want 0: no cache flush is owed for a rule ctrld does not own", f.flushCount()) + } + if p.dnsInterceptState != nil { + t.Error("dnsInterceptState survived shutdown") + } +} + +// TestStopRemovesOrphanWhileLeavingGPPolicy is the same shutdown with a ctrld rule left +// behind by an earlier unclean exit. External policy still stays, but the orphan must go: +// while GP mode hides the local store this stop is the last chance anything looks there. +func TestStopRemovesOrphanWhileLeavingGPPolicy(t *testing.T) { + f := &fakeNRPTOps{gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, "{GP-RULE}") + + if err := p.stopDNSIntercept(); err != nil { + t.Fatalf("stopDNSIntercept() = %v", err) + } + + add, remove, signal, _ := f.counts() + if remove != 1 || signal != 1 { + t.Errorf("removeRule = %d, signal = %d, want 1/1: the orphaned ctrld rule must be removed on the way out", remove, signal) + } + if add != 0 { + t.Errorf("addRule calls = %d, want 0", add) + } + if f.hasCtrldRule() { + t.Error("the orphaned ctrld rule survived shutdown; a later GP removal would activate it") + } + if f.flushCount() != 0 { + t.Errorf("flush calls = %d, want 0: the ctrld-owned branch is what owes a flush, not the GP branch", f.flushCount()) + } +} + +// TestRemoveOrphanedCtrldNRPTRule covers the sweep directly, including that it is a no-op +// when there is nothing of ctrld's on disk - it runs on shutdown paths where the common +// case is a clean machine. +func TestRemoveOrphanedCtrldNRPTRule(t *testing.T) { + t.Run("removes and signals when a ctrld rule exists", func(t *testing.T) { + f := &fakeNRPTOps{ctrldRule: true} + installFakeNRPTOps(t, f) + + p, _, _ := newHandbackTestProg(t) + p.removeOrphanedCtrldNRPTRule("test") + + add, remove, signal, _ := f.counts() + if remove != 1 || signal != 1 || add != 0 { + t.Errorf("removeRule = %d, signal = %d, addRule = %d, want 1/1/0", remove, signal, add) + } + if f.hasCtrldRule() { + t.Error("the ctrld rule is still installed") + } + }) + + t.Run("does nothing when no ctrld rule exists", func(t *testing.T) { + f := &fakeNRPTOps{} + installFakeNRPTOps(t, f) + + p, _, _ := newHandbackTestProg(t) + p.removeOrphanedCtrldNRPTRule("test") + + add, remove, signal, _ := f.counts() + if remove != 0 || signal != 0 || add != 0 { + t.Errorf("removeRule = %d, signal = %d, addRule = %d, want 0/0/0: nothing to sweep must cost no registry writes and no signalling", + remove, signal, add) + } + }) +} + +// TestHandbackThrottleBlocksTheSecondAttempt exercises the throttle through the +// transition, not just its accessor. Each attempt removes the live catch-all for a probe, +// so a second attempt inside the window must cost nothing at all: no probe, no removal. +func TestHandbackThrottleBlocksTheSecondAttempt(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "first"); got != nrptHandbackKeptCtrld { + t.Fatalf("first nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + _, removeAfterFirst, _, probesAfterFirst := f.counts() + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "second"); got != nrptHandbackAborted { + t.Fatalf("second nrptHandbackToExternal() = %v, want nrptHandbackAborted", got) + } + _, remove, _, probes := f.counts() + if probes != probesAfterFirst { + t.Errorf("probe calls went from %d to %d: a throttled attempt must not probe", probesAfterFirst, probes) + } + if remove != removeAfterFirst { + t.Errorf("removeRule calls went from %d to %d: a throttled attempt must not remove the live rule", removeAfterFirst, remove) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("owner = %v, want nrptRuleOwnerCtrld", owner) + } +} + +// TestHealLadderRunsRetriesThenTwoPhaseRecovery drives the ctrld-owned heal cycle end to +// end: the immediate probe, the signal-and-backoff retries, then the two-phase delete and +// re-add, with the last probe finally succeeding. Each nrptTransition phase is asserted +// through its side effects, since these are its only call sites. +func TestHealLadderRunsRetriesThenTwoPhaseRecovery(t *testing.T) { + // Every probe fails until the one after the re-add. + f := &fakeNRPTOps{ + probeResults: []bool{false, false, false, false, true}, + ctrldRule: true, + } + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + p.nrptProbeAndHeal(state) + + add, remove, signal, probes := f.counts() + if remove != 1 { + t.Errorf("removeRule calls = %d, want 1: phase one of the two-phase recovery deletes ctrld's rule once", remove) + } + if add != 1 { + t.Errorf("addRule calls = %d, want 1: phase two re-adds it once", add) + } + if !f.hasCtrldRule() { + t.Error("ctrld's rule is missing after the two-phase recovery") + } + // Three backoff rounds signal, then the delete phase and the re-add phase. + if signal < 5 { + t.Errorf("signal calls = %d, want at least 5 (three retries plus both recovery phases)", signal) + } + if probes < 5 { + t.Errorf("probe calls = %d, want at least 5", probes) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("owner = %v, want nrptRuleOwnerCtrld", owner) + } +} + +// TestHealLadderStopsWhenGPAppearsBeforeTwoPhase covers a Group Policy catch-all landing +// during the backoff retries. The ladder must hand back and stop before the destructive +// delete-and-re-add phase, and must not write ctrld's rule beside the new policy. +func TestHealLadderStopsWhenGPAppearsBeforeTwoPhase(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false}, ctrldRule: true} + installFakeNRPTOps(t, f) + // The administrator's rule appears while the fourth probe is in flight, which is the + // last one before the two-phase recovery. + f.onProbe = func(call int) { + if call == 4 { + f.setGPRule("{GP-RULE}", false) + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + p.nrptProbeAndHeal(state) + + add, remove, _, _ := f.counts() + if remove != 0 { + t.Errorf("removeRule calls = %d, want 0: the ladder must stop before the delete half of the two-phase recovery", remove) + } + if add != 0 { + t.Errorf("addRule calls = %d, want 0: the ladder must not recreate ctrld's rule beside a new GP catch-all", add) + } + if !f.hasCtrldRule() { + t.Error("ctrld's rule was deleted after an administrator catch-all appeared") + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{GP-RULE}: the ladder must defer rather than continue", owner, ruleName) + } +} + +// TestHealLadderCleansEmptyGPParentBeforeRetrying pins the empty-GP-parent shortcut: an +// empty parent key puts the DNS Client in GP mode where ctrld's local rule is invisible, so +// signalling retries cannot succeed until it is deleted. +func TestHealLadderCleansEmptyGPParentBeforeRetrying(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false, true}, ctrldRule: true, parentEmpty: true} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + p.nrptProbeAndHeal(state) + + if f.cleanCalls != 1 { + t.Errorf("cleanParent calls = %d, want 1: the empty GP parent must be removed before burning retries", f.cleanCalls) + } + if add, remove, _, _ := f.counts(); add != 0 || remove != 0 { + t.Errorf("addRule = %d, removeRule = %d, want 0/0: the probe passed after the cleanup, so no recovery was needed", add, remove) + } +} + +// TestRepairMissingWFPRetriesHardModeWithNoEngine covers the entry point a startup WFP +// failure depends on. In hard mode a closed engine means nothing is being blocked, so the +// health monitor must keep retrying; in dns mode a closed engine is the normal state and +// must not trigger anything. +func TestRepairMissingWFPRetriesHardModeWithNoEngine(t *testing.T) { + originalRebuild, originalHard := rebuildDNSInterceptFn, hardIntercept + t.Cleanup(func() { rebuildDNSInterceptFn, hardIntercept = originalRebuild, originalHard }) + + var reasons []string + rebuildDNSInterceptFn = func(_ *prog, _ *wfpState, reason string) interceptRebuildResult { + reasons = append(reasons, reason) + return interceptRebuildDone + } + + t.Run("hard mode retries when the engine never opened", func(t *testing.T) { + reasons = nil + hardIntercept = true + p, state, _ := newHandbackTestProg(t) + + if !p.repairMissingWFP(state) { + t.Error("repairMissingWFP() = false; the monitor must hand over to the rebuilt intercept") + } + if len(reasons) != 1 { + t.Fatalf("rebuild requests = %d, want 1: hard mode with no WFP engine must be retried", len(reasons)) + } + if !strings.Contains(reasons[0], "no WFP engine") { + t.Errorf("rebuild reason = %q, want it to name the missing engine", reasons[0]) + } + }) + + t.Run("dns mode leaves a closed engine alone", func(t *testing.T) { + reasons = nil + hardIntercept = false + p, state, _ := newHandbackTestProg(t) + + if p.repairMissingWFP(state) { + t.Error("repairMissingWFP() = true in dns mode; a closed engine is normal there") + } + if len(reasons) != 0 { + t.Errorf("rebuild requests = %d, want 0", len(reasons)) + } + }) +} + +// TestPhaseTwoRevalidatesOwnershipAfterWaitingForTheLock is the ordering a pre-lock check +// cannot cover. Phase one removes ctrld's rule and releases the transition lock; the +// pre-re-add checks see no GP rule; another health path then takes the lock, hands the +// namespace to a GP catch-all that has just arrived, and records GroupPolicy. Phase two +// acquires the lock afterwards. +// +// Without a re-read inside the locked closure, phase two re-adds ctrld's rule beside the +// administrator's catch-all and stamps ctrld ownership over theirs - exactly the competing +// policy this work exists to prevent. +func TestPhaseTwoRevalidatesOwnershipAfterWaitingForTheLock(t *testing.T) { + p, state, f := newHandbackTestProg(t) + // Every probe fails, so the ladder runs to the two-phase recovery. + f.configure(fakeNRPTSeed{probeResults: []bool{false}, ctrldRule: true}) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + // The last ops call before phase two queues for the lock. Hold the lock from another + // goroutine and complete the competing transition while phase two waits for it. + f.onGPConflicts = func(call int) { + if call != 1 { + return + } + locked := make(chan struct{}) + go func() { + p.nrptTransitionMu.Lock() + close(locked) + f.setGPRule("{GP-RULE}", false) + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, "{GP-RULE}") + // Hold it long enough that phase two must queue behind this transition. + time.Sleep(100 * time.Millisecond) + p.nrptTransitionMu.Unlock() + }() + <-locked + } + + p.nrptProbeAndHeal(state) + + if add, _, _, _ := f.counts(); add != 0 { + t.Errorf("addRule calls = %d, want 0: phase two re-added ctrld's rule beside a GP catch-all that took ownership while it waited for the lock", add) + } + if f.hasCtrldRule() { + t.Error("ctrld's rule is installed beside the administrator's catch-all") + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{GP-RULE}: phase two overwrote a newer ownership transition", owner, ruleName) + } +} + +// TestRetiredHealDoesNotDeleteSuccessorRule covers the other side of a rebuild. The heal +// cycle re-adds ctrld's rule, then its final settle wait observes that its own state was +// retired - while a successor intercept is already running with the same deterministic key. +// +// The key is process-global, so any cleanup from here would delete the successor's route +// and leave hard mode blocking DNS with nothing redirecting it. Teardown of the retired +// state owns that cleanup instead. +func TestRetiredHealDoesNotDeleteSuccessorRule(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{probeResults: []bool{false}, ctrldRule: true}) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + successor := &wfpState{stopCh: make(chan struct{}), listenerIP: "127.0.0.1"} + // Retire this cycle's state during the final wait - the one after the re-add, which is + // the only wait where both a removal and an add have happened - and publish a successor + // that owns the same deterministic key. + f.waitHook = func() bool { + if add, remove, _, _ := f.counts(); add == 0 || remove == 0 { + return true + } + close(state.stopCh) // a rebuild retired the old state + p.dnsInterceptState = successor + successor.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + return false + } + + p.nrptProbeAndHeal(state) + + if !f.hasCtrldRule() { + t.Error("the successor's NRPT rule was deleted by a retired heal cycle") + } + if owner, _ := successor.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("successor owner = %v, want nrptRuleOwnerCtrld", owner) + } +} + +// TestStartupReportsFailureWhenExternalPolicyNeverRoutes is the readiness direction of the +// GP-managed contract, driven through the real startup control flow. +// +// An exact GP child is present and no probe ever reaches ctrld. Startup must keep the +// recovery state and the monitor - the rule may start routing later, and only external +// policy can fix it - must leave adapter DNS alone, and must not write an owned fallback. +// What it must not do is return success: setDNS records readiness from a nil error, so +// reporting ready here publishes "service healthy" while the DNS Client is not delivering +// queries to ctrld, and in hard mode WFP is blocking every other resolver at the same time. +func TestStartupReportsFailureWhenExternalPolicyNeverRoutes(t *testing.T) { + originalCfg, originalMode, originalIntercept, originalHard := cfg, interceptMode, dnsIntercept, hardIntercept + t.Cleanup(func() { + cfg, interceptMode, dnsIntercept, hardIntercept = originalCfg, originalMode, originalIntercept, originalHard + }) + cfg = ctrld.Config{} + cfg.Listener = map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}} + interceptMode, dnsIntercept, hardIntercept = "dns", true, false + + p, _, f := newHandbackTestProg(t) + p.cfg = &cfg + p.dnsInterceptState = nil // startup publishes its own state + // An exact GP child that never answers a probe. + f.configure(fakeNRPTSeed{probeResults: []bool{false}, gpRule: "{GP-RULE}"}) + + err := p.startDNSInterceptLocked() + if err == nil { + t.Fatal("startDNSInterceptLocked() = nil: startup reported success while no probe reached ctrld, so setDNS would mark the service ready") + } + if !interceptFailedUnderExternalDNSPolicy(err) { + t.Errorf("err = %v; the failure must tell setDNS to leave adapter DNS untouched", err) + } + if interceptFailedWithVerifiedExternalDNS(err) { + t.Errorf("err = %v; this route was never verified, so it must not read as the verified case", err) + } + if add, remove, signal, _ := f.counts(); add != 0 || remove != 0 || signal != 0 { + t.Errorf("addRule = %d, removeRule = %d, signal = %d, want 0/0/0: no owned fallback may be written beside an administrator catch-all", add, remove, signal) + } + + // Recovery must stay live so the rule can be re-tested. + state, ok := p.dnsInterceptState.(*wfpState) + if !ok || state == nil { + t.Fatal("intercept state was not published; nothing would keep re-testing the external rule") + } + if owner, ruleName := state.nrptPolicyOwner(); owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{GP-RULE}", owner, ruleName) + } + // Stop the monitor startup launched. + _ = p.stopDNSIntercept() +} + +// TestFakeNRPTOpsPreconditionDetectsBypass guards the guard. The fixtures' promise is that +// no test can reach the production registry and signalling functions, and that promise is +// only as good as this precondition: if it stops detecting a bypass, every "no side +// effects" assertion in the Windows suite silently becomes vacuous while the real host +// policy is at risk. +func TestFakeNRPTOpsPreconditionDetectsBypass(t *testing.T) { + f := &fakeNRPTOps{} + + // The exact bypass to catch: a fixture that returns a fake it never installed. + if err := checkFakeNRPTOpsInstalled(f); err == nil { + t.Error("an uninstalled fake passed the precondition; a test could fall through to production NRPT operations") + } + + installFakeNRPTOps(t, f) + if err := checkFakeNRPTOpsInstalled(f); err != nil { + t.Errorf("an installed fake failed the precondition: %v", err) + } + + // A different fake than the installed one must not pass either: side-effect counters + // would be read from an object nothing consults. + if err := checkFakeNRPTOpsInstalled(&fakeNRPTOps{}); err == nil { + t.Error("a fake that is not the installed one passed the precondition") + } +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 04b7ead..6f13349 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -172,6 +172,22 @@ type prog struct { // On Windows: *wfpState, on macOS: *pfState, nil on other platforms. dnsInterceptState any + // dnsInterceptMu serializes DNS intercept lifecycle transitions - start, stop and + // the health monitor's rebuild - and guards every write to dnsInterceptState, so a + // service stop can never interleave with a monitor-driven rebuild. + dnsInterceptMu sync.Mutex //lint:ignore U1000 used on windows + + // dnsInterceptStopRequested is set while a stop waits for dnsInterceptMu. The + // health and recovery flows read it as a shutdown signal and abandon their work, + // rather than making the stop wait out their probe backoffs. + dnsInterceptStopRequested atomic.Bool //lint:ignore U1000 used on windows + + // nrptTransitionMu makes one NRPT ownership transition - observe, mutate, signal, + // record owner - atomic against shutdown and against another transition. It is + // deliberately finer-grained than dnsInterceptMu: it is taken for the duration of a + // single transition, never across the recovery flows' probe backoffs. + nrptTransitionMu sync.Mutex //lint:ignore U1000 used on windows + // lastTunnelIfaces tracks the tunnel set included in the last successfully loaded // pf anchor. Pending tunnel state is kept separately so failed PF work is retried // instead of being mistaken for an applied update. Protected by mu. @@ -220,15 +236,22 @@ type prog struct { // existing delayed checks provide a trailing reconciliation after churn. pfIgnoredChangeLastReconcile atomic.Int64 //lint:ignore U1000 used on darwin - // pfProbeExpected holds the domain name of a pending pf interception probe. - // When non-empty, the DNS handler checks incoming queries against this value - // and signals pfProbeCh if matched. The probe verifies that pf's rdr rules - // are actually translating packets (not just present in rule text). - pfProbeExpected atomic.Value // string - - // pfProbeCh is signaled when the DNS handler receives the expected probe query. - // The channel is created by probePFIntercept() and closed when the probe arrives. - pfProbeCh atomic.Value // *chan struct{} + // interceptProbes maps the domain of each pending interception probe to the channel + // that probe waits on. A probe verifies that interception is actually translating or + // redirecting packets, not merely present in rule text: the DNS handler looks up + // incoming queries here and signals the matching waiter. + // + // It holds one entry per in-flight probe rather than a single slot, because probes do + // overlap - the health monitor, a handback and a heal cycle can each have one out at + // the same time - and a single slot means the last registration wins and the loser + // waits out its timeout for a query that was answered. A false failure then triggers + // recovery work that was not needed. + // + // Registrations are rare and lookups happen on every query, so the map is stored as + // an immutable snapshot behind an atomic: readers never take a lock, writers copy + // under interceptProbeMu. + interceptProbes atomic.Value // map[string]chan struct{} + interceptProbeMu sync.Mutex //lint:ignore U1000 written only by registerInterceptProbe, used on darwin/windows // VPN DNS manager for split DNS routing when intercept mode is active. vpnDNS *vpnDNSManager @@ -424,7 +447,12 @@ func (p *prog) postRun() { p.runningOnDomainController = isDC mainLog.Load().Debug().Msgf("running on domain controller: %t, role: %d", p.runningOnDomainController, roleInt) } - p.resetDNS(false, false) + // A Windows organization can install a GP-owned NRPT catch-all before + // starting ctrld. Detect that policy before resetDNS touches adapter DNS; + // startDNSIntercept will then prove the rule functionally before adopting it. + if !p.skipInitialDNSReset() { + p.resetDNS(false, false) + } ns := ctrld.InitializeOsResolver(false) mainLog.Load().Debug().Msgf("initialized OS resolver with nameservers: %v", ns) p.setDNS() @@ -907,7 +935,7 @@ func (p *prog) setDNS() { mainLog.Load().Fatal().Msgf("invalid --intercept-mode value %q: must be 'off', 'dns', or 'hard'", interceptMode) } if interceptMode == "" || interceptMode == "off" { - interceptMode = cfg.Service.InterceptMode + interceptMode = p.configuredInterceptMode() if interceptMode != "" && interceptMode != "off" { mainLog.Load().Info().Msgf("Intercept mode enabled via config (intercept_mode = %q)", interceptMode) } @@ -927,6 +955,30 @@ func (p *prog) setDNS() { // software that also manages DNS. See issue #489. if dnsIntercept { if err := startDNSInterceptFn(p); err != nil { + // This check comes first: it is the one failure where DNS already works + // without ctrld touching anything else, so neither the refusal below nor the + // fallback applies. + // + // An externally managed rule was proved - by probe, not by registry shape - + // to be routing DNS to this listener. Falling through would rewrite adapter + // DNS after explicitly preserving it, and DNS still works, so stop here. + // + // Only a verified route earns this. A rule that merely exists does not: if it + // is not actually routing and intercept failed too, the machine would be left + // with no NRPT, no WFP and no adapter fallback - that is, unfiltered - so + // every other failure takes the paths below. + if interceptFailedUnderExternalDNSPolicy(err) { + if interceptFailedWithVerifiedExternalDNS(err) { + mainLog.Load().Error().Err(err).Msg("DNS intercept mode failed but externally managed DNS policy is verified routing to ctrld — not falling back to interface DNS settings") + } else { + // Owned by external policy but not proved to route: DNS is not + // reaching ctrld. Adapter DNS still stays as the organization set it, + // and setDnsOK stays false, so this start reports as failed until a + // probe succeeds. + mainLog.Load().Error().Err(err).Msg("DNS intercept mode failed and externally managed DNS policy is not routing to ctrld — leaving interface DNS settings untouched; the service is not ready") + } + return + } // Interface DNS cannot express a port: macOS interface settings and Windows // NRPT rules both name a resolver by IP alone. So it is only a usable // fallback when the listener actually bound :53. When something else owns @@ -1037,6 +1089,17 @@ func (p *prog) setDNS() { } } +// configuredInterceptMode resolves the service's effective intercept mode without +// mutating package state. Platform startup preflights use the same precedence as +// setDNS so they do not make adapter-DNS decisions from a different mode value. +func (p *prog) configuredInterceptMode() string { + im := interceptMode + if im == "" || im == "off" { + im = p.cfg.Service.InterceptMode + } + return im +} + func (p *prog) setDnsForRunningIface(nameservers []string) (runningIface *net.Interface) { if p.runningIface == "" { return diff --git a/docs/dns-intercept-mode.md b/docs/dns-intercept-mode.md index c089ea1..0b92dea 100644 --- a/docs/dns-intercept-mode.md +++ b/docs/dns-intercept-mode.md @@ -67,28 +67,27 @@ Separating them into modes means most users get `dns` mode (safe, can never brea #### Startup Sequence (dns mode) -1. Creates NRPT catch-all registry rule (`.` → `127.0.0.1`) under `HKLM\...\DnsPolicyConfig\CtrldCatchAll` -2. Triggers Group Policy refresh via `RefreshPolicyEx` (userenv.dll) so DNS Client loads NRPT immediately -3. Flushes DNS cache to clear stale entries -4. **Activates loopback WFP protect** — adds 4 permit filters (IPv4/IPv6 × UDP/TCP) for DNS to localhost with `FWPM_FILTER_FLAG_CLEAR_ACTION_RIGHT`. These prevent third-party WFP block filters from blocking the NRPT → `127.0.0.1` path (see [Loopback WFP Protect](#loopback-wfp-protect) below). Non-fatal if this fails. -5. Starts NRPT health monitor (30s periodic check) -6. Launches async NRPT probe-and-heal to verify NRPT is actually routing queries +1. Checks for a non-ctrld GP child whose only namespace is `.` and whose only nameserver is ctrld's actual listener IP. +2. When that candidate exists, preserves adapter DNS, sends a DNS Client probe before any NRPT mutation, and re-reads the same GP child. A matching before/after rule plus a received probe enters **GP-managed mode**; ctrld does not write NRPT, call `RefreshPolicyEx`/`paramchange`, or flush DNS for policy activation. +3. Without a still-matching GP candidate, creates the normal ctrld-owned catch-all, signals DNS Client, and flushes stale cache entries. +4. **Activates loopback WFP protect** — adds 4 permit filters (IPv4/IPv6 × UDP/TCP) for DNS to localhost with `FWPM_FILTER_FLAG_CLEAR_ACTION_RIGHT`. These prevent third-party WFP block filters from blocking the NRPT → listener path (see [Loopback WFP Protect](#loopback-wfp-protect) below). Non-fatal if this fails. +5. Starts the 30-second ownership-aware NRPT health monitor. +6. Re-verifies an initially ineffective GP candidate synchronously after WFP setup; ctrld-owned NRPT uses the asynchronous probe-and-heal sequence. #### Startup Sequence (hard mode) -1. Creates NRPT catch-all rule + GP refresh + DNS flush (same as dns mode) -2. Opens WFP engine with `RPC_C_AUTHN_DEFAULT` (0xFFFFFFFF) -3. Cleans up any stale sublayer from a previous unclean shutdown -4. Creates sublayer with maximum weight (0xFFFF) -5. Adds **permit** filters (weight 10) for DNS to localhost (`127.0.0.1`/`::1` port 53) -6. Adds **permit** filters (weight 10) for DNS to RFC1918 + CGNAT subnets (10/8, 172.16/12, 192.168/16, 100.64/10) -7. Adds **block** filters (weight 1) for all other outbound DNS (port 53 UDP+TCP) -8. Starts NRPT health monitor (also verifies WFP sublayer in hard mode) -9. Launches async NRPT probe-and-heal +1. Establishes NRPT routing using the same GP-managed adoption or ctrld-owned fallback sequence as `dns` mode. +2. Opens WFP engine with `RPC_C_AUTHN_DEFAULT` (0xFFFFFFFF). +3. Cleans up any stale sublayer from a previous unclean shutdown. +4. Creates sublayer with maximum weight (0xFFFF). +5. Adds **permit** filters (weight 10) for DNS to localhost (`127.0.0.1`/`::1` port 53). +6. Adds **permit** filters (weight 10) for DNS to RFC1918 + CGNAT subnets (10/8, 172.16/12, 192.168/16, 100.64/10). +7. Adds **block** filters (weight 1) for all other outbound DNS (port 53 UDP+TCP). +8. Starts the NRPT/WFP health monitor. -**Atomic guarantee:** NRPT must succeed before WFP starts. If NRPT fails, WFP is not attempted. If WFP fails, NRPT is rolled back. This prevents DNS blackholes where WFP blocks everything but nothing routes to ctrld. +**Atomic guarantee:** NRPT routing must exist before WFP starts. If WFP setup fails, ctrld rolls back only a rule it owns. A GP-managed child is never deleted, rewritten, or replaced with interface DNS merely because ctrld's WFP setup failed. -On shutdown: stops health monitor, removes NRPT rule, flushes DNS, then (hard mode only) removes all WFP filters and closes engine. +On shutdown, ctrld stops its monitor and WFP session. It removes and signals only ctrld-owned NRPT state; a GP-managed catch-all remains untouched. #### NRPT Details @@ -103,7 +102,27 @@ The **Name Resolution Policy Table** is a Windows feature (originally for Direct **Registry path**: `HKLM\SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig\CtrldCatchAll` -**Group Policy refresh**: The DNS Client service only reads NRPT from registry during Group Policy processing cycles (default: every 90 minutes). ctrld calls `RefreshPolicyEx(bMachine=TRUE, dwOptions=RP_FORCE)` from `userenv.dll` to trigger an immediate refresh. Falls back to `gpupdate /target:computer /force` if the DLL call fails. +**Group Policy refresh**: The DNS Client service only reads NRPT from registry during Group Policy processing cycles (default: every 90 minutes). ctrld calls `RefreshPolicyEx(bMachine=TRUE, dwOptions=RP_FORCE)` when activating or repairing rules it owns. While Group Policy remains the owner, ctrld does not run NRPT activation/heal signaling; the one transition that removes a ctrld fallback is signaled after the external rule has been proven. + +#### GP-managed NRPT ownership + +Enterprise deployments may install a computer-scoped GP child before starting ctrld with: + +- exactly one namespace: `.`; +- exactly one `GenericDNSServers` value; and +- that nameserver equal to ctrld's actual loopback listener (`127.0.0.1` or the alternate loopback selected on an AD DNS server). + +At service startup ctrld reads that candidate before the normal adapter reset, probes through Windows DNS Client while its listener is already bound, and re-reads the same child. When the rule remains present and the probe arrives, ctrld records **Group Policy** as the NRPT owner. Adapter DNS stays on the organization's resolvers, and ctrld does not create, delete, refresh, or flush NRPT policy. + +The health monitor keeps using functional probes: + +- matching GP rule + successful probe: observe only; +- matching GP rule + failed probe: retry loopback WFP protection, then report the external policy as ineffective without running NRPT heal signals; +- matching GP rule disappears: create the normal ctrld-owned fallback and verify it, unless another GP catch-all targets a different resolver; +- GP catch-all targets another resolver: report the conflict and do not create a second ambiguous catch-all; +- matching GP rule returns: prove it with a probe, remove only ctrld's deterministic fallback keys, and return ownership to Group Policy. + +Deploy the GPO **before** starting or restarting ctrld if adapter DNS must remain completely untouched. Remove or unlink the GP rule before intentionally removing the ctrld service. A GP catch-all that remains pointed at loopback while no listener is running causes DNS failure by design; ctrld cannot safely delete an administrator-owned policy during uninstall. #### WFP Filter Architecture @@ -145,17 +164,19 @@ See: [Issue #526](https://gitlab.int.windscribe.com/controld/clients/ctrld/-/iss ctrld verifies NRPT is actually working by sending a probe DNS query (`_nrpt-probe-.nrpt-probe.ctrld.test`) through Go's `net.Resolver` (which calls `GetAddrInfoW` → DNS Client → NRPT path). If ctrld receives the probe on its listener, NRPT is active. -**Startup probe (async, non-blocking):** After NRPT setup, an async goroutine probes with escalating remediation: (1) immediate probe, (2) GP refresh + retry, (3) DNS Client service restart + retry, (4) final retry. Only one probe sequence runs at a time. +**Startup probes:** A matching GP candidate is probed synchronously before any NRPT mutation and re-read afterward. ctrld-owned rules keep the asynchronous activation/heal sequence: immediate probe, bounded policy signaling retries, then two-phase delete/re-add recovery. Only one probe sequence runs at a time. -**DNS Client restart (nuclear option):** If GP refresh alone isn't enough, ctrld restarts the `Dnscache` service to force full NRPT re-initialization. This briefly interrupts all DNS (~100ms) but only fires when NRPT is already not working. +**Ownership boundary:** When the active owner is Group Policy, a failed probe never enters ctrld's NRPT refresh/delete/re-add sequence. ctrld may repair its narrowly scoped loopback WFP permits, but leaves the external registry child and DNS Client policy signaling to the administrator. #### NRPT Health Monitor A dedicated background goroutine (`nrptHealthMonitor`) runs every 30 seconds and now performs active probing: -1. **Registry check:** If the NRPT catch-all rule is missing from the registry, restore it + GP refresh + probe-and-heal -2. **Active probe:** If the rule exists, send a probe query to verify it's actually routing — catches cases where the registry key is present but DNS Client hasn't loaded it -3. **(hard mode)** Verify WFP sublayer exists; full restart on loss +1. **Ownership check:** Distinguish a matching external GP child from ctrld's deterministic local/GP keys. +2. **Active probe:** Verify Windows DNS Client still routes to the listener. +3. **Transition:** If the external child disappears, activate ctrld's normal fallback. If it returns while the fallback is active, prove it before removing only ctrld's keys. +4. **Owned recovery:** Restore/heal only when ctrld owns the NRPT rule. +5. **(hard mode)** Verify the WFP sublayer exists and fully restart intercept state on loss. This is periodic (not just network-event-driven) because VPN software can clear NRPT at any time. Additionally, `scheduleDelayedRechecks()` (called on network change events) performs immediate NRPT verification at 2s and 4s after changes. diff --git a/docs/wfp-dns-intercept.md b/docs/wfp-dns-intercept.md index 527d20d..60b5428 100644 --- a/docs/wfp-dns-intercept.md +++ b/docs/wfp-dns-intercept.md @@ -7,9 +7,7 @@ On Windows, DNS intercept mode uses a two-layer architecture: - **`dns` mode (default)**: NRPT only — graceful DNS routing via the Windows DNS Client service - **`hard` mode**: NRPT + WFP — full enforcement with kernel-level block filters -This dual-mode design ensures that `dns` mode can never break DNS (at worst, a VPN -overwrites NRPT and queries bypass ctrld temporarily), while `hard` mode provides -the same enforcement guarantees as macOS pf. +`dns` mode avoids ctrld's outbound block filters and therefore degrades more gracefully when owned NRPT is removed. Resolution still depends on the active NRPT target being reachable; an administrator-owned GP catch-all intentionally remains fail-closed if its loopback listener is stopped. ## Architecture: dns vs hard Mode @@ -24,9 +22,8 @@ the same enforcement guarantees as macOS pf. │ localhost, CLEAR_ACTION_RIGHT) prevent third-party VPN WFP │ │ blocks (e.g., OpenVPN block-outside-dns) from breaking NRPT. │ │ │ -│ If VPN clears NRPT: health monitor re-adds within 30s │ -│ Worst case: queries go to VPN DNS until NRPT restored │ -│ DNS never breaks — graceful degradation │ +│ Owned rule missing → restore; GP missing → owned fallback │ +│ GP rule + dead listener remains intentionally fail-closed │ └─────────────────────────────────────────────────────────────────┘ ┌─────────────────────────────────────────────────────────────────┐ @@ -37,9 +34,8 @@ the same enforcement guarantees as macOS pf. │ Bypass attempt (raw 8.8.8.8:53) → WFP BLOCK filter │ │ VPN DNS on private IP → WFP subnet PERMIT filter → allowed │ │ │ -│ NRPT must be active before WFP starts (atomic guarantee) │ -│ If NRPT fails → WFP not started (avoids DNS blackhole) │ -│ If WFP fails → NRPT rolled back (all-or-nothing) │ +│ NRPT route is established before WFP starts │ +│ WFP failure rolls back only ctrld-owned NRPT; GP is untouched │ └─────────────────────────────────────────────────────────────────┘ ``` @@ -79,7 +75,7 @@ unreliable. If we write to the GP path, DNS Client enters GP mode but the rules never activate — resulting in `Get-DnsClientNrptPolicy` returning empty even though `Get-DnsClientNrptRule` shows the rule in registry. -ctrld uses an adaptive strategy (matching [Tailscale's approach](https://github.com/tailscale/tailscale/blob/main/net/dns/nrpt_windows.go)): +ctrld uses an adaptive strategy (matching [Tailscale's approach](https://github.com/tailscale/tailscale/blob/main/net/dns/nrpt_windows.go)) when it owns NRPT: 1. **Always write to the local path** using a deterministic GUID key name (`{B2E9A3C1-7F4D-4A8E-9D6B-5C1E0F3A2B8D}`). This is the baseline that works @@ -91,6 +87,23 @@ ctrld uses an adaptive strategy (matching [Tailscale's approach](https://github. the empty GP parent key. This ensures DNS Client stays in "local mode" where the local-path rule activates immediately via `paramchange`. +### Adopting an Organization-Owned GP Catch-All + +Before applying that ctrld-owned strategy, service startup looks for a non-ctrld GP child with exactly `Name=["."]` and one `GenericDNSServers` value equal to the actual listener IP. The registry match is only an ownership candidate: ctrld sends its unique DNS Client probe before any NRPT write and re-reads the same child afterward. + +Intercept state stays unpublished while that happens. Startup publishes nothing until it has fully succeeded, which is why the probe and heal flows take the state as an argument instead of reading the published field — publishing early would expose a half-built `wfpState`, with no engine handle and filter IDs still being assigned, to callers such as the VPN DNS exemption path. The one deliberate exception is hard mode when WFP setup fails while GP-managed NRPT is verified routing: that state is published so the health monitor can keep retrying WFP, and the service start is still reported as failed. + +When the rule remains unchanged and the probe arrives, Group Policy owns NRPT: + +- the startup adapter reset is skipped; +- no ctrld NRPT key is created; +- `RefreshPolicyEx`, Dnscache `paramchange`, and cache flush are not used for that policy; +- shutdown/uninstall leave the GP child untouched. + +A matching GP child that remains present but fails its probe is reported as ineffective and is not rewritten. If the child disappears, ctrld activates its normal owned fallback. A GP catch-all that instead changes to another resolver is reported as a conflict; ctrld does not create a second ambiguous catch-all. If the matching rule later returns, ctrld probes first, removes only its deterministic fallback keys, signals the ownership transition once, and resumes observing Group Policy. + +**Deployment ordering:** apply the GPO before starting ctrld to guarantee adapter DNS is never reset. Remove/unlink it before intentionally stopping or uninstalling ctrld. A GP catch-all still targeting loopback with no listener running is a deliberate fail-closed state and will break DNS. + ### Reproducing the Empty GP Parent Case This is a production code reference, so the temporary repro script is not kept in @@ -209,28 +222,28 @@ by the VPN's own WFP rules. **Startup (hard mode):** ``` -1. Add NRPT catch-all rule + GP refresh + DNS flush +1. Adopt a proven matching GP catch-all, or install ctrld-owned NRPT 2. FwpmEngineOpen0() with RPC_C_AUTHN_DEFAULT (0xFFFFFFFF) 3. Delete stale sublayer (crash recovery) 4. FwpmSubLayerAdd0() — weight 0xFFFF 5. Add 4 localhost permit filters 6. Add 4 block filters 7. Add RFC1918 + CGNAT subnet permits -8. Start NRPT health monitor goroutine +8. Start ownership-aware NRPT/WFP health monitor ``` **Startup (dns mode):** ``` -1. Add NRPT catch-all rule + GP refresh + DNS flush +1. Adopt a proven matching GP catch-all, or install ctrld-owned NRPT 2. Activate loopback WFP protect (4 hard-permit filters for localhost DNS) -3. Start NRPT health monitor goroutine +3. Start ownership-aware NRPT health monitor ``` **Shutdown:** ``` 1. Stop NRPT health monitor -2. Remove NRPT catch-all rule + DNS flush -3. (hard mode only) Clean up all WFP filters, sublayer, close engine +2. Remove + signal only ctrld-owned NRPT; leave GP-managed policy untouched +3. Clean up ctrld WFP filters, sublayer, and engine session ``` **Crash Recovery:** @@ -259,42 +272,24 @@ Windows DNS Client path to verify NRPT is actually working: 4. ctrld's DNS handler recognizes the probe prefix and signals success 5. If the probe times out (2s), NRPT isn't loaded yet → retry with remediation -### Startup Probe (Async) +### Startup Probes -After NRPT setup, an async goroutine runs the probe-and-heal sequence without -blocking startup: +For a matching GP candidate, startup blocks for one 2-second probe before any NRPT mutation and then re-reads the same child. A received query plus the unchanged rule proves both routing and external ownership. If that first probe fails, ctrld installs its normal WFP protection and performs one more ownership-safe probe before advertising startup readiness. + +ctrld-owned NRPT retains the asynchronous sequence: ``` -Probe attempt 1 (2s timeout) +Immediate probe ├─ Success → "NRPT verified working", done - └─ Timeout → GP refresh + DNS flush, sleep 1s - Probe attempt 2 (2s timeout) - ├─ Success → done - └─ Timeout → Restart DNS Client service (nuclear), sleep 2s - Re-add NRPT + GP refresh + DNS flush - Probe attempt 3 (2s timeout) - ├─ Success → done - └─ Timeout → GP refresh + DNS flush, sleep 4s - Probe attempt 4 (2s timeout) - ├─ Success → done - └─ Timeout → log error, continue + └─ Timeout + ├─ Empty GP parent → clean once, signal once, re-probe + └─ Otherwise → bounded 1s/2s/4s signal + probe retries + └─ Still failing → two-phase remove/signal/re-add/final probe ``` -### DNS Client Restart (Nuclear Option) +### GP-Managed Probe Failure -If GP refresh alone isn't enough, ctrld restarts the Windows DNS Client service -(`Dnscache`). This forces the DNS Client to fully re-initialize, including -re-reading all NRPT rules from the registry. This is the equivalent of macOS -`forceReloadPFMainRuleset()`. - -**Trade-offs:** -- Briefly interrupts ALL DNS resolution (few hundred ms during restart) -- Clears the system DNS cache (all apps need to re-resolve) -- VPN NRPT rules survive (they're in registry, re-read on restart) -- Enterprise security tools may log the service restart event - -This only fires as attempt #3 after two GP refresh attempts fail — at that point -DNS isn't working through ctrld anyway, so a brief DNS blip is acceptable. +A matching GP child still owns Windows' effective NRPT store even when its probe fails. Writing a local rule cannot override that precedence, and rewriting the GP child would violate administrator ownership. ctrld therefore retries only its loopback WFP permit protection, reports the ineffective external policy, and leaves NRPT registry values and policy signals untouched. ### Health Monitor Integration @@ -302,19 +297,20 @@ The 30s periodic health monitor now does actual probing, not just registry check ``` Every 30s: - ├─ Registry check: nrptCatchAllRuleExists()? - │ ├─ Missing → re-add + GP refresh + flush + probe-and-heal - │ └─ Present → probe to verify it's actually routing - │ ├─ Probe success → OK - │ └─ Probe failure → probe-and-heal cycle + ├─ GP-managed owner + │ ├─ Matching child + probe success → observe only + │ ├─ Matching child + probe failure → WFP-only retry; no NRPT mutation + │ └─ Matching child gone → activate ctrld-owned fallback + verify │ - └─ (hard mode only) Check: wfpSublayerExists()? - ├─ Missing → full restart (stopDNSIntercept + startDNSIntercept) - └─ Present → OK + ├─ ctrld-owned owner + │ ├─ Working matching GP child returns → remove only ctrld keys; adopt GP + │ ├─ ctrld key missing → restore + signal + verify + │ └─ ctrld key present → probe; run owned heal sequence on failure + │ + └─ (hard mode) Check WFP sublayer; full intercept restart if missing ``` -**Singleton guard:** Only one probe-and-heal sequence runs at a time (atomic bool). -The startup probe and health monitor cannot overlap. +**Singleton guard:** Only one asynchronous probe-and-heal sequence runs at a time (atomic bool). Startup's GP-candidate probe completes before the health monitor starts; direct periodic probes finish before they schedule a heal sequence. **Why periodic, not just network-event?** VPN software or Group Policy updates can clear NRPT at any time, not just during network changes. A 30s periodic check ensures From a828c8853a8d65f27eea35b06c71af146c489437 Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Mon, 10 Aug 2026 07:54:39 +0000 Subject: [PATCH 03/16] fix: retry macOS OS resolver with route-selected source --- resolver.go | 83 +++++++++++---- resolver_test.go | 267 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 331 insertions(+), 19 deletions(-) diff --git a/resolver.go b/resolver.go index a73bfa4..d84acf9 100644 --- a/resolver.go +++ b/resolver.go @@ -21,6 +21,7 @@ import ( "tailscale.com/net/tsaddr" "github.com/Control-D-Inc/ctrld/internal/dnscache" + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) const ( @@ -227,6 +228,10 @@ type osResolver struct { publicServers atomic.Pointer[[]string] group *singleflight.Group cache *sync.Map + // Per-resolver seams let tests exercise the production Resolve path without + // mutating process-wide resolver state. + exchangeDNS dnsExchangeFunc + localIP func(string) net.IP } type osResolverResult struct { @@ -342,21 +347,65 @@ func GetDefaultLocalIPv6() net.IP { return nil } -// customDNSExchange wraps the DNS exchange to use our debug dialer. -// It uses dns.ExchangeWithConn so that our custom dialer is used directly. -func customDNSExchange(ctx context.Context, msg *dns.Msg, server string, desiredLocalIP net.IP) (*dns.Msg, time.Duration, error) { +type dnsExchangeFunc func(context.Context, *dns.Msg, string, net.IP) (*dns.Msg, time.Duration, error) + +func exchangeDNS(ctx context.Context, msg *dns.Msg, server string, localIP net.IP) (*dns.Msg, time.Duration, error) { baseDialer := &net.Dialer{ Timeout: 3 * time.Second, Resolver: &net.Resolver{PreferGo: true}, } - if desiredLocalIP != nil { - baseDialer.LocalAddr = &net.UDPAddr{IP: desiredLocalIP, Port: 0} + if localIP != nil { + baseDialer.LocalAddr = &net.UDPAddr{IP: localIP, Port: 0} } dnsClient := &dns.Client{Net: "udp"} dnsClient.Dialer = baseDialer return dnsClient.ExchangeContext(ctx, msg, server) } +func defaultLocalIPForServer(server string) net.IP { + if runtime.GOOS != "darwin" { + return nil + } + host, _, err := net.SplitHostPort(server) + if err != nil { + return nil + } + ip := net.ParseIP(host) + if ip != nil && ip.To4() == nil { + return GetDefaultLocalIPv6() + } + return GetDefaultLocalIPv4() +} + +func preSendUnreachable(err error) bool { + var opErr *net.OpError + if !errors.As(err, &opErr) || (opErr.Op != "dial" && opErr.Op != "write") { + return false + } + return ctrldnet.IsUnreachable(err) +} + +// customDNSExchangeWith preserves the preferred source first. A route-selected +// retry is allowed only when the caller knows the server is an OS-selected resolver, +// not ctrld's synthetic public fallback. This includes public DNS pushed by a VPN: +// unbinding changes the source route, not the recipient. +func customDNSExchangeWith(ctx context.Context, msg *dns.Msg, server string, desiredLocalIP net.IP, allowRouteSelectedRetry bool, exchange dnsExchangeFunc) (*dns.Msg, time.Duration, error) { + answer, rtt, err := exchange(ctx, msg, server, desiredLocalIP) + if answer != nil || err == nil || ctx.Err() != nil || desiredLocalIP == nil || !allowRouteSelectedRetry || !preSendUnreachable(err) { + return answer, rtt, err + } + + Log(ctx, ProxyLogger.Load().Debug(), "OS resolver source binding is unreachable; retrying with route-selected source") + return exchange(ctx, msg.Copy(), server, nil) +} + +// allowRouteSelectedRetryForOSServer excludes only ctrld's synthetic public +// fallback. System-provided resolvers remain eligible even when their addresses +// are public, as with VPNs that push public DNS servers. +func allowRouteSelectedRetryForOSServer(server string) bool { + return server != controldPublicDnsWithPort +} + const hotCacheTTL = time.Second // Resolve resolves DNS queries using pre-configured nameservers. @@ -465,6 +514,14 @@ func (o *osResolver) resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error ch := make(chan *osResolverResult, numServers) wg := &sync.WaitGroup{} + exchange := o.exchangeDNS + if exchange == nil { + exchange = exchangeDNS + } + localIPForServer := o.localIP + if localIPForServer == nil { + localIPForServer = defaultLocalIPForServer + } wg.Add(numServers) go func() { wg.Wait() @@ -484,20 +541,8 @@ func (o *osResolver) resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error var answer *dns.Msg var err error - var localOSResolverIP net.IP - if runtime.GOOS == "darwin" { - host, _, err := net.SplitHostPort(server) - if err == nil { - ip := net.ParseIP(host) - if ip != nil && ip.To4() == nil { - // IPv6 nameserver; use default IPv6 address (if set) - localOSResolverIP = GetDefaultLocalIPv6() - } else { - localOSResolverIP = GetDefaultLocalIPv4() - } - } - } - answer, _, err = customDNSExchange(ctx, msg.Copy(), server, localOSResolverIP) + localOSResolverIP := localIPForServer(server) + answer, _, err = customDNSExchangeWith(ctx, msg.Copy(), server, localOSResolverIP, allowRouteSelectedRetryForOSServer(server), exchange) ch <- &osResolverResult{answer: answer, err: err, server: server, lan: isLan} }(server) } diff --git a/resolver_test.go b/resolver_test.go index 30a3af7..930871a 100644 --- a/resolver_test.go +++ b/resolver_test.go @@ -4,9 +4,12 @@ import ( "context" "crypto/rand" "encoding/hex" + "errors" "net" + "os" "sync" "sync/atomic" + "syscall" "testing" "time" @@ -70,6 +73,270 @@ func Test_osResolver_ResolveLanHostname(t *testing.T) { } } +func Test_customDNSExchangeWith_RetriesUnboundOnUnreachableSource(t *testing.T) { + tests := []struct { + name string + boundIP net.IP + server string + errno syscall.Errno + }{ + {"ipv4 network unreachable", net.ParseIP("192.0.2.10"), "192.0.2.53:53", syscall.ENETUNREACH}, + {"ipv4 host unreachable", net.ParseIP("192.0.2.10"), "192.0.2.53:53", syscall.EHOSTUNREACH}, + {"ipv6 network unreachable", net.ParseIP("2001:db8::10"), "[2001:db8::53]:53", syscall.ENETUNREACH}, + {"ipv6 host unreachable", net.ParseIP("2001:db8::10"), "[2001:db8::53]:53", syscall.EHOSTUNREACH}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + var localIPs []net.IP + var servers []string + exchange := func(_ context.Context, msg *dns.Msg, server string, localIP net.IP) (*dns.Msg, time.Duration, error) { + localIPs = append(localIPs, append(net.IP(nil), localIP...)) + servers = append(servers, server) + if localIP != nil { + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: &os.SyscallError{Syscall: "write", Err: tt.errno}} + } + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + answer, _, err := customDNSExchangeWith(context.Background(), msg, tt.server, tt.boundIP, true, exchange) + if err != nil { + t.Fatal(err) + } + if answer == nil { + t.Fatal("expected answer from route-selected retry") + } + if len(localIPs) != 2 { + t.Fatalf("exchange calls: got %d, want 2", len(localIPs)) + } + if !localIPs[0].Equal(tt.boundIP) { + t.Fatalf("first source: got %v, want %v", localIPs[0], tt.boundIP) + } + if localIPs[1] != nil { + t.Fatalf("retry source: got %v, want route-selected nil", localIPs[1]) + } + if len(servers) != 2 || servers[0] != tt.server || servers[1] != tt.server { + t.Fatalf("exchange servers: got %v, want two attempts to %s", servers, tt.server) + } + }) + } +} + +func Test_customDNSExchangeWith_PreservesReachableBoundSource(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + boundIP := net.ParseIP("192.0.2.10") + calls := 0 + exchange := func(_ context.Context, msg *dns.Msg, _ string, localIP net.IP) (*dns.Msg, time.Duration, error) { + calls++ + if !localIP.Equal(boundIP) { + t.Fatalf("source: got %v, want %v", localIP, boundIP) + } + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + answer, _, err := customDNSExchangeWith(context.Background(), msg, "192.0.2.53:53", boundIP, true, exchange) + if err != nil { + t.Fatal(err) + } + if answer == nil { + t.Fatal("expected answer from bound exchange") + } + if calls != 1 { + t.Fatalf("exchange calls: got %d, want 1", calls) + } +} + +func Test_customDNSExchangeWith_DoesNotRetryOtherFailures(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + calls := 0 + exchange := func(_ context.Context, _ *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + calls++ + return nil, 0, context.DeadlineExceeded + } + + _, _, err := customDNSExchangeWith(context.Background(), msg, "192.0.2.53:53", net.ParseIP("192.0.2.10"), true, exchange) + if err == nil { + t.Fatal("expected exchange failure") + } + if calls != 1 { + t.Fatalf("exchange calls: got %d, want 1", calls) + } +} + +func Test_customDNSExchangeWith_DoesNotRetryWithoutBoundSource(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + calls := 0 + exchange := func(_ context.Context, _ *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + calls++ + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: syscall.EHOSTUNREACH} + } + + _, _, err := customDNSExchangeWith(context.Background(), msg, "192.0.2.53:53", nil, true, exchange) + if err == nil { + t.Fatal("expected exchange failure") + } + if calls != 1 { + t.Fatalf("exchange calls: got %d, want 1", calls) + } +} + +func Test_customDNSExchangeWith_DoesNotRetryCanceledContext(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + calls := 0 + exchange := func(_ context.Context, _ *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + calls++ + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: syscall.EHOSTUNREACH} + } + + _, _, err := customDNSExchangeWith(ctx, msg, "192.0.2.53:53", net.ParseIP("192.0.2.10"), true, exchange) + if err == nil { + t.Fatal("expected exchange failure") + } + if calls != 1 { + t.Fatalf("exchange calls: got %d, want 1", calls) + } +} + +func Test_customDNSExchangeWith_ReturnsUnboundRetryFailure(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + retryErr := errors.New("route-selected exchange failed") + calls := 0 + exchange := func(_ context.Context, _ *dns.Msg, _ string, localIP net.IP) (*dns.Msg, time.Duration, error) { + calls++ + if localIP != nil { + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: syscall.EHOSTUNREACH} + } + return nil, 0, retryErr + } + + _, _, err := customDNSExchangeWith(context.Background(), msg, "192.0.2.53:53", net.ParseIP("192.0.2.10"), true, exchange) + if !errors.Is(err, retryErr) { + t.Fatalf("exchange error: got %v, want retry error %v", err, retryErr) + } + if calls != 2 { + t.Fatalf("exchange calls: got %d, want 2", calls) + } +} + +func Test_customDNSExchangeWith_DoesNotRetryReadSideUnreachable(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + calls := 0 + exchange := func(_ context.Context, _ *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + calls++ + return nil, 0, &net.OpError{Op: "read", Net: "udp", Err: syscall.EHOSTUNREACH} + } + + _, _, err := customDNSExchangeWith(context.Background(), msg, "192.0.2.53:53", net.ParseIP("192.0.2.10"), true, exchange) + if err == nil { + t.Fatal("expected exchange failure") + } + if calls != 1 { + t.Fatalf("exchange calls: got %d, want 1", calls) + } +} + +func Test_osResolver_ResolveUsesRouteSelectedFallbackForLANServer(t *testing.T) { + const server = "10.0.0.53:53" + boundIP := net.ParseIP("192.0.2.10") + resolver := newResolverWithNameserver([]string{server}) + resolver.localIP = func(string) net.IP { return boundIP } + + var localIPs []net.IP + var servers []string + resolver.exchangeDNS = func(_ context.Context, msg *dns.Msg, gotServer string, localIP net.IP) (*dns.Msg, time.Duration, error) { + servers = append(servers, gotServer) + localIPs = append(localIPs, append(net.IP(nil), localIP...)) + if localIP != nil { + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: syscall.EHOSTUNREACH} + } + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + answer, err := resolver.Resolve(context.Background(), msg) + if err != nil { + t.Fatal(err) + } + if answer == nil { + t.Fatal("expected answer from route-selected retry") + } + if len(localIPs) != 2 || !localIPs[0].Equal(boundIP) || localIPs[1] != nil { + t.Fatalf("exchange sources: got %v, want [%v ]", localIPs, boundIP) + } + if len(servers) != 2 || servers[0] != server || servers[1] != server { + t.Fatalf("exchange servers: got %v, want two attempts to %s", servers, server) + } +} + +// A VPN-pushed public DNS address is categorized as public by IP, but it is +// still a system-selected resolver and must get the same route-compatible retry. +func Test_osResolver_ResolveUsesRouteSelectedFallbackForPublicVPNServer(t *testing.T) { + const server = "192.0.2.53:53" + boundIP := net.ParseIP("198.51.100.10") + resolver := newResolverWithNameserver([]string{server}) + resolver.localIP = func(string) net.IP { return boundIP } + + var localIPs []net.IP + resolver.exchangeDNS = func(_ context.Context, msg *dns.Msg, _ string, localIP net.IP) (*dns.Msg, time.Duration, error) { + localIPs = append(localIPs, append(net.IP(nil), localIP...)) + if localIP != nil { + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: syscall.EHOSTUNREACH} + } + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + answer, err := resolver.Resolve(context.Background(), msg) + if err != nil { + t.Fatal(err) + } + if answer == nil { + t.Fatal("expected answer from route-selected retry") + } + if len(localIPs) != 2 || !localIPs[0].Equal(boundIP) || localIPs[1] != nil { + t.Fatalf("exchange sources: got %v, want [%v ]", localIPs, boundIP) + } +} + +func Test_osResolver_ResolveDoesNotRetrySyntheticControlDFallbackUnbound(t *testing.T) { + resolver := newResolverWithNameserver([]string{controldPublicDnsWithPort}) + resolver.localIP = func(string) net.IP { return net.ParseIP("198.51.100.10") } + calls := 0 + resolver.exchangeDNS = func(_ context.Context, _ *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + calls++ + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: syscall.EHOSTUNREACH} + } + + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + _, err := resolver.Resolve(context.Background(), msg) + if err == nil { + t.Fatal("expected exchange failure") + } + if calls != 1 { + t.Fatalf("exchange calls: got %d, want one bound synthetic fallback attempt", calls) + } +} + func Test_osResolver_ResolveWithNonSuccessAnswer(t *testing.T) { // Set up a LAN nameserver that returns a success response. lanPC, err := net.ListenPacket("udp", "127.0.0.1:0") // 127.0.0.1 is considered LAN (loopback) From d7c30b18ed42606830066449c6785d36b31bffe7 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 10 Aug 2026 17:53:38 +0700 Subject: [PATCH 04/16] fix(darwin): probe interception when stabilization finishes The post-stabilization reconcile verifies rule text, which cannot tell a live redirect from an anchor pf has stopped evaluating. Sleep/wake QA caught exactly that split: references intact, anchor rules intact, post-load verification passed, and every query through the system resolver timing out while the direct listener answered. Nothing else probed. The interception probe monitor stands down while stabilization owns pf and is never re-armed afterwards, so functional recovery waited for the periodic watchdog - 11 seconds in the captured run, up to a full 30-second interval - on a host whose link and default route were already back. The watchdog's probe then failed once, forced a reload, and public and VPN split-DNS both recovered immediately. Probe once at the end of stabilization and, if it fails, force exactly one reload and confirm with one more probe. Not the probe monitor: that keeps probing for ~7.5s and can force a reload per failed probe, where this path needs a single bounded repair before handing back to the watchdog. Skipped when a monitor already owns probing, when intercept state is gone, or during exec backoff. Hand ownership over deterministically rather than skipping on sight. A probe monitor started by an ignored network change claimed functional-probe ownership before checking whether it could work, then stood down because stabilization still owned pf; the verifier read that claimed flag as "somebody is probing" and skipped, so neither path probed and recovery fell back to the watchdog anyway. The monitor now checks eligibility before claiming, and the verifier waits out a holder that releases, yielding only to one that keeps probing. Extract the completion block into finishPFStabilization so the wiring is testable, and cover the bounded repair, the healthy path that must not reload, a prober that claims and stands down, a prober that keeps working, and a monitor that must not claim ownership while stabilizing. --- cmd/cli/dns_intercept_darwin.go | 141 ++++++++++++++++++--- cmd/cli/dns_intercept_darwin_test.go | 180 +++++++++++++++++++++++++++ 2 files changed, 303 insertions(+), 18 deletions(-) diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go index 6b30941..e828103 100644 --- a/cmd/cli/dns_intercept_darwin.go +++ b/cmd/cli/dns_intercept_darwin.go @@ -1403,25 +1403,123 @@ func (p *prog) pfStabilizationLoopWithMaxWait(ctx context.Context, stableRequire } if time.Since(stableSince) >= stableRequired { - // The active loop retains ownership until the dedicated post-stable - // repair finishes. It never re-enters stabilization recursively. - mainLog.Load().Info().Msgf("DNS intercept: pf stable for %s — reconciling anchor rules", stableRequired) - result := p.reconcilePFAnchorAfterStabilization() - if result != pfAnchorCheckRestored && result != pfAnchorCheckIntact { - p.scheduleDelayedRechecks() - } - routes, domainlessServers, exemptions := p.refreshDNSAfterVPNSettle("pf_stabilized") - if routes == 0 && domainlessServers == 0 && exemptions == 0 { - p.scheduleDNSAfterVPNSettleRefresh("pf_stabilized_followup", pfAnchorRecheckDelayLong) - } - if p.hasPendingTunnelReconcile() { - p.scheduleDelayedRechecks() - } + p.finishPFStabilization(stableRequired) return } } } +// finishPFStabilization runs the work stabilization exists to do, once the ruleset has +// held still for the required window. The active loop retains ownership throughout: it +// never re-enters stabilization recursively. +func (p *prog) finishPFStabilization(stableRequired time.Duration) { + mainLog.Load().Info().Msgf("DNS intercept: pf stable for %s — reconciling anchor rules", stableRequired) + result := p.reconcilePFAnchorAfterStabilization() + if result != pfAnchorCheckRestored && result != pfAnchorCheckIntact { + p.scheduleDelayedRechecks() + } + routes, domainlessServers, exemptions := p.refreshDNSAfterVPNSettle("pf_stabilized") + if routes == 0 && domainlessServers == 0 && exemptions == 0 { + p.scheduleDNSAfterVPNSettleRefresh("pf_stabilized_followup", pfAnchorRecheckDelayLong) + } + if p.hasPendingTunnelReconcile() { + p.scheduleDelayedRechecks() + } + p.verifyInterceptAfterStabilization() +} + +// probePFInterceptFn and forceReloadPFInterceptFn are the functional verification seams +// shared by both probers. +var ( + probePFInterceptFn = (*prog).probePFIntercept + forceReloadPFInterceptFn = (*prog).forceReloadPFMainRuleset +) + +// pfFunctionalProbeOwnerWait bounds how long the post-stabilization verifier waits for +// another prober to release functional-probe ownership. A probe monitor that is only +// standing down releases it at once; one that is genuinely probing holds it for its whole +// window, and then the verifier steps aside. A var so tests can shorten the wait. +var pfFunctionalProbeOwnerWait = 2 * time.Second + +// pfFunctionalProbeOwnerPoll is how often that wait re-tries the claim. +const pfFunctionalProbeOwnerPoll = 25 * time.Millisecond + +// interceptProbeMonitorAllowed reports whether the probe monitor may run at all. +// +// The monitor must consult this before claiming ownership. Claiming first and checking +// second means a monitor that is about to stand down still takes the flag, and the +// post-stabilization verifier - which sees a set flag as "somebody else is probing" - +// skips. Neither probes, and the outage lasts until the next watchdog tick. +func (p *prog) interceptProbeMonitorAllowed() bool { + return p.dnsInterceptState != nil && !p.pfStabilizing.Load() +} + +// claimFunctionalProbeOwner takes ownership of functional probing, waiting up to wait for +// a current owner to release it. It reports whether ownership was acquired; the caller +// releases with pfMonitorRunning.Store(false). +func (p *prog) claimFunctionalProbeOwner(wait time.Duration) bool { + deadline := time.Now().Add(wait) + for { + if p.pfMonitorRunning.CompareAndSwap(false, true) { + return true + } + if !time.Now().Before(deadline) { + return false + } + time.Sleep(pfFunctionalProbeOwnerPoll) + } +} + +// verifyInterceptAfterStabilization proves pf is actually translating once stabilization +// has finished, and repairs it once if it is not. +// +// The reconcile above verifies rule text. That cannot distinguish a live redirect from an +// anchor pf has stopped evaluating, and after sleep/wake with a VPN reconnect those come +// apart: rules present, references present, post-load verification passed, and every query +// through the system resolver timing out. Nothing else notices until the periodic watchdog +// runs its own probe - the interception probe monitor stands down while stabilization owns +// pf and is not re-armed afterwards - so recovery waits up to a full watchdog interval on a +// host whose link and default route are already back. +// +// One probe, then at most one forced reload and one confirming probe. Deliberately not the +// probe monitor: that keeps probing for ~7.5s and can force a reload per failed probe, +// where this path needs a single bounded repair and then hands back to the watchdog. +func (p *prog) verifyInterceptAfterStabilization() { + if p.dnsInterceptState == nil || p.pfExecBackoffActive() { + return + } + // The probe monitor is the other functional prober, so only one of us may run - but + // "somebody holds the flag" is not the same as "somebody is probing". A monitor that + // started while stabilization owns pf stands down immediately, and skipping on sight + // left nobody probing at all. Wait briefly for the holder to release instead. + if !p.claimFunctionalProbeOwner(pfFunctionalProbeOwnerWait) { + mainLog.Load().Warn().Msgf("DNS intercept: post-stabilization probe skipped — another prober held ownership for %s; leaving recovery to the watchdog", pfFunctionalProbeOwnerWait) + return + } + defer p.pfMonitorRunning.Store(false) + + // Ownership can take a moment to arrive; make sure there is still an intercept to check. + if p.dnsInterceptState == nil { + return + } + + if probePFInterceptFn(p) { + mainLog.Load().Debug().Msg("DNS intercept: post-stabilization probe passed — interception is translating") + return + } + + mainLog.Load().Warn().Msg("DNS intercept: post-stabilization rules are intact but the probe FAILED — forcing one reload") + if !forceReloadPFInterceptFn(p) { + mainLog.Load().Error().Msg("DNS intercept: post-stabilization forced reload did not run — leaving recovery to the watchdog") + return + } + if probePFInterceptFn(p) { + mainLog.Load().Info().Msg("DNS intercept: interception restored by the post-stabilization reload") + return + } + mainLog.Load().Error().Msg("DNS intercept: interception still not translating after the post-stabilization reload — the watchdog will retry") +} + var runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { return exec.Command("pfctl", args...).CombinedOutput() } @@ -1930,6 +2028,13 @@ func buildDNSQueryPacket(domain string) []byte { // The backoff schedule provides both fast detection (immediate + 500ms) and extended // coverage (up to ~8s) to win the race against async pf reloads by hypervisors. func (p *prog) pfInterceptMonitor() { + // Eligibility first, ownership second. A monitor that is about to stand down must not + // take the flag on its way out: the post-stabilization verifier reads that flag as + // "another prober is working" and would step aside for a prober that never probes. + if !p.interceptProbeMonitorAllowed() { + mainLog.Load().Debug().Msg("DNS intercept monitor: not starting — intercept disabled or stabilizing") + return + } if !p.pfMonitorRunning.CompareAndSwap(false, true) { mainLog.Load().Debug().Msg("DNS intercept monitor: already running, skipping") return @@ -1946,23 +2051,23 @@ func (p *prog) pfInterceptMonitor() { if delay > 0 { time.Sleep(delay) } - if p.dnsInterceptState == nil || p.pfStabilizing.Load() { + if !p.interceptProbeMonitorAllowed() { mainLog.Load().Debug().Msg("DNS intercept monitor: aborting — intercept disabled or stabilizing") return } - if p.probePFIntercept() { + if probePFInterceptFn(p) { mainLog.Load().Debug().Msgf("DNS intercept monitor: probe %d/%d passed", i+1, len(delays)) continue // working now — keep monitoring in case it breaks later in the window } // Probe failed — pf translation is broken. Force full reload. mainLog.Load().Warn().Msgf("DNS intercept monitor: probe %d/%d FAILED — pf translation broken, forcing full ruleset reload", i+1, len(delays)) - p.forceReloadPFMainRuleset() + forceReloadPFInterceptFn(p) // Verify the reload fixed it time.Sleep(200 * time.Millisecond) - if p.probePFIntercept() { + if probePFInterceptFn(p) { mainLog.Load().Info().Msg("DNS intercept monitor: probe passed after reload — interception restored") // Continue monitoring in case the hypervisor reloads pf again } else { diff --git a/cmd/cli/dns_intercept_darwin_test.go b/cmd/cli/dns_intercept_darwin_test.go index e375d08..f8515c5 100644 --- a/cmd/cli/dns_intercept_darwin_test.go +++ b/cmd/cli/dns_intercept_darwin_test.go @@ -828,3 +828,183 @@ func TestExemptVPNDNSServersDeferredWhileStabilizing(t *testing.T) { t.Error("pfEnsureRunning was left held by a deferred exemption") } } + +// stubStabilizationProbe replaces the post-stabilization verification seams and returns +// counters for probe and forced-reload calls. +func stubStabilizationProbe(t *testing.T, probeResults []bool, reloadOK bool) (probes, reloads *int) { + t.Helper() + originalProbe, originalReload := probePFInterceptFn, forceReloadPFInterceptFn + t.Cleanup(func() { + probePFInterceptFn, forceReloadPFInterceptFn = originalProbe, originalReload + }) + probeCalls, reloadCalls := 0, 0 + probePFInterceptFn = func(*prog) bool { + result := false + if probeCalls < len(probeResults) { + result = probeResults[probeCalls] + } + probeCalls++ + return result + } + forceReloadPFInterceptFn = func(*prog) bool { + reloadCalls++ + return reloadOK + } + return &probeCalls, &reloadCalls +} + +// TestPostStabilizationVerifiesInterceptionFunctionally is the post-wake continuity +// boundary: the reconcile above it only proves rule text, and QA saw rules intact, +// references intact and post-load verification passed while every query through the system +// resolver timed out. Nothing else probes until the periodic watchdog, because the probe +// monitor stands down while stabilization owns pf, so recovery waited for that tick. +func TestPostStabilizationVerifiesInterceptionFunctionally(t *testing.T) { + // Probe fails once, then passes after the reload. + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterStabilization() + + if *probes != 2 { + t.Errorf("probe calls = %d, want 2: one to detect and one to confirm the repair", *probes) + } + if *reloads != 1 { + t.Errorf("forced reloads = %d, want exactly 1 bounded repair", *reloads) + } +} + +// TestPostStabilizationProbePassSkipsReload keeps the healthy path free of a pf reload, +// which would flush states and kill in-flight DoH connections for nothing. +func TestPostStabilizationProbePassSkipsReload(t *testing.T) { + probes, reloads := stubStabilizationProbe(t, []bool{true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterStabilization() + + if *probes != 1 || *reloads != 0 { + t.Errorf("probe calls = %d, forced reloads = %d, want 1/0", *probes, *reloads) + } +} + +// TestPostStabilizationRepairIsBounded pins the "one bounded recovery" contract: a probe +// that never passes must not turn into a reload loop here - the watchdog owns retries. +func TestPostStabilizationRepairIsBounded(t *testing.T) { + probes, reloads := stubStabilizationProbe(t, []bool{false, false, false}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterStabilization() + + if *reloads != 1 { + t.Errorf("forced reloads = %d, want 1: the repair must not loop", *reloads) + } + if *probes != 2 { + t.Errorf("probe calls = %d, want 2", *probes) + } +} + +// TestPostStabilizationWaitsForAProberThatStandsDown is the interleaving that made +// "skip when the flag is set" wrong. A probe monitor started by an ignored network change +// claims functional-probe ownership and then aborts, because stabilization still owns pf. +// If the verifier treats the claimed flag as "somebody is probing", neither path probes and +// the outage lasts until the next watchdog tick - the exact window this is meant to close. +func TestPostStabilizationWaitsForAProberThatStandsDown(t *testing.T) { + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + // Model the monitor's claim-then-abort: ownership is held, then released. + p.pfMonitorRunning.Store(true) + released := make(chan struct{}) + go func() { + time.Sleep(50 * time.Millisecond) + p.pfMonitorRunning.Store(false) + close(released) + }() + + p.verifyInterceptAfterStabilization() + <-released + + if *probes != 2 { + t.Errorf("probe calls = %d, want 2: the verifier must wait out a prober that stands down", *probes) + } + if *reloads != 1 { + t.Errorf("forced reloads = %d, want 1", *reloads) + } +} + +// TestPostStabilizationYieldsToAProberThatKeepsProbing is the other half of the handoff: +// when the holder is genuinely working through its probe sequence, the verifier must step +// aside rather than run a second prober against the same pf state. +func TestPostStabilizationYieldsToAProberThatKeepsProbing(t *testing.T) { + originalWait := pfFunctionalProbeOwnerWait + pfFunctionalProbeOwnerWait = 30 * time.Millisecond + t.Cleanup(func() { pfFunctionalProbeOwnerWait = originalWait }) + + probes, reloads := stubStabilizationProbe(t, []bool{false}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfMonitorRunning.Store(true) // held for the whole wait + p.verifyInterceptAfterStabilization() + + if *probes != 0 || *reloads != 0 { + t.Errorf("probe calls = %d, forced reloads = %d, want 0/0 while another prober is working", *probes, *reloads) + } +} + +// TestInterceptMonitorDoesNotClaimOwnershipWhileStabilizing pins the source of that race: +// a monitor which cannot do useful work must not take functional-probe ownership on its way +// out, or it starves the post-stabilization verifier. +func TestInterceptMonitorDoesNotClaimOwnershipWhileStabilizing(t *testing.T) { + probes, reloads := stubStabilizationProbe(t, []bool{false}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + if p.interceptProbeMonitorAllowed() { + t.Fatal("the probe monitor considers itself eligible while stabilization owns pf") + } + + p.pfInterceptMonitor() + + if *probes != 0 || *reloads != 0 { + t.Errorf("probe calls = %d, forced reloads = %d, want 0/0 from a monitor that cannot run", *probes, *reloads) + } + if !p.claimFunctionalProbeOwner(0) { + t.Error("the aborted monitor left functional-probe ownership taken; the verifier would skip") + } + p.pfMonitorRunning.Store(false) +} + +// TestFinishPFStabilizationRunsFunctionalVerification wires the fix to the production +// completion path: deleting the verification call, or reordering it before the reconcile, +// makes this fail. +func TestFinishPFStabilizationRunsFunctionalVerification(t *testing.T) { + stubPFAnchorCheckCommand(t, map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", + }) + originalResolver := initializeOsResolver + initializeOsResolver = func(bool) []string { return nil } + t.Cleanup(func() { initializeOsResolver = originalResolver }) + + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + p.finishPFStabilization(time.Millisecond) + + if *probes == 0 { + t.Fatal("stabilization completed without probing functional interception; recovery would wait for the watchdog") + } + if *reloads != 1 { + t.Errorf("forced reloads = %d, want 1", *reloads) + } + + p.pfDelayedRecheckMu.Lock() + timers := append([]*time.Timer(nil), p.pfDelayedRecheckTimers...) + p.pfDelayedRecheckTimers = nil + p.pfDelayedRecheckMu.Unlock() + for _, timer := range timers { + timer.Stop() + } +} From dfaad4a20d9eaa2406a9fa7c82a9d589a685ed78 Mon Sep 17 00:00:00 2001 From: Codescribe Date: Sat, 28 Mar 2026 20:37:09 -0400 Subject: [PATCH 05/16] Redact provision token in logs --- cmd/cli/cli.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 4428d10..5cd9d1d 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -1646,7 +1646,7 @@ func cdUIDFromProvToken() string { // Process provision token if provided. resolverConfig, err := controld.FetchResolverUID(req, rootCmd.Version, cdDev) if err != nil { - mainLog.Load().Fatal().Err(err).Msgf("failed to fetch resolver uid with provision token: %s", cdOrg) + mainLog.Load().Fatal().Err(err).Msgf("failed to fetch resolver uid with provision token: %s", redactToken(cdOrg)) } return resolverConfig.UID } @@ -2104,3 +2104,12 @@ func uninstallInvalidCdUID(p *prog, logger zerolog.Logger, doStop bool) bool { } return false } + +// redactToken returns the first 4 characters of a token followed by ***, +// or just *** if the token is 4 characters or shorter. +func redactToken(s string) string { + if len(s) <= 4 { + return "***" + } + return s[:4] + "***" +} From b74937fcf3552486fdee6b950fbb70ac1edb6ecb Mon Sep 17 00:00:00 2001 From: Codescribe Date: Sat, 28 Mar 2026 20:35:23 -0400 Subject: [PATCH 06/16] security: default metrics server to loopback Fixes unauthenticated metrics exposure be defaulting to 127.0.0.1 when no host is provided. Logs a warning when bound to non-loopback addresses. --- cmd/cli/metrics.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmd/cli/metrics.go b/cmd/cli/metrics.go index 565cdcc..9597883 100644 --- a/cmd/cli/metrics.go +++ b/cmd/cli/metrics.go @@ -113,6 +113,22 @@ func (p *prog) runMetricsServer(ctx context.Context, reloadCh chan struct{}) { } addr := p.cfg.Service.MetricsListener + if addr != "" { + host, port, err := net.SplitHostPort(addr) + if err != nil { + mainLog.Load().Warn().Err(err).Msgf("Invalid metrics listener address (%s); expected host:port", addr) + } else { + if host == "" { + host = "127.0.0.1" + addr = net.JoinHostPort(host, port) + } + ip := net.ParseIP(host) + if (ip != nil && !ip.IsLoopback()) || (ip == nil && host != "localhost") { + mainLog.Load().Warn().Msgf("Metrics server is bound to a non-loopback address (%s). This exposes sensitive data without authentication.", addr) + } + } + } + ms, err := newMetricsServer(addr, reg) if err != nil { mainLog.Load().Warn().Err(err).Msg("could not create new metrics server") From 4f730167d498035976cfa13f3cc049b9b11cc225 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 14 Aug 2026 15:17:23 +0700 Subject: [PATCH 07/16] cmd/cli: bound API preflight by service lifetime processCDFlags retries the resolver-config fetch indefinitely by design: a device that has no working network at boot must eventually come up. The loop had no cancellation, so a stop request arriving while the API is unreachable was ignored - the process kept retrying long after the service reported itself stopped, doing work on behalf of a service the OS considers stopped. Thread a context through processCDFlags and derive it from p.stopCh, in both the startup preflight and the config-reload path. The loop now returns as soon as the context is cancelled, checked both before a retry and after backoff returns (backoff can wake up on cancellation). A stop during preflight exits the way a normal stop does, without Fatal, so the service manager does not treat it as a failed start and apply its restart policy to a service the operator just asked to stop. Bind the two API requests themselves as well, so a stop does not have to wait out an in-flight request. Without this the loop honours a stop only between attempts, which leaves up to defaultTimeout (20s) of a request the service is no longer interested in - the same "still working after Service stopped" the loop change exists to end, one layer down. Doing so means a context parameter on FetchResolverConfig, FetchResolverUID, UpdateCustomLastFailed and SendLogs, since all four reach a request builder. The callers that have no context pass context.Background(), which is what master effectively does at those sites: its loggerCtx carries a logger, not cancellation. doWithFallback needs no parameter, because it clones the request with req.Context() and so inherits the binding. This also repairs internal/controld/controld_test.go, which is behind //go:build controld and had already been written against the context-taking signature, so it could not compile. Cover the cancellation paths; removing either check makes the tests hang until timeout. Sampling the stop state is the whole point of runAPIPreflight rather than doing this inline. A stop and a failure need opposite handling - one exits quietly, the other self-uninstalls a deleted device, surfaces the error to a mobile app, and reports a failed start - so the two must not be confused. Reading it from the context after cancelling would report "stopped" for every failure, since CancelFunc sets ctx.Err() regardless of whether anyone asked to stop; the stop channel is read directly instead, which also does not depend on the context watcher goroutine having been scheduled. --- cmd/cli/cli.go | 183 +++++++++++++- cmd/cli/cli_preflight_test.go | 403 +++++++++++++++++++++++++++++++ cmd/cli/control_server.go | 4 +- cmd/cli/dns_proxy.go | 2 +- cmd/cli/prog.go | 6 +- internal/controld/config.go | 54 +++-- internal/controld/config_test.go | 67 +++++ 7 files changed, 686 insertions(+), 33 deletions(-) create mode 100644 cmd/cli/cli_preflight_test.go diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 5cd9d1d..610dfc4 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -318,23 +318,56 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { } if cdUID != "" { validateCdUpstreamProtocol() - if rc, err := processCDFlags(&cfg); err != nil { + // Bound API preflight by the service lifetime. Without this, a stop request + // arriving while the API is unreachable leaves this retry/backoff loop running + // after "service stopped" was logged, so the process keeps working on behalf of + // a service the OS considers stopped. + pf := runAPIPreflight(p.stopCh, &cfg) + switch { + case pf.stopRequested: + // Stop requested during preflight, whether or not the fetch itself + // succeeded. A successful fetch does not entitle startup to continue: the + // operator asked for a stop, and carrying on would set up listeners and + // interception for a service the OS already considers stopping. + // + // Exit the way a normal stop does: no Fatal, so the OS service manager does + // not see a failed start and apply its restart policy to a service the + // operator just asked to stop. + mainLog.Load().Notice().Msg("stop requested while fetching resolver config, shutting down") + notifyExitToLogServer() + return + case pf.err != nil: if isMobile() { - appCallback.Exit(err.Error()) + appCallback.Exit(pf.err.Error()) return } cdLogger := mainLog.Load().With().Str("mode", "cd").Logger() // Performs self-uninstallation if the ControlD device does not exist. var uer *controld.ErrorResponse - if errors.As(err, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode { + if errors.As(pf.err, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode { _ = uninstallInvalidCdUID(p, cdLogger, false) } + if rejection, ok := permanentAPIRejection(pf.err); ok { + // The API answered and rejected this request permanently. Restarting + // cannot change that answer, so exit cleanly rather than through Fatal: + // an abnormal exit spends one of the service manager's restart actions, + // and on Windows those are what bring enforcement back after a real + // crash. Burning that budget on a config problem also buries the API's + // reason under repeated start failures. + cdLogger.Error().Err(pf.err).Int("status", rejection.StatusCode).Msg("failed to fetch resolver config, the API rejected this configuration") + notifyExitToLogServer() + return + } notifyExitToLogServer() - cdLogger.Fatal().Err(err).Msg("failed to fetch resolver config") - } else { + // Everything else - a denied socket, an unreachable API, a proxy in the way, + // an API that is having a bad day - is a condition a later start may not hit, + // so keep the abnormal exit and let the service manager's recovery policy + // retry. + cdLogger.Fatal().Err(pf.err).Msg("failed to fetch resolver config") + default: p.mu.Lock() - p.rc = rc + p.rc = pf.rc p.mu.Unlock() } } @@ -649,24 +682,148 @@ func deactivationPinSet() bool { return cdDeactivationPin.Load() != defaultDeactivationPin } -func processCDFlags(cfg *ctrld.Config) (*controld.ResolverConfig, error) { +// fetchResolverConfig is a test seam for the ControlD resolver-config API call. +var fetchResolverConfig = controld.FetchResolverConfig + +// apiPreflight is the outcome of the API preflight fetch: the resolver config, the +// error if any, and whether the service was asked to stop while it ran. +type apiPreflight struct { + rc *controld.ResolverConfig + err error + stopRequested bool +} + +// runAPIPreflight fetches the ControlD resolver config bounded by the service +// lifetime, and reports whether a stop was requested while it ran. +// +// The distinction matters because the caller does very different things with it: a stop +// exits quietly, while a failure self-uninstalls a deleted device, surfaces the error to +// a mobile app, and reports a failed start to the service manager. +// +// stopRequested must not be derived from the context once it has been cancelled. +// context.CancelFunc sets ctx.Err() unconditionally, so reading it after the cancel +// classifies *every* failure - a deleted device, an exhausted retry, a mobile caller +// with no stop channel - as an operator stop. Reading the stop channel directly is also +// independent of whether the context's watcher goroutine has been scheduled yet. +func runAPIPreflight(stopCh <-chan struct{}, cfg *ctrld.Config) apiPreflight { + rc, err := fetchCDConfigBoundedBy(stopCh, cfg) + return apiPreflight{rc: rc, err: err, stopRequested: stopRequested(stopCh)} +} + +// permanentAPIRejection reports whether err is the API refusing this request in a way +// that a restart cannot change, and returns the rejection when it is. +// +// The type alone does not answer this. controld builds an *ErrorResponse for *any* +// non-200 whose body decodes, so a 502 from a load balancer and a 404 for a deleted +// device arrive as the same Go type. Treating both as permanent would let a few minutes +// of API trouble stop ctrld on every host with no service-manager retry behind it, which +// is strictly worse than the abnormal exit it replaced. +// +// So the HTTP status decides, and only a client-error status counts: +// +// - 4xx: the API examined this request and refused it - a deleted device, a revoked +// token, a malformed UID. The same request will be refused again. +// - 408 and 429 are the exceptions: they are the API asking for another attempt later. +// - 5xx, or no recorded status, says nothing about this configuration. Retry. +func permanentAPIRejection(err error) (*controld.ErrorResponse, bool) { + var uer *controld.ErrorResponse + if !errors.As(err, &uer) { + return nil, false + } + switch uer.StatusCode { + case http.StatusRequestTimeout, http.StatusTooManyRequests: + return nil, false + } + if uer.StatusCode < 400 || uer.StatusCode >= 500 { + return nil, false + } + return uer, true +} + +// processCDFlagsFn is the API fetch, indirected so the lifetime binding around it can be +// tested without reaching the network. +var processCDFlagsFn = processCDFlags + +// fetchCDConfigBoundedBy runs the API fetch bounded by stopCh, so a fetch that cannot +// reach the API stops when the service is asked to stop instead of working on behalf of a +// service the OS already considers stopped. The derived context is always cancelled, which +// releases the goroutine watching stopCh. +func fetchCDConfigBoundedBy(stopCh <-chan struct{}, cfg *ctrld.Config) (*controld.ResolverConfig, error) { + ctx, cancel := contextFromStopCh(stopCh) + defer cancel() + return processCDFlagsFn(ctx, cfg) +} + +// fetchCDConfigBoundedByLifetime is the reload path's fetch. Reload binds the same stop +// primitives as startup - it used to wire them up itself, where a dropped cancel or the +// wrong channel would have failed nothing. +func (p *prog) fetchCDConfigBoundedByLifetime(cfg *ctrld.Config) (*controld.ResolverConfig, error) { + return fetchCDConfigBoundedBy(p.stopCh, cfg) +} + +// stopRequested reports whether stopCh has been closed. A nil channel - mobile passes +// none - blocks forever, so the default case is taken and it reads as "no stop". +func stopRequested(stopCh <-chan struct{}) bool { + select { + case <-stopCh: + return true + default: + return false + } +} + +// contextFromStopCh returns a context that is cancelled when stopCh closes, so +// long-running startup work stops as soon as the service is asked to stop. The +// returned cancel func must be called to release the watcher goroutine. +func contextFromStopCh(stopCh <-chan struct{}) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + if stopCh == nil { + return ctx, cancel + } + go func() { + select { + case <-stopCh: + cancel() + case <-ctx.Done(): + } + }() + return ctx, cancel +} + +// processCDFlags fetches the ControlD configuration for cdUID and applies it to cfg. +// +// ctx bounds the bootstrap-DNS retry loop below. That loop retries indefinitely by +// design (a device with no network yet must eventually come up), so it must be +// cancellable: otherwise a stop request during preflight is ignored and the process +// keeps retrying after the service reports itself stopped. +func processCDFlags(ctx context.Context, cfg *ctrld.Config) (*controld.ResolverConfig, error) { logger := mainLog.Load().With().Str("mode", "cd").Logger() logger.Info().Msgf("fetching Controld D configuration from API: %s", cdUID) bo := backoff.NewBackoff("processCDFlags", logf, 30*time.Second) bo.LogLongerThan = 30 * time.Second - ctx := context.Background() + if ctx == nil { + ctx = context.Background() + } req := &controld.ResolverConfigRequest{ RawUID: cdUID, Version: rootCmd.Version, - Metadata: ctrld.SystemMetadataRuntime(context.Background()), + Metadata: ctrld.SystemMetadataRuntime(ctx), } - resolverConfig, err := controld.FetchResolverConfig(req, cdDev) + resolverConfig, err := fetchResolverConfig(ctx, req, cdDev) for { + if ctxErr := ctx.Err(); ctxErr != nil { + logger.Debug().Msg("resolver config fetch cancelled") + return nil, ctxErr + } if errUrlNetworkError(err) { bo.BackOff(ctx, err) + if ctxErr := ctx.Err(); ctxErr != nil { + logger.Debug().Msg("resolver config fetch cancelled during backoff") + return nil, ctxErr + } logger.Warn().Msg("could not fetch resolver using bootstrap DNS, retrying...") - resolverConfig, err = controld.FetchResolverConfig(req, cdDev) + resolverConfig, err = fetchResolverConfig(ctx, req, cdDev) continue } break @@ -1644,7 +1801,7 @@ func cdUIDFromProvToken() string { Metadata: ctrld.SystemMetadata(context.Background()), } // Process provision token if provided. - resolverConfig, err := controld.FetchResolverUID(req, rootCmd.Version, cdDev) + resolverConfig, err := controld.FetchResolverUID(context.Background(), req, rootCmd.Version, cdDev) if err != nil { mainLog.Load().Fatal().Err(err).Msgf("failed to fetch resolver uid with provision token: %s", redactToken(cdOrg)) } @@ -1998,7 +2155,7 @@ func doValidateCdRemoteConfig(cdUID string, fatal bool) error { Version: rootCmd.Version, Metadata: ctrld.SystemMetadataRuntime(context.Background()), } - rc, err := controld.FetchResolverConfig(req, cdDev) + rc, err := controld.FetchResolverConfig(context.Background(), req, cdDev) if err != nil { logger := mainLog.Load().Fatal() if !fatal { diff --git a/cmd/cli/cli_preflight_test.go b/cmd/cli/cli_preflight_test.go new file mode 100644 index 0000000..2f1847b --- /dev/null +++ b/cmd/cli/cli_preflight_test.go @@ -0,0 +1,403 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +func TestContextFromStopCh(t *testing.T) { + t.Run("cancels when stopCh closes", func(t *testing.T) { + stopCh := make(chan struct{}) + ctx, cancel := contextFromStopCh(stopCh) + defer cancel() + + if ctx.Err() != nil { + t.Fatalf("context cancelled before the stop request: %v", ctx.Err()) + } + close(stopCh) + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("context was not cancelled after stopCh closed") + } + if !errors.Is(ctx.Err(), context.Canceled) { + t.Errorf("ctx.Err() = %v, want %v", ctx.Err(), context.Canceled) + } + }) + + t.Run("cancel releases the watcher", func(t *testing.T) { + // stopCh is never closed: cancel() must still end the goroutine watching it. + ctx, cancel := contextFromStopCh(make(chan struct{})) + cancel() + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("context was not cancelled by cancel()") + } + }) + + t.Run("nil stopCh is usable", func(t *testing.T) { + // Mobile callers have no stop channel; preflight must still run. + ctx, cancel := contextFromStopCh(nil) + defer cancel() + if ctx.Err() != nil { + t.Fatalf("context cancelled immediately: %v", ctx.Err()) + } + }) +} + +// retryableNetworkErr is the shape processCDFlags treats as "retry with bootstrap +// DNS": a url.Error wrapping a network failure. +func retryableNetworkErr() error { + return &url.Error{ + Op: "Post", + URL: "https://api.controld.com/utility", + Err: &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}, + } +} + +func TestProcessCDFlagsStopsWhenCancelled(t *testing.T) { + oldFetch := fetchResolverConfig + oldUID := cdUID + t.Cleanup(func() { + fetchResolverConfig = oldFetch + cdUID = oldUID + }) + cdUID = "testuid" + + var calls atomic.Int64 + fetchResolverConfig = func(ctx context.Context, req *controld.ResolverConfigRequest, dev bool) (*controld.ResolverConfig, error) { + calls.Add(1) + return nil, retryableNetworkErr() + } + + // A stop request arriving while the API is unreachable. Before this was + // cancellable, the retry loop kept running after the service reported itself + // stopped, which is what kept the incident's process alive and enforcing. + stopCh := make(chan struct{}) + ctx, cancel := contextFromStopCh(stopCh) + defer cancel() + + done := make(chan error, 1) + go func() { + cfg := ctrld.Config{} + _, err := processCDFlags(ctx, &cfg) + done <- err + }() + + // Let it fail at least once and settle into backoff before stopping. + deadline := time.After(10 * time.Second) + for calls.Load() == 0 { + select { + case <-deadline: + t.Fatal("resolver config was never fetched") + case err := <-done: + t.Fatalf("processCDFlags returned before any fetch: %v", err) + default: + time.Sleep(5 * time.Millisecond) + } + } + close(stopCh) + + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Errorf("processCDFlags err = %v, want it to report %v", err, context.Canceled) + } + case <-time.After(30 * time.Second): + t.Fatal("processCDFlags did not return after the stop request") + } +} + +func TestProcessCDFlagsReturnsImmediatelyWhenAlreadyCancelled(t *testing.T) { + oldFetch := fetchResolverConfig + oldUID := cdUID + t.Cleanup(func() { + fetchResolverConfig = oldFetch + cdUID = oldUID + }) + cdUID = "testuid" + + var calls atomic.Int64 + fetchResolverConfig = func(ctx context.Context, req *controld.ResolverConfigRequest, dev bool) (*controld.ResolverConfig, error) { + calls.Add(1) + return nil, retryableNetworkErr() + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + cfg := ctrld.Config{} + _, err := processCDFlags(ctx, &cfg) + if !errors.Is(err, context.Canceled) { + t.Errorf("processCDFlags err = %v, want %v", err, context.Canceled) + } + // One attempt is made before the loop notices; it must not retry past that. + if got := calls.Load(); got > 1 { + t.Errorf("fetched %d times with a cancelled context, want at most 1", got) + } +} + +// TestRunAPIPreflightClassification is the regression guard for classifying a preflight +// failure as an operator stop. +// +// runAPIPreflight cancels the context it derived from stopCh. Sampling the stop state +// from that context afterwards reports "stopped" unconditionally, because +// context.CancelFunc sets ctx.Err() whether or not anyone asked to stop. run() then +// takes the stop branch for every failure, which skips self-uninstalling a deleted +// device, skips the mobile exit callback, and tells the service manager a failed start +// was a clean exit. +func TestRunAPIPreflightClassification(t *testing.T) { + oldFetch := fetchResolverConfig + oldUID := cdUID + t.Cleanup(func() { + fetchResolverConfig = oldFetch + cdUID = oldUID + }) + cdUID = "testuid" + + // A deleted ControlD device: non-retryable, so preflight returns promptly. + deletedDevice := func() error { + e := &controld.ErrorResponse{} + e.ErrorField.Code = controld.InvalidConfigCode + e.ErrorField.Message = "device does not exist" + return e + } + + openCh := make(chan struct{}) + closedCh := make(chan struct{}) + close(closedCh) + + tests := []struct { + name string + stopCh <-chan struct{} + fetchErr func() error + wantStop bool + }{ + { + // The P1: no stop was requested, so this must reach the failure branch. + name: "api error with no stop request", + stopCh: openCh, + fetchErr: deletedDevice, + }, + { + // Mobile passes no stop channel at all, so it could never have stopped. + name: "api error with a nil stop channel", + stopCh: nil, + fetchErr: deletedDevice, + }, + { + name: "stop requested during preflight", + stopCh: closedCh, + fetchErr: func() error { return retryableNetworkErr() }, + wantStop: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fetchResolverConfig = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + return nil, tc.fetchErr() + } + cfg := ctrld.Config{} + pf := runAPIPreflight(tc.stopCh, &cfg) + + if pf.err == nil { + t.Fatal("expected preflight to fail") + } + if pf.stopRequested != tc.wantStop { + t.Errorf("stopRequested = %v, want %v", pf.stopRequested, tc.wantStop) + } + }) + } +} + +// TestRunAPIPreflightPreservesAPIError verifies the error reaches the caller in a form +// the failure branch can still act on: self-uninstall keys off an *ErrorResponse with +// InvalidConfigCode, and it only runs if that error is both classified as a failure and +// still unwrappable. +func TestRunAPIPreflightPreservesAPIError(t *testing.T) { + oldFetch := fetchResolverConfig + oldUID := cdUID + t.Cleanup(func() { + fetchResolverConfig = oldFetch + cdUID = oldUID + }) + cdUID = "testuid" + + want := &controld.ErrorResponse{} + want.ErrorField.Code = controld.InvalidConfigCode + fetchResolverConfig = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + return nil, want + } + + cfg := ctrld.Config{} + pf := runAPIPreflight(make(chan struct{}), &cfg) + + if pf.stopRequested { + t.Error("a device-deleted failure must not be reported as an operator stop") + } + var got *controld.ErrorResponse + if !errors.As(pf.err, &got) { + t.Fatalf("error no longer unwraps to *controld.ErrorResponse: %v", pf.err) + } + if got.ErrorField.Code != controld.InvalidConfigCode { + t.Errorf("code = %d, want %d (self-uninstall would not trigger)", got.ErrorField.Code, controld.InvalidConfigCode) + } +} + +// TestPermanentAPIRejectionNarrowsToClientErrors is the regression guard for the clean +// exit added above. +// +// controld builds an *ErrorResponse for any non-200 whose body decodes, so the Go type +// says nothing about whether the API's answer will change on a retry. Keying the clean +// exit off the type alone meant a 502 from a load balancer, or an API having a bad ten +// minutes, stopped ctrld on every affected host with no service-manager retry behind it - +// worse than the abnormal exit it replaced, because a Fatal at least gets restarted. +// +// Only a client-error status may take that path. +func TestPermanentAPIRejectionNarrowsToClientErrors(t *testing.T) { + rejection := func(status, code int) error { + e := &controld.ErrorResponse{StatusCode: status} + e.ErrorField.Code = code + e.ErrorField.Message = "api said no" + return e + } + + tests := []struct { + name string + err error + wantPermanent bool + }{ + { + // The case the clean exit exists for: the device is gone, and every restart + // will be told the same thing. + name: "deleted device", + err: rejection(http.StatusNotFound, controld.InvalidConfigCode), + wantPermanent: true, + }, + {"revoked credentials", rejection(http.StatusUnauthorized, 0), true}, + {"forbidden", rejection(http.StatusForbidden, 0), true}, + {"malformed request", rejection(http.StatusBadRequest, 0), true}, + + // Server-side trouble. These must keep the abnormal exit so the service + // manager's recovery policy retries. + {"bad gateway", rejection(http.StatusBadGateway, 0), false}, + {"internal error", rejection(http.StatusInternalServerError, 0), false}, + {"service unavailable", rejection(http.StatusServiceUnavailable, 0), false}, + + // 4xx, but both are the API asking for a later attempt rather than refusing + // this configuration. + {"request timeout", rejection(http.StatusRequestTimeout, 0), false}, + {"rate limited", rejection(http.StatusTooManyRequests, 0), false}, + + // An *ErrorResponse built without a recorded status carries no verdict. A + // hand-constructed one, or a decode path that forgets to record the status, + // must not silently gain the clean exit. + {"no recorded status", rejection(0, controld.InvalidConfigCode), false}, + + // Not an API answer at all: the incident's denied socket reaches Fatal. + {"network failure", retryableNetworkErr(), false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, ok := permanentAPIRejection(tc.err) + if ok != tc.wantPermanent { + t.Errorf("permanentAPIRejection() = %v, want %v", ok, tc.wantPermanent) + } + if ok && got == nil { + t.Error("a permanent rejection must return the rejection for reporting") + } + }) + } + + // The wrapped form matters too: preflight composes the fetch error, and errors.As has + // to reach through that for either branch to be chosen correctly. + wrapped := fmt.Errorf("processCDFlags: %w", rejection(http.StatusNotFound, controld.InvalidConfigCode)) + if _, ok := permanentAPIRejection(wrapped); !ok { + t.Error("a wrapped API rejection must still be recognised") + } + wrappedTransient := fmt.Errorf("processCDFlags: %w", rejection(http.StatusBadGateway, 0)) + if _, ok := permanentAPIRejection(wrappedTransient); ok { + t.Error("a wrapped 502 must not be treated as a permanent rejection") + } +} + +func TestStopRequested(t *testing.T) { + closedCh := make(chan struct{}) + close(closedCh) + + if stopRequested(nil) { + t.Error("a nil stop channel must read as no stop (mobile passes none)") + } + if stopRequested(make(chan struct{})) { + t.Error("an open stop channel must read as no stop") + } + if !stopRequested(closedCh) { + t.Error("a closed stop channel must read as a stop") + } +} + +// TestReloadFetchIsBoundedByServiceLifetime covers the reload path's stop wiring. +// +// Reload fetches the ControlD config too, and it used to build the bounded context +// itself. Nothing tested that: the wrong channel, or a dropped cancel, would have left a +// reload retrying against an unreachable API after "service stopped" was logged, and no +// test would have failed. Both paths now go through one bounded fetch, so this pins it. +func TestReloadFetchIsBoundedByServiceLifetime(t *testing.T) { + original := processCDFlagsFn + t.Cleanup(func() { processCDFlagsFn = original }) + + t.Run("a stop request cancels the reload fetch", func(t *testing.T) { + stopCh := make(chan struct{}) + close(stopCh) + + var sawCancelled bool + processCDFlagsFn = func(ctx context.Context, _ *ctrld.Config) (*controld.ResolverConfig, error) { + select { + case <-ctx.Done(): + sawCancelled = true + case <-time.After(2 * time.Second): + } + return nil, ctx.Err() + } + + p := &prog{stopCh: stopCh} + if _, err := p.fetchCDConfigBoundedByLifetime(&ctrld.Config{}); !errors.Is(err, context.Canceled) { + t.Errorf("reload fetch err = %v, want %v", err, context.Canceled) + } + if !sawCancelled { + t.Error("the reload fetch did not observe the stop request: it is not bound to the service lifetime") + } + }) + + t.Run("the derived context is always released", func(t *testing.T) { + // stopCh stays open: the fetch's own cancel is what must end the watcher, or + // every reload leaks a goroutine. + var captured context.Context + processCDFlagsFn = func(ctx context.Context, _ *ctrld.Config) (*controld.ResolverConfig, error) { + captured = ctx + return nil, nil + } + + p := &prog{stopCh: make(chan struct{})} + if _, err := p.fetchCDConfigBoundedByLifetime(&ctrld.Config{}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + select { + case <-captured.Done(): + case <-time.After(time.Second): + t.Error("the reload fetch left its context uncancelled") + } + }) +} diff --git a/cmd/cli/control_server.go b/cmd/cli/control_server.go index 976569b..5606521 100644 --- a/cmd/cli/control_server.go +++ b/cmd/cli/control_server.go @@ -237,7 +237,7 @@ func (p *prog) registerControlServerHandler() { Version: rootCmd.Version, Metadata: ctrld.SystemMetadataRuntime(context.Background()), } - if rc, err := controld.FetchResolverConfig(rcReq, cdDev); rc != nil { + if rc, err := controld.FetchResolverConfig(context.Background(), rcReq, cdDev); rc != nil { if rc.DeactivationPin != nil { cdDeactivationPin.Store(*rc.DeactivationPin) } else { @@ -351,7 +351,7 @@ func (p *prog) registerControlServerHandler() { } mainLog.Load().Debug().Msg("sending log file to ControlD server") resp := logSentResponse{Size: r.size} - if err := controld.SendLogs(req, cdDev); err != nil { + if err := controld.SendLogs(context.Background(), req, cdDev); err != nil { mainLog.Load().Error().Msgf("could not send log file to ControlD server: %v", err) resp.Error = err.Error() w.WriteHeader(http.StatusInternalServerError) diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index b34013c..b7d2038 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -1190,7 +1190,7 @@ func (p *prog) doSelfUninstall(answer *dns.Msg) { Version: rootCmd.Version, Metadata: ctrld.SystemMetadataRuntime(context.Background()), } - _, err := controld.FetchResolverConfig(req, cdDev) + _, err := controld.FetchResolverConfig(context.Background(), req, cdDev) logger.Debug().Msg("maximum number of refused queries reached, checking device status") selfUninstallCheck(err, p, logger) diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 6f13349..acb12e1 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -324,7 +324,7 @@ func (p *prog) runWait() { continue } if cdUID != "" { - rc, err := processCDFlags(newCfg) + rc, err := p.fetchCDConfigBoundedByLifetime(newCfg) if err != nil { logger.Err(err).Msg("could not fetch ControlD config") waitOldRunDone() @@ -491,7 +491,7 @@ func (p *prog) apiConfigReload() { Version: rootCmd.Version, Metadata: ctrld.SystemMetadataRuntime(context.Background()), } - resolverConfig, err := controld.FetchResolverConfig(req, cdDev) + resolverConfig, err := controld.FetchResolverConfig(context.Background(), req, cdDev) selfUninstallCheck(err, p, logger) if err != nil { logger.Warn().Err(err).Msg("could not fetch resolver config") @@ -549,7 +549,7 @@ func (p *prog) apiConfigReload() { } if cfgErr != nil { logger.Warn().Err(err).Msg("skipping invalid custom config") - if _, err := controld.UpdateCustomLastFailed(cdUID, rootCmd.Version, cdDev, true); err != nil { + if _, err := controld.UpdateCustomLastFailed(context.Background(), cdUID, rootCmd.Version, cdDev, true); err != nil { logger.Error().Err(err).Msg("could not mark custom last update failed") } return diff --git a/internal/controld/config.go b/internal/controld/config.go index 181358c..05edc68 100644 --- a/internal/controld/config.go +++ b/internal/controld/config.go @@ -63,12 +63,35 @@ type ErrorResponse struct { Message string `json:"message"` Code int `json:"code"` } `json:"error"` + // StatusCode is the HTTP status the API answered with. It is not part of the JSON + // body: this type is built for *any* non-200 whose body decodes, so the body alone + // cannot tell a permanent rejection of the request from a transient server-side + // failure, and callers that act differently on the two need the status to tell them + // apart. Zero means the status was not recorded. + StatusCode int `json:"-"` } func (u ErrorResponse) Error() string { return u.ErrorField.Message } +// apiErrorFromResponse builds the error for a non-200 API answer, recording the HTTP +// status alongside the decoded body. +// +// The status is what tells a caller whether the answer will change on a retry: this type +// is built for every non-200 whose body decodes, so a 502 from a load balancer and a 404 +// for a deleted device are otherwise indistinguishable. Both response paths go through +// here so neither can decode a body and forget to record it. +func apiErrorFromResponse(statusCode int, d *json.Decoder) (*ErrorResponse, error) { + errResp := &ErrorResponse{StatusCode: statusCode} + if err := d.Decode(errResp); err != nil { + return nil, err + } + // Decode fills exported fields from the body; StatusCode is json:"-", so it survives. + errResp.StatusCode = statusCode + return errResp, nil +} + type utilityRequest struct { UID string `json:"uid"` ClientID string `json:"client_id,omitempty"` @@ -96,7 +119,7 @@ type LogsRequest struct { } // FetchResolverConfig fetch Control D config for given uid. -func FetchResolverConfig(req *ResolverConfigRequest, cdDev bool) (*ResolverConfig, error) { +func FetchResolverConfig(ctx context.Context, req *ResolverConfigRequest, cdDev bool) (*ResolverConfig, error) { uid, clientID := ParseRawUID(req.RawUID) uReq := utilityRequest{ UID: uid, @@ -106,11 +129,11 @@ func FetchResolverConfig(req *ResolverConfigRequest, cdDev bool) (*ResolverConfi uReq.ClientID = clientID } body, _ := json.Marshal(uReq) - return postUtilityAPI(req.Version, cdDev, false, bytes.NewReader(body)) + return postUtilityAPI(ctx, req.Version, cdDev, false, bytes.NewReader(body)) } // FetchResolverUID fetch resolver uid from a given request. -func FetchResolverUID(req *UtilityOrgRequest, version string, cdDev bool) (*ResolverConfig, error) { +func FetchResolverUID(ctx context.Context, req *UtilityOrgRequest, version string, cdDev bool) (*ResolverConfig, error) { if req == nil { return nil, errors.New("invalid request") } @@ -131,26 +154,29 @@ func FetchResolverUID(req *UtilityOrgRequest, version string, cdDev bool) (*Reso ctrld.ProxyLogger.Load().Debug().Msgf("Sending UID request to ControlD API") body, _ := json.Marshal(req) - return postUtilityAPI(version, cdDev, false, bytes.NewReader(body)) + return postUtilityAPI(ctx, version, cdDev, false, bytes.NewReader(body)) } // UpdateCustomLastFailed calls API to mark custom config is bad. -func UpdateCustomLastFailed(rawUID, version string, cdDev, lastUpdatedFailed bool) (*ResolverConfig, error) { +func UpdateCustomLastFailed(ctx context.Context, rawUID, version string, cdDev, lastUpdatedFailed bool) (*ResolverConfig, error) { uid, clientID := ParseRawUID(rawUID) req := utilityRequest{UID: uid} if clientID != "" { req.ClientID = clientID } body, _ := json.Marshal(req) - return postUtilityAPI(version, cdDev, true, bytes.NewReader(body)) + return postUtilityAPI(ctx, version, cdDev, true, bytes.NewReader(body)) } -func postUtilityAPI(version string, cdDev, lastUpdatedFailed bool, body io.Reader) (*ResolverConfig, error) { +func postUtilityAPI(ctx context.Context, version string, cdDev, lastUpdatedFailed bool, body io.Reader) (*ResolverConfig, error) { apiUrl := resolverDataURLCom if cdDev { apiUrl = resolverDataURLDev } - req, err := http.NewRequest("POST", apiUrl, body) + // Context-bound so an in-flight request is abandoned when the caller is + // cancelled - a service stop during API preflight must not wait out the + // request timeout, let alone keep retrying. + req, err := http.NewRequestWithContext(ctx, "POST", apiUrl, body) if err != nil { return nil, fmt.Errorf("http.NewRequest: %w", err) } @@ -174,8 +200,8 @@ func postUtilityAPI(version string, cdDev, lastUpdatedFailed bool, body io.Reade defer resp.Body.Close() d := json.NewDecoder(resp.Body) if resp.StatusCode != http.StatusOK { - errResp := &ErrorResponse{} - if err := d.Decode(errResp); err != nil { + errResp, err := apiErrorFromResponse(resp.StatusCode, d) + if err != nil { return nil, err } return nil, errResp @@ -189,13 +215,13 @@ func postUtilityAPI(version string, cdDev, lastUpdatedFailed bool, body io.Reade } // SendLogs sends runtime log to ControlD API. -func SendLogs(lr *LogsRequest, cdDev bool) error { +func SendLogs(ctx context.Context, lr *LogsRequest, cdDev bool) error { defer lr.Data.Close() apiUrl := logURLCom if cdDev { apiUrl = logURLDev } - req, err := http.NewRequest("POST", apiUrl, lr.Data) + req, err := http.NewRequestWithContext(ctx, "POST", apiUrl, lr.Data) if err != nil { return fmt.Errorf("http.NewRequest: %w", err) } @@ -215,8 +241,8 @@ func SendLogs(lr *LogsRequest, cdDev bool) error { defer resp.Body.Close() d := json.NewDecoder(resp.Body) if resp.StatusCode != http.StatusOK { - errResp := &ErrorResponse{} - if err := d.Decode(errResp); err != nil { + errResp, err := apiErrorFromResponse(resp.StatusCode, d) + if err != nil { return err } return errResp diff --git a/internal/controld/config_test.go b/internal/controld/config_test.go index b266142..5973de0 100644 --- a/internal/controld/config_test.go +++ b/internal/controld/config_test.go @@ -1,6 +1,9 @@ package controld import ( + "encoding/json" + "net/http" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -29,3 +32,67 @@ func Test_parseUID(t *testing.T) { }) } } + +// TestAPIErrorRecordsHTTPStatus pins the plumbing the caller's exit decision rests on. +// +// cmd/cli treats a 4xx as "this configuration is refused, restarting cannot help" and +// exits cleanly, while a 5xx keeps the abnormal exit so the service manager retries. Both +// readings need the status, and it is not in the JSON body - so a decode path that +// forgets to record it would quietly send every API error down the retry branch, +// including a deleted device that should self-uninstall and stop. +func TestAPIErrorRecordsHTTPStatus(t *testing.T) { + tests := []struct { + name string + statusCode int + body string + wantCode int + wantMsg string + }{ + { + name: "deleted device", + statusCode: http.StatusNotFound, + body: `{"error":{"message":"device does not exist","code":40402}}`, + wantCode: InvalidConfigCode, + wantMsg: "device does not exist", + }, + { + // A gateway error body carries no error object at all, which decodes + // cleanly into the zero value - so the status is the only thing that + // distinguishes it from a real rejection. + name: "gateway error with an empty body", + statusCode: http.StatusBadGateway, + body: `{}`, + }, + { + name: "service unavailable", + statusCode: http.StatusServiceUnavailable, + body: `{"error":{"message":"try again later","code":0}}`, + wantMsg: "try again later", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + d := json.NewDecoder(strings.NewReader(tc.body)) + errResp, err := apiErrorFromResponse(tc.statusCode, d) + if err != nil { + t.Fatalf("unexpected decode error: %v", err) + } + if errResp.StatusCode != tc.statusCode { + t.Errorf("StatusCode = %d, want %d: the caller cannot tell a permanent rejection from a transient failure without it", errResp.StatusCode, tc.statusCode) + } + if errResp.ErrorField.Code != tc.wantCode { + t.Errorf("code = %d, want %d", errResp.ErrorField.Code, tc.wantCode) + } + if errResp.Error() != tc.wantMsg { + t.Errorf("message = %q, want %q", errResp.Error(), tc.wantMsg) + } + }) + } + + t.Run("an undecodable body is reported as a decode failure", func(t *testing.T) { + d := json.NewDecoder(strings.NewReader("502 Bad Gateway")) + if _, err := apiErrorFromResponse(http.StatusBadGateway, d); err == nil { + t.Error("expected a decode error for a non-JSON body") + } + }) +} From 2400f2796254a3dfa8ba6fa73466d13f0536aa2b Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 14 Aug 2026 15:18:37 +0700 Subject: [PATCH 08/16] all: keep the first-attempt error when the direct-ip fallback also fails Both API requests and binary downloads retry against a hard-coded IP when the attempt via hostname fails. Both then overwrote the first error with the fallback's, so only the last failure was reported. That discarded the diagnosis. When the hostname attempt is denied locally - WSAEACCES on Windows, "An attempt was made to access a socket in a way forbidden by its access permissions", which means the host is blocking ctrld - and the direct-ip fallback fails with an unreachable IPv6 route, what surfaces to the operator is "dial tcp6: no route to host": a routing problem that does not exist, while the error naming the real cause is visible only in debug logs. Report both failures instead, keeping the error chain intact so errors.Is still matches either one. Also switch the final wrap in doWithRetry from %v to %w, which had been flattening the chain even when a single error was reported. This also changes retry classification, which is worth stating explicitly because it is not obvious from "report both errors". processCDFlags decides whether to keep backing off with errUrlNetworkError, which uses errors.As - and errors.As returns the *first* match in the tree. Wrapping the hostname attempt first therefore hands the predicate that attempt's failure, where previously only the fallback's error survived to be classified. The effect is intended. A locally denied socket (WSAEACCES) is not a transient network error, so preflight now fails fast and reports instead of retrying against a firewall that is not going to clear on its own. The case that justifies retrying forever, a network unreachable on both attempts at boot, is unchanged. Both classifications are pinned by tests, along with the wrap order they depend on at each composition site, so reversing it fails loudly rather than silently restoring the old behaviour. --- cmd/cli/library.go | 23 ++- cmd/cli/library_retry_test.go | 241 +++++++++++++++++++++++++++++ cmd/cli/prog.go | 48 +++++- internal/controld/config.go | 34 +++- internal/controld/fallback_test.go | 155 +++++++++++++++++++ 5 files changed, 485 insertions(+), 16 deletions(-) create mode 100644 cmd/cli/library_retry_test.go create mode 100644 internal/controld/fallback_test.go diff --git a/cmd/cli/library.go b/cmd/cli/library.go index 7847dd7..148ba81 100644 --- a/cmd/cli/library.go +++ b/cmd/cli/library.go @@ -50,8 +50,13 @@ func httpClientWithFallback(timeout time.Duration) *http.Client { // doWithRetry performs an HTTP request with retries func doWithRetry(req *http.Request, maxRetries int, ip string) (*http.Response, error) { + return doWithRetryClient(httpClientWithFallback(defaultHTTPTimeout), req, maxRetries, ip) +} + +// doWithRetryClient is doWithRetry with an injectable client, so the retry and +// error-composition behaviour can be tested without real network access. +func doWithRetryClient(client *http.Client, req *http.Request, maxRetries int, ip string) (*http.Response, error) { var lastErr error - client := httpClientWithFallback(defaultHTTPTimeout) var ipReq *http.Request if ip != "" { ipReq = req.Clone(req.Context()) @@ -67,22 +72,28 @@ func doWithRetry(req *http.Request, maxRetries int, ip string) (*http.Response, if err == nil { return resp, nil } + // Keep the hostname attempt's error: it carries the diagnosis (on Windows, + // a local firewall denying the socket shows up here as WSAEACCES), while the + // direct-IP fallback often fails for an unrelated reason such as an + // unreachable IPv6 route. + attemptErr := err if ipReq != nil { mainLog.Load().Warn().Err(err).Msgf("dial to %q failed", req.Host) mainLog.Load().Warn().Msgf("fallback to direct IP to download prod version: %q", ip) - resp, err = client.Do(ipReq) - if err == nil { + resp, fallbackErr := client.Do(ipReq) + if fallbackErr == nil { return resp, nil } + attemptErr = fmt.Errorf("%w; fallback to direct ip %s failed: %w", attemptErr, ip, fallbackErr) } - lastErr = err - mainLog.Load().Debug().Err(err). + lastErr = attemptErr + mainLog.Load().Debug().Err(attemptErr). Str("method", req.Method). Str("url", req.URL.String()). Msgf("HTTP request attempt %d/%d failed", attempt+1, maxRetries) } - return nil, fmt.Errorf("failed after %d attempts to %s %s: %v", maxRetries, req.Method, req.URL, lastErr) + return nil, fmt.Errorf("failed after %d attempts to %s %s: %w", maxRetries, req.Method, req.URL, lastErr) } // Helper for making GET requests with retries diff --git a/cmd/cli/library_retry_test.go b/cmd/cli/library_retry_test.go new file mode 100644 index 0000000..b28f2b0 --- /dev/null +++ b/cmd/cli/library_retry_test.go @@ -0,0 +1,241 @@ +package cli + +import ( + "errors" + "fmt" + "net" + "net/http" + "net/url" + "syscall" + "testing" + + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// wsaEACCES is WSAEACCES (10013): "An attempt was made to access a socket in a way +// forbidden by its access permissions." This is what Windows reports when a WFP +// filter denies the connect. Used as a plain errno so the test runs everywhere. +const wsaEACCES = syscall.Errno(10013) + +// denyingRoundTripper denies the hostname attempt with firstErr and the direct-ip +// attempt with fbErr, the shape seen during the Firewall Mode incident: the +// hostname attempt was denied by ctrld's own stale block-all filters, while the +// direct-ip fallback failed on an unreachable IPv6 route. +type denyingRoundTripper struct { + hostname string + firstErr error + fbErr error +} + +func (rt *denyingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL.Host == rt.hostname { + return nil, &net.OpError{Op: "dial", Net: "tcp4", Err: rt.firstErr} + } + return nil, &net.OpError{Op: "dial", Net: "tcp6", Err: rt.fbErr} +} + +func TestDoWithRetryPreservesHostnameError(t *testing.T) { + const hostname = "dl.controld.dev" + req, err := http.NewRequest(http.MethodGet, "https://"+hostname+"/v2/windows-amd64/ctrld.exe", nil) + if err != nil { + t.Fatal(err) + } + rt := &denyingRoundTripper{ + hostname: hostname, + firstErr: wsaEACCES, + fbErr: syscall.EHOSTUNREACH, + } + + _, err = doWithRetryClient(&http.Client{Transport: rt}, req, 1, "23.171.240.151") + if err == nil { + t.Fatal("expected doWithRetry to fail when both attempts are denied") + } + if !errors.Is(err, wsaEACCES) { + t.Errorf("hostname-attempt error (WSAEACCES) was lost, got: %v", err) + } + if !errors.Is(err, syscall.EHOSTUNREACH) { + t.Errorf("fallback error was lost, got: %v", err) + } +} + +// composedAttemptErrors builds the error shape the two-attempt paths return: each +// attempt's *url.Error (as produced by http.Client.Do) wrapped by a single fmt.Errorf +// with two %w verbs, hostname attempt first. Mirrors doWithFallback in +// internal/controld and doWithRetryClient above. +func composedAttemptErrors(first, fallback error) error { + attempt := func(network string, cause error) error { + return &url.Error{ + Op: "Post", + URL: "https://api.controld.com/utility", + Err: &net.OpError{Op: "dial", Net: network, Err: cause}, + } + } + return fmt.Errorf("request failed: %w; fallback to direct ip %s failed: %w", + attempt("tcp4", first), "147.185.34.1", attempt("tcp6", fallback)) +} + +// TestComposedFallbackErrorRetryClassification pins which attempt decides whether +// preflight keeps retrying. +// +// Reporting both attempt errors is not purely diagnostic: processCDFlags decides +// retryability with errUrlNetworkError, which uses errors.As, and errors.As is +// order-sensitive - it returns the *first* matching error in the tree. Composing the +// hostname attempt first therefore hands the retry predicate the hostname failure, +// where previously only the fallback's error survived to be classified. +// +// The consequence is deliberate: a locally denied socket (WSAEACCES, a firewall +// blocking ctrld) is no longer treated as a transient network error, so preflight fails +// fast and reports instead of backing off - the incident logged 256 retry cycles +// against filters that were never going to clear on their own. The boot case that +// justifies the indefinite retry, a network unreachable on both attempts, is preserved. +// +// If the wrap order is ever reversed, this test fails rather than silently restoring +// indefinite retries against a host that is actively refusing. +func TestComposedFallbackErrorRetryClassification(t *testing.T) { + tests := []struct { + name string + hostname error + fallback error + wantRetryable bool + }{ + { + // The incident's pair: denied locally, IPv6 route unusable. + name: "denied socket then unreachable fallback fails fast", + hostname: wsaEACCES, + fallback: syscall.EHOSTUNREACH, + wantRetryable: false, + }, + { + // Boot with no network yet: must still retry indefinitely. + name: "network unreachable on both attempts still retries", + hostname: syscall.ENETUNREACH, + fallback: syscall.ENETUNREACH, + wantRetryable: true, + }, + { + name: "connection refused still retries", + hostname: syscall.ECONNREFUSED, + fallback: syscall.EHOSTUNREACH, + wantRetryable: true, + }, + { + name: "permission denied on both attempts fails fast", + hostname: syscall.EACCES, + fallback: syscall.EACCES, + wantRetryable: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := composedAttemptErrors(tc.hostname, tc.fallback) + if got := errUrlNetworkError(err); got != tc.wantRetryable { + t.Errorf("errUrlNetworkError() = %v, want %v", got, tc.wantRetryable) + } + // Both attempts remain reportable regardless of classification. + if !errors.Is(err, tc.hostname) { + t.Error("hostname attempt error was lost") + } + if !errors.Is(err, tc.fallback) { + t.Error("fallback attempt error was lost") + } + }) + } +} + +// TestUnresolvedHostnameDefersToFallbackAttempt covers the asymmetric pair. +// +// Only the hostname attempt resolves DNS, and Go marks a *net.DNSError as temporary only +// for socket failures that reached the server - so a SERVFAIL or "no such host" answer is +// not temporary. At boot behind a captive portal, or before a router's forwarder is up, +// that is exactly how the hostname attempt fails while the network is merely not ready. +// Before the composed error existed only the fallback decided, so this pair retried; +// classifying the hostname attempt alone would fail it fast and reach Fatal. +// +// A name-resolution failure therefore carries no verdict: the fallback attempt decides. +// The locally-denied case above still fails fast, because a denied socket is definitive. +func TestUnresolvedHostnameDefersToFallbackAttempt(t *testing.T) { + dnsFailure := &url.Error{ + Op: "Post", + URL: "https://api.controld.com/utility", + Err: &net.DNSError{Err: "server misbehaving", Name: "api.controld.com", IsTemporary: false}, + } + attempt := func(cause error) error { + return &url.Error{ + Op: "Post", + URL: "https://api.controld.com/utility", + Err: &net.OpError{Op: "dial", Net: "tcp6", Err: cause}, + } + } + + retryable := fmt.Errorf("request failed: %w; fallback to direct ip %s failed: %w", + dnsFailure, "147.185.34.1", attempt(syscall.ECONNREFUSED)) + if !errUrlNetworkError(retryable) { + t.Error("an unresolved hostname with a retryable fallback must keep retrying: at boot the network is simply not up yet") + } + + denied := fmt.Errorf("request failed: %w; fallback to direct ip %s failed: %w", + dnsFailure, "147.185.34.1", attempt(wsaEACCES)) + if errUrlNetworkError(denied) { + t.Error("an unresolved hostname with a denied fallback must fail fast: nothing here clears on its own") + } + + // A resolution failure alone still says nothing, so it must not be read as retryable. + if errUrlNetworkError(dnsFailure) { + t.Error("a bare name-resolution failure must not be classified as retryable") + } +} + +// TestDoWithFallbackClassificationEndToEnd drives the real composition in +// internal/controld through the real predicate, instead of asserting a hand-written copy +// of its error shape against another hand-written copy. A change to either side's format +// string or wrap order is caught here. +func TestDoWithFallbackClassificationEndToEnd(t *testing.T) { + const hostname = "api.controld.com" + req, err := http.NewRequest(http.MethodPost, "https://"+hostname+"/utility", nil) + if err != nil { + t.Fatal(err) + } + rt := &denyingRoundTripper{ + hostname: hostname, + firstErr: wsaEACCES, + fbErr: syscall.EHOSTUNREACH, + } + + _, gotErr := controld.DoWithFallbackForTest(&http.Client{Transport: rt}, req, "147.185.34.1") + if gotErr == nil { + t.Fatal("expected both attempts to fail") + } + if errUrlNetworkError(gotErr) { + t.Errorf("the real composed error was classified as retryable: %v", gotErr) + } + if !errors.Is(gotErr, wsaEACCES) || !errors.Is(gotErr, syscall.EHOSTUNREACH) { + t.Errorf("the real composed error lost an attempt: %v", gotErr) + } +} + +// TestDoWithRetryComposesHostnameAttemptFirst anchors the ordering assumption above to +// the real composition, so a reordering of the wrap in doWithRetryClient is caught here +// and not only in the hand-built shape. +func TestDoWithRetryComposesHostnameAttemptFirst(t *testing.T) { + const hostname = "dl.controld.dev" + req, err := http.NewRequest(http.MethodGet, "https://"+hostname+"/v2/windows-amd64/ctrld.exe", nil) + if err != nil { + t.Fatal(err) + } + rt := &denyingRoundTripper{hostname: hostname, firstErr: wsaEACCES, fbErr: syscall.EHOSTUNREACH} + + _, gotErr := doWithRetryClient(&http.Client{Transport: rt}, req, 1, "23.171.240.151") + if gotErr == nil { + t.Fatal("expected both attempts to fail") + } + + // errors.As must reach the hostname attempt first: that is what the retry + // predicate classifies. + var opErr *net.OpError + if !errors.As(gotErr, &opErr) { + t.Fatalf("no net.OpError in the chain: %v", gotErr) + } + if !errors.Is(opErr.Err, wsaEACCES) { + t.Errorf("first OpError in the chain is %v, want the hostname attempt (%v)", opErr.Err, wsaEACCES) + } +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index acb12e1..7a0c3b0 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -1540,14 +1540,56 @@ var ( windowsEADDRINUSE = syscall.Errno(10048) ) +// errUrlNetworkError reports whether a failed HTTP attempt is worth retrying. +// +// The two-attempt paths compose one *url.Error per attempt - hostname first, then the +// direct-IP fallback - so this walks them in order rather than classifying only the first +// one errors.As happens to find. Each attempt can say one of three things: +// +// - retryable (unreachable, refused, temporary): retry, whichever attempt said it; +// - a name-resolution failure: no verdict. Only the hostname attempt resolves DNS, and +// at boot behind a captive portal or before the router's forwarder is up it fails +// this way while the network is merely not ready yet. Consult the next attempt; +// - anything else, notably a locally denied socket (WSAEACCES from a firewall blocking +// ctrld): definitive. Stop, because retrying cannot clear it - the Firewall Mode +// incident spent 256 retry cycles against filters that were never going to clear. func errUrlNetworkError(err error) bool { - var urlErr *url.Error - if errors.As(err, &urlErr) { - return errNetworkError(urlErr.Err) + for _, attempt := range attemptErrors(err) { + var urlErr *url.Error + if !errors.As(attempt, &urlErr) { + continue + } + switch { + case errNetworkError(urlErr.Err): + return true + case errDNSResolutionFailure(urlErr.Err): + // Neutral; let a later attempt decide. + default: + return false + } } return false } +// attemptErrors returns the per-attempt errors recorded in err, in the order they were +// tried. A composed fallback error wraps one per attempt; anything else is a single +// attempt. +func attemptErrors(err error) []error { + if multi, ok := err.(interface{ Unwrap() []error }); ok { + return multi.Unwrap() + } + return []error{err} +} + +// errDNSResolutionFailure reports whether err is a name-resolution failure. Go marks a +// *net.DNSError as temporary only for socket failures that reached the server, so a +// SERVFAIL or "no such host" answer is not temporary - but it is also not evidence that +// retrying is pointless, which is why callers treat it as no verdict. +func errDNSResolutionFailure(err error) bool { + var dnsErr *net.DNSError + return errors.As(err, &dnsErr) +} + func errNetworkError(err error) bool { var opErr *net.OpError if errors.As(err, &opErr) { diff --git a/internal/controld/config.go b/internal/controld/config.go index 05edc68..4b7ad07 100644 --- a/internal/controld/config.go +++ b/internal/controld/config.go @@ -332,16 +332,29 @@ func addrsFromPort(ips []string, port string) []string { return addrs } +// doWithFallback sends req, retrying against apiIp directly if the first attempt +// fails (typically because DNS is not usable yet). +// +// Both failures are reported. The first attempt carries the diagnosis - on Windows +// a local firewall denying the socket surfaces there as WSAEACCES ("An attempt was +// made to access a socket in a way forbidden by its access permissions"), which +// says the host is blocking ctrld rather than that the network is down. Returning +// only the fallback error hid that behind a bare "no route to host" from the IPv6 +// attempt. func doWithFallback(client *http.Client, req *http.Request, apiIp string) (*http.Response, error) { resp, err := client.Do(req) - if err != nil { - ctrld.ProxyLogger.Load().Warn().Err(err).Msgf("failed to send request, fallback to direct IP: %s", apiIp) - ipReq := req.Clone(req.Context()) - ipReq.Host = apiIp - ipReq.URL.Host = apiIp - resp, err = client.Do(ipReq) + if err == nil { + return resp, nil } - return resp, err + ctrld.ProxyLogger.Load().Warn().Err(err).Msgf("failed to send request, fallback to direct IP: %s", apiIp) + ipReq := req.Clone(req.Context()) + ipReq.Host = apiIp + ipReq.URL.Host = apiIp + resp, fallbackErr := client.Do(ipReq) + if fallbackErr != nil { + return nil, fmt.Errorf("request failed: %w; fallback to direct ip %s failed: %w", err, apiIp, fallbackErr) + } + return resp, nil } // apiServerIP returns the direct IP to connect to API server. @@ -351,3 +364,10 @@ func apiServerIP(cdDev bool) string { } return apiDomainComIPv4 } + +// DoWithFallbackForTest exposes doWithFallback so tests outside this package can drive +// the real two-attempt composition through the real retry predicate, rather than +// asserting a copy of this error shape against another copy of it. +func DoWithFallbackForTest(client *http.Client, req *http.Request, apiIp string) (*http.Response, error) { + return doWithFallback(client, req, apiIp) +} diff --git a/internal/controld/fallback_test.go b/internal/controld/fallback_test.go new file mode 100644 index 0000000..9b6eb8c --- /dev/null +++ b/internal/controld/fallback_test.go @@ -0,0 +1,155 @@ +package controld + +import ( + "errors" + "net" + "net/http" + "net/http/httptest" + "strings" + "syscall" + "testing" +) + +// errRoundTripper fails the hostname attempt and the direct-ip attempt with +// different errors, mimicking the Firewall Mode incident: the hostname attempt is +// denied by a local firewall (WSAEACCES on Windows) while the direct-ip fallback +// reports an unreachable IPv6 route. +type errRoundTripper struct { + hostname string + firstErr error + fbErr error + fbCalled bool +} + +func (rt *errRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL.Host == rt.hostname { + return nil, &net.OpError{Op: "dial", Net: "tcp4", Err: rt.firstErr} + } + rt.fbCalled = true + if rt.fbErr == nil { + return &http.Response{ + StatusCode: http.StatusOK, + Body: http.NoBody, + Request: req, + }, nil + } + return nil, &net.OpError{Op: "dial", Net: "tcp6", Err: rt.fbErr} +} + +// wsaEACCES is WSAEACCES (10013): "An attempt was made to access a socket in a way +// forbidden by its access permissions." The value is what Windows reports when a +// WFP filter denies the connect; it is used here as a plain errno so the test runs +// on every platform. +const wsaEACCES = syscall.Errno(10013) + +func TestDoWithFallbackPreservesFirstError(t *testing.T) { + const ( + hostname = "api.controld.com" + apiIP = "147.185.34.1" + ) + rt := &errRoundTripper{ + hostname: hostname, + firstErr: wsaEACCES, + fbErr: syscall.EHOSTUNREACH, + } + req, err := http.NewRequest(http.MethodPost, "https://"+hostname+"/utility", nil) + if err != nil { + t.Fatal(err) + } + + resp, err := doWithFallback(&http.Client{Transport: rt}, req, apiIP) + if err == nil { + t.Fatalf("expected an error, got response %v", resp) + } + if !rt.fbCalled { + t.Error("direct-ip fallback was not attempted") + } + + // The actionable failure must survive: an operator reading this error has to be + // able to tell "the host is blocking us" from "the network is down". + if !errors.Is(err, wsaEACCES) { + t.Errorf("first-attempt error (WSAEACCES) was lost, got: %v", err) + } + if !errors.Is(err, syscall.EHOSTUNREACH) { + t.Errorf("fallback error was lost, got: %v", err) + } + if got := err.Error(); !strings.Contains(got, apiIP) { + t.Errorf("error does not mention the fallback ip %q: %v", apiIP, got) + } +} + +func TestDoWithFallbackSucceedsOnFallback(t *testing.T) { + const hostname = "api.controld.com" + rt := &errRoundTripper{hostname: hostname, firstErr: wsaEACCES} + req, err := http.NewRequest(http.MethodPost, "https://"+hostname+"/utility", nil) + if err != nil { + t.Fatal(err) + } + + resp, err := doWithFallback(&http.Client{Transport: rt}, req, "147.185.34.1") + if err != nil { + t.Fatalf("expected the fallback to succeed, got: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("StatusCode = %d, want %d", resp.StatusCode, http.StatusOK) + } +} + +func TestDoWithFallbackNoFallbackOnSuccess(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPost, srv.URL, nil) + if err != nil { + t.Fatal(err) + } + resp, err := doWithFallback(srv.Client(), req, "127.0.0.2") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("StatusCode = %d, want %d", resp.StatusCode, http.StatusOK) + } +} + +// TestDoWithFallbackComposesHostnameAttemptFirst pins the order of the composed error. +// +// The order is not cosmetic. cmd/cli's preflight retry predicate classifies this error +// with errors.As, which returns the first match in the tree, so whichever attempt is +// wrapped first decides whether processCDFlags keeps backing off or fails fast. That +// predicate lives in another package and cannot be called from here, so this test +// guards the property it depends on: the hostname attempt - the one that carries the +// diagnosis - must come first. +func TestDoWithFallbackComposesHostnameAttemptFirst(t *testing.T) { + const hostname = "api.controld.com" + rt := &errRoundTripper{ + hostname: hostname, + firstErr: wsaEACCES, + fbErr: syscall.EHOSTUNREACH, + } + req, err := http.NewRequest(http.MethodPost, "https://"+hostname+"/utility", nil) + if err != nil { + t.Fatal(err) + } + + _, gotErr := doWithFallback(&http.Client{Transport: rt}, req, "147.185.34.1") + if gotErr == nil { + t.Fatal("expected both attempts to fail") + } + + var opErr *net.OpError + if !errors.As(gotErr, &opErr) { + t.Fatalf("no net.OpError in the chain: %v", gotErr) + } + if !errors.Is(opErr.Err, wsaEACCES) { + t.Errorf("first OpError in the chain is %v, want the hostname attempt (%v)", opErr.Err, wsaEACCES) + } + // The tcp4/tcp6 split distinguishes the two attempts in the fake transport. + if opErr.Net != "tcp4" { + t.Errorf("first OpError is from the %s attempt, want tcp4 (hostname)", opErr.Net) + } +} From 5c9d3dec4e842d9508287580448fe08c1bd87e36 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 14 Aug 2026 15:21:23 +0700 Subject: [PATCH 09/16] cmd/cli: stop the replacement before rolling its binary back Rollback ran os.Remove(bin) while the replacement service was still running from that image. Windows locks a running executable, so the remove failed with "Access is denied" - and it was fatal, so the os.Rename that restores the previous binary never ran. The upgrade ended with the broken replacement still installed and the working binary stranded at its _previous name. Readiness failing is not evidence the process exited: the service manager can report a started service whose process never became operational. So rollback now stops the service and waits until the manager reports it stopped before touching the executable, then cleans up DNS the way the restart path's Cleanup task does. Restoring is now conditional on the previous binary reporting a version, since a _previous file that exists but produces no version output would trade a service that starts and hangs for one that cannot start at all. When it is unusable, rollback keeps it for inspection, leaves the installed binary alone, and says so instead of pressing on. The --version probe is bounded by a timeout so a binary that hangs cannot hang the upgrade. Remaining failures are reported rather than fatal, so each one says what state the host was left in. os.Remove is retried while the path stays locked, since Windows releases an image lock asynchronously after the process exits. The helpers live in a new file rather than in commands.go, and the rollback is extracted into rollbackToPreviousBinary() so it can be covered: the stop happens while the executable is still present, an unusable previous binary is kept without swapping or restarting, and a failed stop aborts before anything is modified. Reversing the stop and the remove fails these tests. The version probe is called through a variable so those tests do not have to stage a runnable executable. Staging one is not portable: oldBin is bin+"_previous", so a fixture named "ctrld" yields the extension-less "ctrld_previous", which Windows refuses to execute, and a symlink to the test binary needs a privilege Windows does not grant by default. The probe itself is still covered against the real test binary. Production is unaffected: ctrld.exe _previous does have an extension, and os/exec only appends PATHEXT entries when a path has none at all - noted at binaryVersion so the suffix is not renamed into something extension-less by accident. --- cmd/cli/commands.go | 31 ++-- cmd/cli/main_test.go | 21 +++ cmd/cli/upgrade_rollback.go | 195 +++++++++++++++++++++ cmd/cli/upgrade_rollback_test.go | 288 +++++++++++++++++++++++++++++++ 4 files changed, 521 insertions(+), 14 deletions(-) create mode 100644 cmd/cli/upgrade_rollback.go create mode 100644 cmd/cli/upgrade_rollback_test.go diff --git a/cmd/cli/commands.go b/cmd/cli/commands.go index ec7b47c..b32f6db 100644 --- a/cmd/cli/commands.go +++ b/cmd/cli/commands.go @@ -1501,28 +1501,31 @@ func initUpgradeCmd() *cobra.Command { if doRestart() { _ = os.Remove(oldBin) _ = os.Chmod(bin, 0755) - ver := "unknown version" - out, err := exec.Command(bin, "--version").CombinedOutput() + ver, err := binaryVersion(bin) if err != nil { mainLog.Load().Warn().Err(err).Msg("Failed to get new binary version") - } - if after, found := strings.CutPrefix(string(out), "ctrld version "); found { - ver = after + ver = "unknown version" } mainLog.Load().Notice().Msgf("Upgrade successful - %s", ver) return } - mainLog.Load().Warn().Msgf("Upgrade failed, restoring previous binary: %s", oldBin) - if err := os.Remove(bin); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to remove new binary") + mainLog.Load().Warn().Msg("Upgrade failed: the new binary did not become ready") + stop := func() error { + if !svcInstalled { + return nil + } + if err := stopServiceAndWait(s, upgradeStopTimeout); err != nil { + return err + } + // Mirror the Cleanup task in doRestart: leave DNS settings as the OS + // had them, not as a half-started ctrld left them. + p.router.Cleanup() + p.resetDNS(false, true) + return nil } - if err := os.Rename(oldBin, bin); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to restore old binary") - } - if doRestart() { - mainLog.Load().Notice().Msg("Restored previous binary successfully") - return + if err := rollbackToPreviousBinary(bin, oldBin, stop, doRestart); err != nil { + mainLog.Load().Error().Err(err).Msg("Rollback did not complete") } }, } diff --git a/cmd/cli/main_test.go b/cmd/cli/main_test.go index 3e9cd72..6e2257f 100644 --- a/cmd/cli/main_test.go +++ b/cmd/cli/main_test.go @@ -1,6 +1,7 @@ package cli import ( + "fmt" "os" "os/exec" "strings" @@ -11,7 +12,27 @@ import ( var logOutput strings.Builder +// envFakeVersionOutput makes this test binary impersonate a ctrld executable: when +// set, the process writes the value to stdout and exits without running any test, so +// binaryVersion() can be exercised on every platform without building or shipping a +// fixture binary. The value envFakeVersionSilent produces no output at all, which +// reproduces a ctrld.exe_previous that exists but reports no version. +// +// This must be handled before m.Run(), which is what parses the test flags: the child +// is invoked as " --version" and would otherwise die on an unknown flag. +const ( + envFakeVersionOutput = "CTRLD_TEST_FAKE_VERSION_OUTPUT" + envFakeVersionSilent = "" +) + func TestMain(m *testing.M) { + if out := os.Getenv(envFakeVersionOutput); out != "" { + if out != envFakeVersionSilent { + fmt.Println(out) + } + os.Exit(0) + } + l := zerolog.New(&logOutput) mainLog.Store(&l) diff --git a/cmd/cli/upgrade_rollback.go b/cmd/cli/upgrade_rollback.go new file mode 100644 index 0000000..ecda5cf --- /dev/null +++ b/cmd/cli/upgrade_rollback.go @@ -0,0 +1,195 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" + + "github.com/kardianos/service" +) + +const ( + // upgradeStopTimeout bounds how long rollback waits for the replacement process to + // exit, and for Windows to release the lock on its image afterwards. + upgradeStopTimeout = 30 * time.Second + // upgradeStopPollInterval is how often the service status is re-checked while + // waiting for the process to exit. + upgradeStopPollInterval = 500 * time.Millisecond + // binaryVersionTimeout bounds the "--version" probe, so a binary that hangs on + // startup cannot hang the upgrade. + binaryVersionTimeout = 10 * time.Second +) + +// rollbackToPreviousBinary restores oldBin over bin after the replacement failed to +// become ready, and restarts the service on the restored binary. +// +// stop must leave the replacement's process gone, because every step here modifies +// the executable that process is running from. It is called first for that reason: +// readiness failing does not mean the process exited - the service manager can report +// a started service whose process never became operational. Windows holds an +// exclusive lock on a running executable's image, so the previous code's +// os.Remove(bin) failed there with "Access is denied", and because that was fatal the +// restore never ran: the broken binary stayed installed with the previous one +// stranded at its _previous name. +// +// Stopping first also puts the host back in a known state, since a stopped ctrld +// holds no DNS or intercept enforcement. +func rollbackToPreviousBinary(bin, oldBin string, stop func() error, restart func() bool) error { + if err := stop(); err != nil { + mainLog.Load().Error().Err(err).Msg("Could not confirm the service stopped; not modifying its binary") + return err + } + + // Only restore a previous binary that actually runs: a _previous file that exists + // but reports no version would replace a service that starts and hangs with one + // that cannot start at all. + // + // The probe is retried for the same reason removeBinaryWithRetry is: on Windows a + // single exec can fail transiently while antivirus scans the file or the disk is + // busy, and treating that as "no usable previous binary" leaves the host stopped + // with the broken binary installed - an end state worse than restoring a binary + // that turns out to be bad, which the restart check below catches. + // + // Running "--version" proves the file executes. It is not an authenticity check: + // nothing here compares a signature or checksum before a file becomes the installed + // service binary. That is acceptable only because the install directory is writable + // by administrators alone, which is this command's standing assumption. + prevVer, err := binaryVersionWithRetry(oldBin, upgradeStopTimeout) + if err != nil { + mainLog.Load().Error().Err(err).Msgf("Previous binary at %s is not usable, keeping it for inspection", oldBin) + mainLog.Load().Notice().Msgf("Service is stopped and %s is still the installed binary", bin) + return fmt.Errorf("upgrade failed and no usable previous binary to restore: %w", err) + } + + mainLog.Load().Warn().Msgf("Restoring previous binary: %s (%s)", oldBin, prevVer) + if err := removeBinaryWithRetry(bin, upgradeStopTimeout); err != nil { + mainLog.Load().Error().Err(err).Msg("Failed to remove new binary") + mainLog.Load().Notice().Msg("Service is stopped") + return err + } + if err := os.Rename(oldBin, bin); err != nil { + mainLog.Load().Error().Err(err).Msg("Failed to restore old binary") + mainLog.Load().Notice().Msgf("Service is stopped and %s is missing; reinstall ctrld to recover", bin) + return err + } + if restart() { + mainLog.Load().Notice().Msgf("Restored previous binary successfully - %s", prevVer) + return nil + } + + mainLog.Load().Error().Msg("Restored the previous binary but it did not become ready either") + return errors.New("upgrade failed and the restored binary did not become ready") +} + +// stopServiceAndWait stops the service and waits until the service manager reports +// it stopped. Rollback needs the process gone, not merely asked to stop: a stop +// request returns before the process exits, and on Windows the executable stays +// locked until it does. +func stopServiceAndWait(s service.Service, timeout time.Duration) error { + if err := s.Stop(); err != nil { + // Not fatal: the service may already be stopped, or stopping may fail while + // the process is exiting anyway. The status poll below decides. + mainLog.Load().Debug().Err(err).Msg("Stop request failed, waiting for the process to exit anyway") + } + deadline := time.Now().Add(timeout) + statusReadable := false + var lastErr error + for { + status, err := s.Status() + switch { + case errors.Is(err, service.ErrNotInstalled): + return nil + case err == nil: + statusReadable = true + if status == service.StatusStopped { + return nil + } + default: + lastErr = err + } + if !time.Now().Before(deadline) { + if !statusReadable { + // The status was never readable, so "did not stop" was never observed - + // only "could not be observed". Refusing to continue here would leave the + // broken binary installed with the service stopped, which is the outcome + // rollback exists to avoid. Let the caller proceed: the remove is retried + // while the image is locked, and the restart check still has to pass + // before this reports success. + mainLog.Load().Warn().Err(lastErr).Msgf("Could not read service status within %s; continuing with rollback", timeout) + return nil + } + return fmt.Errorf("service did not stop within %s", timeout) + } + time.Sleep(upgradeStopPollInterval) + } +} + +// binaryVersionWithRetry probes a binary's version, retrying transient exec failures +// until timeout. Only the last error is reported: the earlier attempts are noise once a +// retry has been made. +func binaryVersionWithRetry(path string, timeout time.Duration) (string, error) { + deadline := time.Now().Add(timeout) + for { + version, err := binaryVersionFn(path) + if err == nil { + return version, nil + } + if !time.Now().Before(deadline) { + return "", err + } + mainLog.Load().Debug().Err(err).Msgf("Version probe of %s failed, retrying", path) + time.Sleep(upgradeStopPollInterval) + } +} + +// removeBinaryWithRetry removes path, retrying while it is still locked. Windows +// releases the lock on an executable's image asynchronously after its process exits, +// so a remove issued immediately after the service reports stopped can still fail +// with "Access is denied". +func removeBinaryWithRetry(path string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + err := os.Remove(path) + if err == nil || errors.Is(err, os.ErrNotExist) { + return nil + } + if !time.Now().Before(deadline) { + return fmt.Errorf("could not remove %s within %s: %w", path, timeout, err) + } + time.Sleep(upgradeStopPollInterval) + } +} + +// binaryVersionFn is indirected so rollback can be tested without staging a runnable +// executable per platform. The probe itself is covered directly against the test +// binary; see TestBinaryVersion. +var binaryVersionFn = binaryVersion + +// binaryVersion runs path with "--version" and returns the version it reports. It +// answers "can this binary actually run on this host", which is what rollback needs +// to know before making a file the installed ctrld. +// +// On Windows path is ctrld.exe_previous, whose extension is not in PATHEXT. That +// resolves because os/exec only falls back to appending PATHEXT entries when the path +// has no extension at all (lp_windows.go findExecutable): with one present and the +// file on disk, it is used as-is. A suffix that left no extension - renaming +// oldBinSuffix such that the result is "ctrld_previous" - would break this probe with +// "executable file not found in %PATH%", and rollback would then refuse to restore a +// perfectly good binary. +func binaryVersion(path string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), binaryVersionTimeout) + defer cancel() + out, err := exec.CommandContext(ctx, path, "--version").CombinedOutput() + if err != nil { + return "", fmt.Errorf("running %s --version: %w", path, err) + } + ver, found := strings.CutPrefix(strings.TrimSpace(string(out)), "ctrld version ") + if !found { + return "", fmt.Errorf("unexpected --version output from %s: %q", path, strings.TrimSpace(string(out))) + } + return ver, nil +} diff --git a/cmd/cli/upgrade_rollback_test.go b/cmd/cli/upgrade_rollback_test.go new file mode 100644 index 0000000..0c2f169 --- /dev/null +++ b/cmd/cli/upgrade_rollback_test.go @@ -0,0 +1,288 @@ +package cli + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kardianos/service" +) + +// fakeService implements the parts of service.Service that rollback uses. Any other +// method panics, which keeps accidental dependencies visible. +type fakeService struct { + service.Service + + stopErr error + stopCalls int + statuses []service.Status // consumed one per Status() call; the last repeats + statusErr error + onStopCall func() +} + +func (f *fakeService) Stop() error { + f.stopCalls++ + if f.onStopCall != nil { + f.onStopCall() + } + return f.stopErr +} + +func (f *fakeService) Status() (service.Status, error) { + if f.statusErr != nil { + return service.StatusUnknown, f.statusErr + } + if len(f.statuses) == 0 { + return service.StatusStopped, nil + } + st := f.statuses[0] + if len(f.statuses) > 1 { + f.statuses = f.statuses[1:] + } + return st, nil +} + +func TestStopServiceAndWait(t *testing.T) { + tests := []struct { + name string + svc *fakeService + timeout time.Duration + wantErr bool + }{ + { + name: "stops after a few polls", + svc: &fakeService{statuses: []service.Status{service.StatusRunning, service.StatusRunning, service.StatusStopped}}, + timeout: 5 * time.Second, + }, + { + name: "already stopped", + svc: &fakeService{statuses: []service.Status{service.StatusStopped}}, + timeout: 5 * time.Second, + }, + { + // A stop request that errors is not fatal on its own: the process may be + // exiting anyway, so the status poll decides. + name: "stop errors but service is stopped", + svc: &fakeService{stopErr: errors.New("already stopped"), statuses: []service.Status{service.StatusStopped}}, + timeout: 5 * time.Second, + }, + { + name: "not installed", + svc: &fakeService{statusErr: service.ErrNotInstalled}, + timeout: 5 * time.Second, + }, + { + // The process never exits. Rollback must be told so, because modifying a + // running executable is what produced "Access is denied". + name: "never stops", + svc: &fakeService{statuses: []service.Status{service.StatusRunning}}, + timeout: time.Millisecond, + wantErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := stopServiceAndWait(tc.svc, tc.timeout) + if tc.wantErr && err == nil { + t.Fatal("expected an error, got nil") + } + if !tc.wantErr && err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tc.svc.stopCalls != 1 { + t.Errorf("Stop() called %d times, want 1", tc.svc.stopCalls) + } + }) + } +} + +func TestRemoveBinaryWithRetry(t *testing.T) { + t.Run("removes an existing file", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "ctrld") + if err := os.WriteFile(path, []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + if err := removeBinaryWithRetry(path, time.Second); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Errorf("file still exists after removal: %v", err) + } + }) + + t.Run("missing file is not an error", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "absent") + if err := removeBinaryWithRetry(path, time.Second); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("gives up and reports when the path cannot be removed", func(t *testing.T) { + // A non-empty directory stands in for a locked executable: os.Remove keeps + // failing, so the retry loop must surface the error rather than hang. + dir := filepath.Join(t.TempDir(), "locked") + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "child"), nil, 0o644); err != nil { + t.Fatal(err) + } + if err := removeBinaryWithRetry(dir, time.Millisecond); err == nil { + t.Fatal("expected an error for a path that cannot be removed") + } + }) +} + +func TestBinaryVersion(t *testing.T) { + t.Run("reports the version", func(t *testing.T) { + t.Setenv(envFakeVersionOutput, "ctrld version dev-94fbd3f") + got, err := binaryVersion(os.Args[0]) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != "dev-94fbd3f" { + t.Errorf("binaryVersion() = %q, want %q", got, "dev-94fbd3f") + } + }) + + t.Run("rejects a binary that prints no version", func(t *testing.T) { + // A ctrld.exe_previous that exists and runs, but produces no version output. + // Restoring it would replace a hung service with one that cannot start at all. + t.Setenv(envFakeVersionOutput, envFakeVersionSilent) + if _, err := binaryVersion(os.Args[0]); err == nil { + t.Fatal("expected an error for a binary with no version output") + } + }) + + t.Run("rejects a missing binary", func(t *testing.T) { + if _, err := binaryVersion(filepath.Join(t.TempDir(), "absent")); err == nil { + t.Fatal("expected an error for a missing binary") + } + }) +} + +// stubBinaryVersion makes the version probe report ver for any path, so a rollback +// test does not have to stage a runnable executable. +// +// Staging one is not portable: oldBin is bin+"_previous", so a fixture named "ctrld" +// yields the extension-less "ctrld_previous", which Windows refuses to execute +// ("executable file not found in %PATH%"), and a symlink to the test binary needs a +// privilege Windows does not grant by default. The probe itself is covered against the +// real test binary in TestBinaryVersion; these tests are about rollback's ordering. +func stubBinaryVersion(t *testing.T, ver string, err error) { + t.Helper() + prev := binaryVersionFn + binaryVersionFn = func(string) (string, error) { return ver, err } + t.Cleanup(func() { binaryVersionFn = prev }) +} + +func TestRollbackToPreviousBinaryStopsBeforeTouchingTheBinary(t *testing.T) { + dir := t.TempDir() + bin := filepath.Join(dir, "ctrld") + oldBin := bin + oldBinSuffix + if err := os.WriteFile(bin, []byte("replacement"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(oldBin, []byte("previous"), 0o755); err != nil { + t.Fatal(err) + } + stubBinaryVersion(t, "dev-a75d669", nil) + + // The invariant: when stop runs, the replacement's executable is still untouched. + // Reversing these two is exactly the "Access is denied" defect. + var stopped bool + var binExistedAtStop bool + stop := func() error { + stopped = true + _, err := os.Stat(bin) + binExistedAtStop = err == nil + return nil + } + restarted := false + restart := func() bool { restarted = true; return true } + + if err := rollbackToPreviousBinary(bin, oldBin, stop, restart); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !stopped { + t.Error("rollback did not stop the service") + } + if !binExistedAtStop { + t.Error("the binary was modified before the service was stopped") + } + if !restarted { + t.Error("rollback did not restart the service") + } + if _, err := os.Stat(oldBin); !errors.Is(err, os.ErrNotExist) { + t.Errorf("previous binary was not moved into place: %v", err) + } + if _, err := os.Stat(bin); err != nil { + t.Errorf("restored binary is missing: %v", err) + } +} + +func TestRollbackToPreviousBinaryKeepsUnusablePrevious(t *testing.T) { + dir := t.TempDir() + bin := filepath.Join(dir, "ctrld") + oldBin := bin + oldBinSuffix + if err := os.WriteFile(bin, []byte("replacement"), 0o755); err != nil { + t.Fatal(err) + } + // A previous binary that exists but does not report a version. + if err := os.WriteFile(oldBin, []byte("not a working binary"), 0o755); err != nil { + t.Fatal(err) + } + // Stubbed rather than left to the real probe: that would fail here for the right + // reason on unix (not an executable) but the wrong one on Windows (the fixture's + // name has no extension), so the assertion would not be about usability at all. + stubBinaryVersion(t, "", errors.New("unexpected --version output")) + + stopped := false + restarted := false + err := rollbackToPreviousBinary(bin, oldBin, + func() error { stopped = true; return nil }, + func() bool { restarted = true; return true }, + ) + if err == nil { + t.Fatal("expected an error when the previous binary is unusable") + } + if !stopped { + t.Error("the service must still be stopped: a broken replacement holds enforcement") + } + if restarted { + t.Error("must not restart the service with an unusable binary") + } + // Nothing was swapped, and the previous file is kept for inspection. + if _, err := os.Stat(oldBin); err != nil { + t.Errorf("unusable previous binary was not preserved: %v", err) + } + if _, err := os.Stat(bin); err != nil { + t.Errorf("installed binary was removed despite having nothing to restore: %v", err) + } +} + +func TestRollbackToPreviousBinaryAbortsWhenStopFails(t *testing.T) { + dir := t.TempDir() + bin := filepath.Join(dir, "ctrld") + oldBin := bin + oldBinSuffix + for _, p := range []string{bin, oldBin} { + if err := os.WriteFile(p, []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + } + + stopErr := errors.New("service did not stop within 30s") + err := rollbackToPreviousBinary(bin, oldBin, + func() error { return stopErr }, + func() bool { t.Error("must not restart after a failed stop"); return false }, + ) + if !errors.Is(err, stopErr) { + t.Fatalf("error = %v, want %v", err, stopErr) + } + // The executable of a process that may still be running must be left alone. + if _, err := os.Stat(bin); err != nil { + t.Errorf("binary was modified even though the stop failed: %v", err) + } +} From 084c785ed5e47b2df21a84f2d7f48311a744e486 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 14 Aug 2026 15:22:50 +0700 Subject: [PATCH 10/16] cmd/cli: report whether ctrld finished starting up, not just what SCM thinks "ctrld status" reported the service manager's view and nothing else, so it printed "Service is running" and exited 0 for a process that was alive and registered as started but had never got past startup: no control socket, no DNS listener, no policy applied. The one command an operator reaches for first confirmed the service was fine while the host had no working DNS. Probe the control server's /started endpoint before reporting success. That endpoint only answers once the onStarted hooks have completed, which is after the listeners are up, so a successful probe means the process is serving rather than merely alive. A service that is registered as running but cannot confirm startup is now reported as such, with a pointer to the log, and exits 3 - distinct from stopped (1) and unknown (2), because it needs a different response. A probe blocked by permissions is not evidence of a broken service: an unprivileged caller still gets "Service is running", with a note that startup was not verified. The probe is bounded by a short timeout so status stays fast. Document the exit codes in the command's help, and cover the probe (ready, not finished starting, no socket, timed out) and the classification, including that an unreadable socket is not reported as a failure. The not-ready verdict is only reported when the probe could have found the daemon's socket. socketDir() is caller-relative on unix - the system directory when writable, the caller's home otherwise - so an unprivileged "ctrld status" looks somewhere the root-owned daemon never listened and gets ENOENT, which is "wrong path", not "not ready". Since only darwin has an elevation PreRun and the root-level alias has none, that is the normal invocation; reporting exit 3 there would have told a monitoring check to restart healthy daemons. Such a caller now gets the service manager's view with startup reported as unverified. Windows and mobile resolve the same directory for every caller, so the verdict stays fully available on the platform the hung start was seen on. A successful probe is still conclusive whoever ran it. --- cmd/cli/commands.go | 22 +- cmd/cli/service_image_path.go | 59 ++++++ cmd/cli/service_image_path_others.go | 8 + cmd/cli/service_image_path_test.go | 103 ++++++++++ cmd/cli/service_image_path_windows.go | 41 ++++ cmd/cli/service_status.go | 177 ++++++++++++++++ cmd/cli/service_status_test.go | 281 ++++++++++++++++++++++++++ 7 files changed, 687 insertions(+), 4 deletions(-) create mode 100644 cmd/cli/service_image_path.go create mode 100644 cmd/cli/service_image_path_others.go create mode 100644 cmd/cli/service_image_path_test.go create mode 100644 cmd/cli/service_image_path_windows.go create mode 100644 cmd/cli/service_status.go create mode 100644 cmd/cli/service_status_test.go diff --git a/cmd/cli/commands.go b/cmd/cli/commands.go index b32f6db..5b1a968 100644 --- a/cmd/cli/commands.go +++ b/cmd/cli/commands.go @@ -1047,6 +1047,7 @@ func initStatusCmd() *cobra.Command { statusCmd := &cobra.Command{ Use: "status", Short: "Show status of the ctrld service", + Long: statusCmdLong, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { s, err := newService(&prog{}, svcConfig) @@ -1062,13 +1063,25 @@ func initStatusCmd() *cobra.Command { switch status { case service.StatusUnknown: mainLog.Load().Notice().Msg("Unknown status") - os.Exit(2) + os.Exit(statusExitUnknown) case service.StatusRunning: - mainLog.Load().Notice().Msg("Service is running") - os.Exit(0) + // The service manager only knows a process was created. It reports a + // service as running even when the process is still in startup, with + // no control socket, no DNS listener and no policy applied - so + // "Service is running" can describe a host with no working DNS. + // Probe readiness before claiming it. + ready, probeErr := serviceReady() + if probeErr != nil { + mainLog.Load().Debug().Err(probeErr).Msg("Readiness probe did not confirm startup") + } + r := classifyReadiness(ready, probeErr, readinessVerifiable()) + for _, msg := range r.messages { + mainLog.Load().Notice().Msg(msg) + } + os.Exit(r.exitCode) case service.StatusStopped: mainLog.Load().Notice().Msg("Service is stopped") - os.Exit(1) + os.Exit(statusExitStopped) } }, } @@ -1082,6 +1095,7 @@ func initStatusCmd() *cobra.Command { statusCmdAlias := &cobra.Command{ Use: "status", Short: "Show status of the ctrld service", + Long: statusCmdLong, Args: cobra.NoArgs, Run: statusCmd.Run, } diff --git a/cmd/cli/service_image_path.go b/cmd/cli/service_image_path.go new file mode 100644 index 0000000..6b2fc8c --- /dev/null +++ b/cmd/cli/service_image_path.go @@ -0,0 +1,59 @@ +package cli + +import "strings" + +// serviceBinaryFromImagePath extracts the executable path from a Windows service +// ImagePath value, which carries the command line rather than a bare path: it may be +// quoted and is usually followed by arguments, e.g. +// +// "C:\Program Files\Control D\ctrld.exe" run --config C:\...\ctrld.toml +// +// It returns "" when no path can be read, which callers must treat as "cannot tell" +// rather than "does not match". +func serviceBinaryFromImagePath(imagePath string) string { + imagePath = strings.TrimSpace(imagePath) + if imagePath == "" { + return "" + } + if imagePath[0] == '"' { + // Quoted form: everything up to the closing quote is the path, so a directory + // containing spaces stays intact. + if end := strings.IndexByte(imagePath[1:], '"'); end >= 0 { + return strings.TrimSpace(imagePath[1 : 1+end]) + } + return strings.TrimSpace(imagePath[1:]) + } + // Unquoted form: the path cannot contain spaces, so the first field is it. + if idx := strings.IndexByte(imagePath, ' '); idx >= 0 { + return strings.TrimSpace(imagePath[:idx]) + } + return imagePath +} + +// sameExecutableDir reports whether two Windows executable paths live in the same +// directory, compared case-insensitively because Windows paths are. +// +// The separator handling is explicit rather than filepath's, because filepath follows the +// *host* rules: off Windows it does not treat "\\" as a separator, so every backslash path +// would reduce to the same directory and any two paths would compare equal. Doing it here +// keeps the comparison correct and testable on any host. +// +// A path with no directory part answers false, which callers read as "cannot tell". +func sameExecutableDir(a, b string) bool { + dirA, dirB := windowsExecutableDir(a), windowsExecutableDir(b) + if dirA == "" || dirB == "" { + return false + } + return strings.EqualFold(dirA, dirB) +} + +// windowsExecutableDir returns the directory part of a Windows path, accepting either +// separator and normalising to a backslash. It returns "" when there is no directory part. +func windowsExecutableDir(path string) string { + path = strings.TrimSpace(path) + idx := strings.LastIndexAny(path, `\/`) + if idx <= 0 { + return "" + } + return strings.ReplaceAll(path[:idx], "/", `\`) +} diff --git a/cmd/cli/service_image_path_others.go b/cmd/cli/service_image_path_others.go new file mode 100644 index 0000000..866308b --- /dev/null +++ b/cmd/cli/service_image_path_others.go @@ -0,0 +1,8 @@ +//go:build !windows + +package cli + +// installedServiceDirMatches is Windows-only: it exists because socketDir() there is +// relative to the running executable. Other platforms answer this question through +// hasElevatedPrivilege in readinessVerifiable. +func installedServiceDirMatches() bool { return true } diff --git a/cmd/cli/service_image_path_test.go b/cmd/cli/service_image_path_test.go new file mode 100644 index 0000000..2fb050f --- /dev/null +++ b/cmd/cli/service_image_path_test.go @@ -0,0 +1,103 @@ +package cli + +import "testing" + +// TestServiceBinaryFromImagePath covers the ImagePath shapes Windows stores. Getting this +// wrong makes readinessVerifiable compare the wrong directories, and "ctrld status" would +// then report a healthy service as not-ready - the false positive the readiness exit code +// exists to avoid. +func TestServiceBinaryFromImagePath(t *testing.T) { + tests := []struct { + name string + imagePath string + want string + }{ + { + // The installed form: quoted because the directory contains a space, with the + // service arguments following it. + name: "quoted path with arguments", + imagePath: `"C:\Program Files\Control D\ctrld.exe" run --config "C:\ProgramData\Control D\ctrld.toml"`, + want: `C:\Program Files\Control D\ctrld.exe`, + }, + { + name: "quoted path without arguments", + imagePath: `"C:\Program Files\Control D\ctrld.exe"`, + want: `C:\Program Files\Control D\ctrld.exe`, + }, + { + name: "unquoted path with arguments", + imagePath: `C:\ctrld\ctrld.exe run --cd abc123`, + want: `C:\ctrld\ctrld.exe`, + }, + { + name: "unquoted path alone", + imagePath: `C:\ctrld\ctrld.exe`, + want: `C:\ctrld\ctrld.exe`, + }, + { + name: "surrounding whitespace", + imagePath: ` "C:\ctrld\ctrld.exe" run `, + want: `C:\ctrld\ctrld.exe`, + }, + { + // Unterminated quote: take what is there rather than returning nothing, since + // "" means "cannot tell" and would silently disable the check. + name: "unterminated quote", + imagePath: `"C:\ctrld\ctrld.exe run`, + want: `C:\ctrld\ctrld.exe run`, + }, + { + name: "empty", + imagePath: "", + want: "", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := serviceBinaryFromImagePath(tc.imagePath); got != tc.want { + t.Errorf("serviceBinaryFromImagePath(%q) = %q, want %q", tc.imagePath, got, tc.want) + } + }) + } +} + +// TestSameExecutableDir pins the comparison itself: Windows paths are case-insensitive, and +// an empty side means "cannot tell", which must never read as a match. +func TestSameExecutableDir(t *testing.T) { + tests := []struct { + name string + a string + b string + want bool + }{ + { + name: "same directory", + a: `C:\Program Files\Control D\ctrld.exe`, + b: `C:\Program Files\Control D\ctrld.exe`, + want: true, + }, + { + name: "same directory different case", + a: `C:\Program Files\Control D\ctrld.exe`, + b: `c:\program files\control d\ctrld.exe`, + want: true, + }, + { + // The case the check exists for: a copy run from a download directory + // resolves a different control socket than the installed service. + name: "different directory", + a: `C:\Program Files\Control D\ctrld.exe`, + b: `C:\Users\admin\Downloads\ctrld.exe`, + want: false, + }, + {name: "unknown installed path", a: "", b: `C:\ctrld\ctrld.exe`, want: false}, + {name: "unknown self path", a: `C:\ctrld\ctrld.exe`, b: "", want: false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sameExecutableDir(tc.a, tc.b); got != tc.want { + t.Errorf("sameExecutableDir(%q, %q) = %v, want %v", tc.a, tc.b, got, tc.want) + } + }) + } +} diff --git a/cmd/cli/service_image_path_windows.go b/cmd/cli/service_image_path_windows.go new file mode 100644 index 0000000..f7c6ef8 --- /dev/null +++ b/cmd/cli/service_image_path_windows.go @@ -0,0 +1,41 @@ +//go:build windows + +package cli + +import ( + "os" + + "golang.org/x/sys/windows/registry" +) + +// installedServiceDirMatches reports whether this executable is the installed service +// binary, by comparing its directory with the one in the service's registered ImagePath. +// +// socketDir() on Windows is relative to the running executable, so a ctrld.exe run from +// somewhere else - a download directory, a build tree - looks for the control socket in +// its own directory and never finds the installed daemon's. A failed probe from there +// says nothing about the service's health, and reporting "not ready" for it would tell +// monitoring to restart a healthy service. +// +// Anything unreadable answers true, keeping the previous behaviour: readiness stays +// verifiable unless there is positive evidence of a different install. +func installedServiceDirMatches() bool { + self, err := os.Executable() + if err != nil { + return true + } + key, err := registry.OpenKey(registry.LOCAL_MACHINE, `SYSTEM\CurrentControlSet\Services\`+ctrldServiceName, registry.QUERY_VALUE) + if err != nil { + return true + } + defer key.Close() + imagePath, _, err := key.GetStringValue("ImagePath") + if err != nil { + return true + } + installed := serviceBinaryFromImagePath(imagePath) + if installed == "" { + return true + } + return sameExecutableDir(installed, self) +} diff --git a/cmd/cli/service_status.go b/cmd/cli/service_status.go new file mode 100644 index 0000000..486c6bc --- /dev/null +++ b/cmd/cli/service_status.go @@ -0,0 +1,177 @@ +package cli + +import ( + "errors" + "fmt" + "io/fs" + "net/http" + "path/filepath" + "runtime" + "time" +) + +// Exit codes reported by "ctrld status". +const ( + statusExitRunning = 0 + statusExitStopped = 1 + statusExitUnknown = 2 + // statusExitNotReady means the service manager considers the service running, + // but the process has not finished starting up, so it is not serving DNS or + // applying policy. This is a distinct code because it needs a distinct response: + // the process exists, so restarting the service is what recovers it, while a + // stopped service needs starting and an unknown state needs investigation. + statusExitNotReady = 3 +) + +// serviceReadinessTimeout bounds the control-socket probe. Status must answer +// quickly, and a service that cannot respond within this window is not usefully +// "running" from a caller's point of view either way. +const serviceReadinessTimeout = 3 * time.Second + +// statusCmdLong documents what the reported states mean, including that a service the +// OS calls running is not necessarily serving. +const statusCmdLong = `Show status of the ctrld service. + +Reports both what the OS service manager thinks and whether ctrld has finished +starting up, since a service can be registered as running while its process is +still in startup and serving nothing. + +Exit codes: + 0 running and serving, or running with startup not verified + 1 stopped + 2 status unknown + 3 registered as running, but startup has not completed + +Verifying startup requires reaching ctrld's control socket. On Linux, BSD and macOS +that socket lives in a directory only the privileged user resolves, so an +unprivileged "ctrld status" reports the service manager's view and says startup was +not verified rather than claiming the service is unhealthy. Exit 3 is only reported +when the check could actually be made.` + +// readiness is what "ctrld status" reports for a service the service manager +// considers running. +type readiness struct { + messages []string + exitCode int +} + +// readinessVerifiable reports whether a failed control-socket probe can be trusted to +// mean "the service has not finished starting up". +// +// It can only mean that if this process resolves the same socket path the daemon +// created, and socketDir() is caller-relative on unix: it returns the system directory +// only when that is writable, and the caller's home directory otherwise. So a +// root-owned daemon listens on /var/run/ctrld_control.sock while an unprivileged +// "ctrld status" looks under $HOME, finds nothing, and gets ENOENT - which means "wrong +// path", not "not ready". Reporting exit 3 there would tell a monitoring check to +// restart a perfectly healthy daemon. +// +// On Windows and mobile socketDir() is the install/home directory for every caller, so +// the probe is comparable - which matters because Windows is where the hung-start this +// exit code exists for was seen. On Windows that only holds while this binary is the +// installed one: a copy run from elsewhere resolves a different socket directory, so its +// failed probe would say nothing about the service. installedServiceDirMatches() checks +// that, and answers true when it cannot tell, preserving the previous behaviour. +func readinessVerifiable() bool { + if isMobile() { + return true + } + if runtime.GOOS == "windows" { + return installedServiceDirMatches() + } + elevated, err := hasElevatedPrivilege() + return err == nil && elevated +} + +// classifyReadiness turns a control-socket probe result into the report for a service +// the service manager calls running. +// +// verifiable comes from readinessVerifiable: when it is false a failed probe says +// nothing about the service, so the report falls back to the service manager's view. +// A *successful* probe is still conclusive either way - reaching the socket at all is +// positive evidence, whoever the caller is. +func classifyReadiness(ready bool, err error, verifiable bool) readiness { + switch { + case ready: + return readiness{ + messages: []string{"Service is running"}, + exitCode: statusExitRunning, + } + case !verifiable: + return readiness{ + messages: []string{"Service is running (startup not verified: re-run with elevated privileges to check readiness)"}, + exitCode: statusExitRunning, + } + case errors.Is(err, errReadinessNotReported): + // The service answered, just not with a verdict - an older daemon without the + // /started route. It is alive and reachable, so the service manager's view is + // the best available answer. + return readiness{ + messages: []string{"Service is running (startup not verified: this ctrld build does not report readiness)"}, + exitCode: statusExitRunning, + } + case errors.Is(err, fs.ErrPermission): + // Without access to the control socket there is nothing to report beyond the + // service manager's view. Do not call a service unhealthy because the caller + // lacks privilege. + return readiness{ + messages: []string{"Service is running (startup not verified: control socket requires elevated privileges)"}, + exitCode: statusExitRunning, + } + default: + return readiness{ + messages: []string{ + "Service is registered as running, but has not completed startup: it is not serving DNS", + "Check the ctrld log for why startup did not finish, then restart the service", + }, + exitCode: statusExitNotReady, + } + } +} + +// serviceReady reports whether a running ctrld has finished starting up, by asking +// its control server. The control server answers /started only once the onStarted +// hooks have completed, which is after the DNS listeners are up, so a successful +// probe means the process is actually serving rather than merely alive. +// +// An error means "could not confirm readiness" and is returned for the caller to +// classify: a refused connection or missing socket is a process that never got that +// far, while a permission error says nothing about the service's health. +func serviceReady() (bool, error) { + dir, err := socketDir() + if err != nil { + return false, err + } + return serviceReadyAt(filepath.Join(dir, ControlSocketName()), serviceReadinessTimeout) +} + +// errReadinessNotReported marks a control server that answered without a readiness +// verdict. +// +// http.Client.Post returns (resp, nil) for any status, so a daemon with no /started +// route answers 404 and an internal failure answers 5xx - neither says the service has +// not started. Reporting "not ready" there tells a monitoring check to restart a healthy +// service, and it happens in normal operation: after an upgrade replaces the binary on +// disk but before the service restarts, and throughout a mixed-version rollout. +var errReadinessNotReported = errors.New("control server did not report readiness") + +// serviceReadyAt is serviceReady against an explicit socket path and timeout. +func serviceReadyAt(sockPath string, timeout time.Duration) (bool, error) { + cc := newControlClient(sockPath) + cc.c.Timeout = timeout + resp, err := cc.post(startedPath, nil) + if err != nil { + return false, err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusRequestTimeout: + // The daemon's own verdict: its onStarted hooks have not completed. This is the + // hung start statusExitNotReady exists for. + return false, nil + default: + return false, fmt.Errorf("%w: HTTP %d", errReadinessNotReported, resp.StatusCode) + } +} diff --git a/cmd/cli/service_status_test.go b/cmd/cli/service_status_test.go new file mode 100644 index 0000000..2d4d3b3 --- /dev/null +++ b/cmd/cli/service_status_test.go @@ -0,0 +1,281 @@ +package cli + +import ( + "errors" + "io/fs" + "net" + "net/http" + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +// startControlSocket serves handler on a unix socket and returns its path. +func startControlSocket(t *testing.T, handler http.HandlerFunc) string { + t.Helper() + // Keep the path short: unix socket paths have a low length limit. + dir, err := os.MkdirTemp("", "ctrldsock") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + + sockPath := filepath.Join(dir, "s.sock") + ln, err := net.Listen("unix", sockPath) + if err != nil { + t.Skipf("cannot listen on a unix socket: %v", err) + } + mux := http.NewServeMux() + mux.Handle(startedPath, handler) + srv := &http.Server{Handler: mux} + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + return sockPath +} + +func TestServiceReadyAt(t *testing.T) { + t.Run("ready when the control server reports started", func(t *testing.T) { + sock := startControlSocket(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + ready, err := serviceReadyAt(sock, time.Second) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !ready { + t.Error("ready = false, want true") + } + }) + + t.Run("not ready when startup has not finished", func(t *testing.T) { + // What /started returns when the onStarted hooks have not completed. + sock := startControlSocket(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusRequestTimeout) + }) + ready, err := serviceReadyAt(sock, time.Second) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ready { + t.Error("ready = true for a control server that has not finished startup") + } + }) + + t.Run("not ready when there is no control socket", func(t *testing.T) { + // The incident: the process was alive but had never created the socket, so + // every control request was refused. + ready, err := serviceReadyAt(filepath.Join(t.TempDir(), "absent.sock"), time.Second) + if ready { + t.Error("ready = true with no control socket") + } + if err == nil { + t.Error("expected an error when the control socket does not exist") + } + }) + + t.Run("not ready when the probe times out", func(t *testing.T) { + sock := startControlSocket(t, func(w http.ResponseWriter, r *http.Request) { + time.Sleep(2 * time.Second) + w.WriteHeader(http.StatusOK) + }) + ready, err := serviceReadyAt(sock, 50*time.Millisecond) + if ready { + t.Error("ready = true for a probe that timed out") + } + if err == nil { + t.Error("expected an error when the probe times out") + } + }) +} + +func TestClassifyReadiness(t *testing.T) { + tests := []struct { + name string + ready bool + err error + verifiable bool + wantCode int + }{ + { + name: "ready", + ready: true, + verifiable: true, + wantCode: statusExitRunning, + }, + { + // The service manager says running, the process is not serving. This + // must not report success. + name: "running but never finished startup", + err: errors.New("connect: connection refused"), + verifiable: true, + wantCode: statusExitNotReady, + }, + { + // A caller without privilege cannot probe; that is not evidence of a + // broken service, so it must not be reported as one. + name: "probe not permitted", + err: fs.ErrPermission, + verifiable: true, + wantCode: statusExitRunning, + }, + { + name: "wrapped permission error", + err: &net.OpError{Op: "dial", Err: fs.ErrPermission}, + verifiable: true, + wantCode: statusExitRunning, + }, + { + // The P2: an unprivileged caller on unix resolves a socket path the + // daemon never used, so the probe fails with ENOENT rather than a + // permission error. That says nothing about the service and must not be + // reported as unhealthy - a monitoring check acting on exit 3 would + // restart a healthy daemon. + name: "missing socket at an unverifiable path", + err: &net.OpError{Op: "dial", Err: os.ErrNotExist}, + verifiable: false, + wantCode: statusExitRunning, + }, + { + name: "connection refused at an unverifiable path", + err: errors.New("connect: connection refused"), + verifiable: false, + wantCode: statusExitRunning, + }, + { + // A probe that actually reached the socket is conclusive whoever ran it. + name: "successful probe is trusted even when unverifiable", + ready: true, + verifiable: false, + wantCode: statusExitRunning, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := classifyReadiness(tc.ready, tc.err, tc.verifiable) + if got.exitCode != tc.wantCode { + t.Errorf("exitCode = %d, want %d", got.exitCode, tc.wantCode) + } + if len(got.messages) == 0 { + t.Error("no message to report") + } + }) + } +} + +// TestReadinessVerifiableMatchesSocketVisibility is the closure test for the P2: the +// not-ready verdict must only be reachable when this process resolves the same socket +// directory the daemon uses. +// +// On unix that is the privileged user's path, so an unprivileged run - which is how +// "ctrld status" is normally invoked, since only darwin has an elevation PreRun and the +// root-level alias has none - must not be able to reach exit 3. +func TestReadinessVerifiableMatchesSocketVisibility(t *testing.T) { + verifiable := readinessVerifiable() + + if runtime.GOOS == "windows" { + if !verifiable { + t.Error("on Windows every caller resolves the install directory, so the probe is always verifiable") + } + return + } + + elevated, err := hasElevatedPrivilege() + if err != nil { + t.Skipf("cannot determine privilege: %v", err) + } + if verifiable != elevated { + t.Errorf("readinessVerifiable() = %v, want %v (elevated)", verifiable, elevated) + } + + if !elevated { + // The shape the review asked to assert: unprivileged, healthy daemon, and a + // probe that cannot see its socket must still report running. + dir, err := socketDir() + if err != nil { + t.Fatalf("socketDir(): %v", err) + } + if dir == "/var/run" { + t.Skip("unprivileged but /var/run is writable, so the probe path does match") + } + r := classifyReadiness(false, &net.OpError{Op: "dial", Err: os.ErrNotExist}, verifiable) + if r.exitCode == statusExitNotReady { + t.Errorf("unprivileged status probing %q reported not-ready (exit %d) for a healthy service", dir, r.exitCode) + } + } +} + +// Every status must map to its own exit code: a caller that cannot tell a hung +// service from a healthy or a stopped one is back to the incident's diagnostics. +// +// The literal values are the contract. statusCmdLong documents them and monitoring +// scripts key off them, so asserting the constants against each other would let a +// renumbering keep the suite green while silently breaking every caller. +func TestStatusExitCodesAreDistinct(t *testing.T) { + for _, tc := range []struct { + name string + got int + want int + }{ + {"running", statusExitRunning, 0}, + {"stopped", statusExitStopped, 1}, + {"unknown", statusExitUnknown, 2}, + {"not ready", statusExitNotReady, 3}, + } { + if tc.got != tc.want { + t.Errorf("%s exit code = %d, want %d: statusCmdLong and monitoring scripts document this value", tc.name, tc.got, tc.want) + } + } + + codes := map[int]string{ + statusExitRunning: "running", + statusExitStopped: "stopped", + statusExitUnknown: "unknown", + statusExitNotReady: "not ready", + } + if len(codes) != 4 { + t.Errorf("status exit codes collide, only %d distinct: %v", len(codes), codes) + } +} + +// TestReadinessProbeStatusHandling covers what each control-server answer means. +// +// http.Client.Post returns (resp, nil) for any status code, so a daemon without the +// /started route answers 404 and the probe must report "cannot confirm" rather than "not +// started". That state is reached in normal operation - after an upgrade replaces the +// binary but before the service restarts, and throughout a mixed-version rollout - and +// reporting exit 3 there tells monitoring to restart a healthy service. +func TestReadinessProbeStatusHandling(t *testing.T) { + tests := []struct { + name string + status int + wantReady bool + wantReported bool // whether the answer carries a readiness verdict + wantExitCode int + }{ + {"started", http.StatusOK, true, true, statusExitRunning}, + {"still starting", http.StatusRequestTimeout, false, true, statusExitNotReady}, + {"no readiness route", http.StatusNotFound, false, false, statusExitRunning}, + {"control server error", http.StatusInternalServerError, false, false, statusExitRunning}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + status := tc.status + sock := startControlSocket(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(status) + }) + + ready, err := serviceReadyAt(sock, time.Second) + if ready != tc.wantReady { + t.Errorf("ready = %v, want %v", ready, tc.wantReady) + } + if reported := !errors.Is(err, errReadinessNotReported); reported != tc.wantReported { + t.Errorf("readiness reported = %v, want %v (err: %v)", reported, tc.wantReported, err) + } + if got := classifyReadiness(ready, err, true).exitCode; got != tc.wantExitCode { + t.Errorf("exit code = %d, want %d", got, tc.wantExitCode) + } + }) + } +} From 8ce3b7ca6c5400060aadae867290527a40014188 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 14 Aug 2026 15:23:11 +0700 Subject: [PATCH 11/16] cmd/cli: log the config error that rejected a custom config The warning reported err, the resolver-config fetch error, which is nil on every path that reaches it - so a rejected custom config was logged with no reason attached. cfgErr holds the validation failure. --- cmd/cli/cli.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 610dfc4..daff7dd 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -855,7 +855,10 @@ func processCDFlags(ctx context.Context, cfg *ctrld.Config) (*controld.ResolverC return resolverConfig, nil } } - mainLog.Load().Warn().Err(err).Msg("disregarding invalid custom config") + // cfgErr, not err: err is the resolver-config fetch error from above, which is + // nil on every path that reaches here, so logging it said nothing about why the + // custom config was rejected. + mainLog.Load().Warn().Err(cfgErr).Msg("disregarding invalid custom config") } bootstrapIP := func(endpoint string) string { From 753d24502957ddb04c4f4623cc2c28957b7c8e32 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 17 Aug 2026 15:51:25 +0700 Subject: [PATCH 12/16] Bump bump insomniacslk/dhcp to c76316d For fixing nclient4 panic. See: https://github.com/insomniacslk/dhcp/pull/583 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index db654b9..ed7205b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/illarion/gonotify/v2 v2.0.3 - github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2 + github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82 github.com/jaypipes/ghw v0.21.0 github.com/jaytaylor/go-hostsfile v0.0.0-20220426042432-61485ac1fa6c github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 diff --git a/go.sum b/go.sum index b80b0cf..3b53054 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,8 @@ github.com/illarion/gonotify/v2 v2.0.3 h1:B6+SKPo/0Sw8cRJh1aLzNEeNVFfzE3c6N+o+vy github.com/illarion/gonotify/v2 v2.0.3/go.mod h1:38oIJTgFqupkEydkkClkbL6i5lXV/bxdH9do5TALPEE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2 h1:9K06NfxkBh25x56yVhWWlKFE8YpicaSfHwoV8SFbueA= -github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2/go.mod h1:3A9PQ1cunSDF/1rbTq99Ts4pVnycWg+vlPkfeD2NLFI= +github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82 h1:y5aU8Uvl7eyM5WNgdQvRxbMJb+zo7pD+S72/Yo4pvnQ= +github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82/go.mod h1:qfvBmyDNp+/liLEYWRvqny/PEz9hGe2Dz833eXILSmo= github.com/jaypipes/ghw v0.21.0 h1:ClG2xWtYY0c1ud9jZYwVGdSgfCI7AbmZmZyw3S5HHz8= github.com/jaypipes/ghw v0.21.0/go.mod h1:GPrvwbtPoxYUenr74+nAnWbardIZq600vJDD5HnPsPE= github.com/jaypipes/pcidb v1.1.1 h1:QmPhpsbmmnCwZmHeYAATxEaoRuiMAJusKYkUncMC0ro= From 1f001a559a61d1268a8661703fb6411cd70c1f4e Mon Sep 17 00:00:00 2001 From: Anthony Wong Date: Thu, 20 Aug 2026 15:51:17 +0000 Subject: [PATCH 13/16] feat(cli): add stable provisioning failure codes for manual and MDM installs --- SPEC.md | 206 +++++++++++ cmd/cli/cli.go | 177 ++++++++-- cmd/cli/cli_provision_test.go | 329 ++++++++++++++++++ cmd/cli/commands.go | 201 ++++++++--- cmd/cli/commands_start_test.go | 122 +++++++ cmd/cli/dns_intercept_darwin.go | 7 +- cmd/cli/provision_result.go | 247 +++++++++++++ cmd/cli/provision_result_test.go | 278 +++++++++++++++ cmd/cli/service.go | 28 +- cmd/cli/service_test.go | 57 +++ docs/provisioning-failure-codes.md | 61 ++++ tasks/plan.md | 213 ++++++++++++ tasks/todo.md | 21 ++ .../test-postinstall-provision-failure.sh | 148 ++++++++ 14 files changed, 1988 insertions(+), 107 deletions(-) create mode 100644 SPEC.md create mode 100644 cmd/cli/cli_provision_test.go create mode 100644 cmd/cli/commands_start_test.go create mode 100644 cmd/cli/provision_result.go create mode 100644 cmd/cli/provision_result_test.go create mode 100644 docs/provisioning-failure-codes.md create mode 100644 tasks/plan.md create mode 100644 tasks/todo.md create mode 100755 test-scripts/darwin/test-postinstall-provision-failure.sh diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 0000000..706baee --- /dev/null +++ b/SPEC.md @@ -0,0 +1,206 @@ +# SPEC: Stable customer-visible provisioning failure codes + +Issue: [#586](https://gitlab.int.windscribe.com/controld/clients/ctrld/-/issues/586) +Requested by: Catt Garrod (@catt). Scope expanded by: Anthony Wong (@anthony). + +## 1. Objective + +Terminal provisioning failures in ctrld — bootstrap/API setup, listener +binding, and service installation/startup — must produce a stable, +support-facing failure identifier that survives process exit and reaches +both manual CLI users and MDM-driven installs. A customer or admin reports +one code; Support maps it to a scenario and a next action without asking +for reruns or verbose logs. + +Motivating incident (v1.5.5, macOS): provisioning reached the Control D +API, then died with only `FTL listener.0 could not find available listen +ip and port`. The per-address UDP/TCP bind errors existed only at Info +level in an in-memory logger and vanished on exit. The macOS pkg +`postinstall` discards ctrld's stdout/stderr entirely and judges success +by plist existence, so nothing useful reached the MDM log. + +**Users:** end customers and IT admins reporting failures; Support agents +triaging them; MDM/RMM operators reading installer logs. + +### Failure contract (agreed design) + +Three surfaces, all carrying the same identifier: + +1. **Result file** — on terminal provisioning failure, ctrld writes a + small redacted JSON file (atomic write: temp + rename) in the ctrld + home directory (same base dir as the internal `ctrld.log`, + via `absHomeDir`). Removed/overwritten on later successful + provisioning so stale failures don't mislead. Schema: + + ```json + { + "version": 1, + "timestamp": "2026-08-18T12:00:00Z", + "stage": "listener", + "code": "LISTENER_BIND_FAILED", + "exit_code": 41, + "message": "could not find available listen ip and port", + "detail": { + "attempts": [ + {"addr": "127.0.0.1:53", "proto": "udp", "os_error": "address already in use"} + ] + } + } + ``` + + `detail` is bounded (cap recorded bind attempts; cap string lengths) + and redacted by construction: no provisioning tokens, resolver IDs, + config contents, or unrelated host data. + +2. **Exit code + final stderr line** — the installer-facing command + (`ctrld start`, and `ctrld run` when run manually in the foreground) + exits with a stage-scoped code and prints one final line containing + the string code and stage, e.g. + `provisioning failed: stage=listener code=LISTENER_BIND_FAILED (exit 41)`. + +3. **Installer log (MDM path)** — `scripts/pkg/postinstall` stops + discarding the signal: it captures `ctrld start`'s output to a + private temp file, extracts only the fixed-charset identifier line + (`stage=[a-z]* code=[A-Z_]* (exit [0-9]*)` — structurally unable to + carry the token), and echoes it with the exit code into the + installer log. The result file's `message`/`detail` fields are + deliberately never surfaced there. The plist-existence check remains + the final success gate. + +### Identifier format + +- **Primary identifier: stable string codes.** Initial set — + bootstrap: `API_UNREACHABLE`, `API_REJECTED`, `API_DEVICE_INVALID`; + listener: `LISTENER_BIND_FAILED`, `LISTENER_CONFIGURED_ADDR_UNAVAILABLE`; + service: `SERVICE_INSTALL_FAILED`, `SERVICE_START_FAILED`, + `SERVICE_SELFCHECK_FAILED`. Codes are append-only; renames are new + codes plus a deprecation note in the mapping doc. +- **Secondary: stage-scoped process exit codes** as a coarse machine + signal: bootstrap 30–39, listener 40–49, service install/start 50–59. + Each string code owns one exit code. Existing contracts are untouched: + `ctrld status` 0–3, deactivation-pin 126, success 0. +- One underlying failure maps to one code on every path (manual CLI and + MDM), on both branches. + +### Propagation (daemon → installer) + +The listener/bootstrap fatals fire inside the daemon process +(`ctrld run` under launchd/systemd/SCM), not in `ctrld start`. The +daemon writes the result file before exiting; the existing log-socket +exit notification (`notifyExitToLogServer`) already unblocks `ctrld +start`'s self-check. `ctrld start` then reads the result file, prints +the identifier, and exits with the mapped stage exit code. The daemon's +own exit-status semantics toward service managers are preserved — +in particular the deliberate exit-0 on permanent API rejection that +protects the restart-policy budget; the result file carries the failure +identity in that case. + +### Support mapping + +`docs/provisioning-failure-codes.md` in this repo: one row per code — +code, stage, exit code, failure scenario, next safe troubleshooting +action or evidence request. Updated in the same MR whenever a code is +added or changed. + +### Branch scope + +Full implementation on **both** `v1.0` (release line for v1.5.5) and +`master`. The branches diverge heavily (`v1.0`: zerolog fork, +`commands.go`, `service_status.go`, macOS pkg scripts; `master`: zap, +inline commands, no pkg scripts), so this is one shared contract +(codes, exit-code ranges, file schema, doc) implemented twice, as two +MRs referencing #586. + +## 2. Commands + +- Build: `go build ./...` +- Test: `go test ./cmd/cli/...` (full: `go test ./...`) +- Vet: `go vet ./...` +- Branch workflow: feature branch off `v1.0` for the v1.0 MR; separate + feature branch off `master` for the port MR. Rebase, never merge the + base branch in. + +## 3. Project structure + +New and touched files on `v1.0` (master port mirrors the same contract +at its equivalent emission points in its `cli.go`): + +- `cmd/cli/provision_result.go` (new) — stage + code enums, exit-code + mapping, result-file schema, atomic write/read/clear helpers, + bounded/redacted detail builders. Pattern follows `service_status.go` + (small file: named constants + classifier + dedicated tests). +- `cmd/cli/provision_result_test.go` (new). +- `cmd/cli/cli.go` — emission points: `run()` bootstrap failure branches + (permanent rejection, invalid-device, fatal fetch), and + `tryUpdateListenerConfig` / `tryUpdateListenerConfigIntercept` fatals, + which now record per-attempt `{addr, proto, os_error}` bind detail. +- `cmd/cli/commands.go` — `initStartCmd`: doTasks install/start failures + and the self-check failure branch read the result file, print the + identifier, and exit with the stage code (replacing bare `os.Exit(1)` + on those paths). +- `scripts/pkg/postinstall` — propagate exit code + result-file contents + into the installer log (v1.0 only; master has no pkg scripts). +- `docs/provisioning-failure-codes.md` (new) — support mapping. + +## 4. Code style + +- Per repo conventions and global rules: guard clauses, small functions, + descriptive names, explicit error handling — never weaken existing + handling (e.g. keep the permanent-rejection exit-0 rationale intact). +- Comments only for non-obvious constraints (e.g. why the daemon must + still exit 0 on permanent rejection), simple-english, self-contained — + no issue/MR references in code. +- Match each branch's logging idiom: zerolog fork on `v1.0`, zap on + `master`. No new dependencies. +- Conventional Commits; MR titles in simple-english; both MRs reference + #586 (release-line MR carries `Closes #586`). + +## 5. Testing strategy + +Test-first where the harness allows. Coverage required by the issue: + +- **Code/mapping unit tests** — every string code maps to exactly one + stage and one in-range exit code; ranges don't collide with existing + contracts (0–3 status, 126 pin). +- **Result file round-trip** — write/read/clear; atomic write; stale + file removed on success. +- **Redaction** — serialize a result built from inputs containing a + provision token, resolver ID, and config content; assert none appear. +- **Listener bind failure (regression test for the incident)** — occupy + a port, drive the listener-config path to exhaustion, assert the + result records `LISTENER_BIND_FAILED` with attempted address, UDP/TCP + operation, and OS error (`address already in use`-class). +- **Bootstrap failures** — mock API: permanent 4xx → `API_REJECTED`; + invalid-device 40402 → `API_DEVICE_INVALID`; unreachable → + `API_UNREACHABLE`. +- **Service install/start/self-check failures** — injected task + failures assert code selection and `ctrld start` exit code. +- **MDM surface** — shell-level check of `postinstall` failure branch + (result file present → correct log line and exit), aligned with the + existing `test-scripts/` approach; manual pkg verification steps + documented in the MR. +- Both branches: the shared contract tests exist on both; branch-specific + emission tests match each branch's structure. + +## 6. Boundaries + +**Always:** +- Redact tokens, resolver IDs, config contents, host data from every + customer-visible surface (result file, stderr line, installer log). +- Preserve existing exit-code contracts (`ctrld status` 0–3, pin 126) + and the daemon's service-manager-facing exit semantics. +- Bound all recorded detail (attempt counts, string lengths). +- Keep codes append-only once merged. + +**Ask first:** +- Changing the daemon's (`ctrld run` under a service manager) exit codes + or restart-relevant behavior beyond writing the result file. +- Adding any persisted file outside the ctrld home directory. +- Expanding scope to runtime (post-provisioning) failures — this ticket + owns terminal provisioning failures only. + +**Never:** +- Print or persist the provisioning token (the reason postinstall + discards output today — the replacement surface must stay token-free). +- Auto-detect or kill conflicting processes (explicitly out of scope). +- Break `ctrld status`'s documented exit-code contract. diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index daff7dd..1f495a3 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -342,29 +342,8 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { return } - cdLogger := mainLog.Load().With().Str("mode", "cd").Logger() - // Performs self-uninstallation if the ControlD device does not exist. - var uer *controld.ErrorResponse - if errors.As(pf.err, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode { - _ = uninstallInvalidCdUID(p, cdLogger, false) - } - if rejection, ok := permanentAPIRejection(pf.err); ok { - // The API answered and rejected this request permanently. Restarting - // cannot change that answer, so exit cleanly rather than through Fatal: - // an abnormal exit spends one of the service manager's restart actions, - // and on Windows those are what bring enforcement back after a real - // crash. Burning that budget on a config problem also buries the API's - // reason under repeated start failures. - cdLogger.Error().Err(pf.err).Int("status", rejection.StatusCode).Msg("failed to fetch resolver config, the API rejected this configuration") - notifyExitToLogServer() - return - } - notifyExitToLogServer() - // Everything else - a denied socket, an unreachable API, a proxy in the way, - // an API that is having a bad day - is a condition a later start may not hit, - // so keep the abnormal exit and let the service manager's recovery policy - // retry. - cdLogger.Fatal().Err(pf.err).Msg("failed to fetch resolver config") + handleAPIPreflightFailure(p, pf.err, notifyExitToLogServer) + return default: p.mu.Lock() p.rc = pf.rc @@ -374,6 +353,10 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { updated := updateListenerConfig(&cfg, notifyExitToLogServer) + // Bootstrap and listener binding both succeeded, so an earlier run's + // recorded failure no longer describes this install. + clearProvisionResult() + if cdUID != "" { processLogAndCacheFlags(v, &cfg) } @@ -740,6 +723,76 @@ func permanentAPIRejection(err error) (*controld.ErrorResponse, bool) { return uer, true } +// apiFailureCode maps a bootstrap preflight error to its provisioning code. +// A deleted device gets its own code because it triggers self-uninstall; +// other permanent rejections are generic; anything else counts as +// reachability trouble worth retrying. +func apiFailureCode(err error) (provisionFailureCode, bool) { + if err == nil { + return "", false + } + var uer *controld.ErrorResponse + if errors.As(err, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode { + return provisionCodeAPIDeviceInvalid, true + } + if _, ok := permanentAPIRejection(err); ok { + return provisionCodeAPIRejected, true + } + return provisionCodeAPIUnreachable, true +} + +// apiRejectionSummary reports the HTTP status only. The API's raw error body +// can echo back the value the caller sent, so it stays out of the artifact. +func apiRejectionSummary(statusCode int) string { + return fmt.Sprintf("ControlD API rejected this configuration (HTTP status %d)", statusCode) +} + +// provisionSecrets lists every secret-bearing value to strip from provisioning +// artifacts, including both parts of a composite "/" --cd +// value, which the API may echo back separately. +func provisionSecrets() []string { + uid, clientID := controld.ParseRawUID(cdUID) + return []string{cdUID, cdOrg, uid, clientID} +} + +// uninstallInvalidCdUIDFn is a var so tests can observe the self-uninstall +// without driving the OS service manager. +var uninstallInvalidCdUIDFn = uninstallInvalidCdUID + +// handleAPIPreflightFailure reports a failed resolver-config fetch. A deleted +// device self-uninstalls; it and any other permanent rejection return cleanly +// so a config problem cannot burn the service manager's restart budget (on +// Windows those restarts are what bring enforcement back after a real crash). +// Anything else exits nonzero through failProvision so the manager retries. +func handleAPIPreflightFailure(p *prog, err error, notify func()) { + cdLogger := mainLog.Load().With().Str("mode", "cd").Logger() + code, _ := apiFailureCode(err) + var uer *controld.ErrorResponse + if errors.As(err, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode { + r := newProvisionResult(code, apiRejectionSummary(uer.StatusCode), nil, provisionSecrets()...) + if werr := writeProvisionResult(r); werr != nil { + cdLogger.Warn().Err(werr).Msg("could not persist provision result") + } + _ = uninstallInvalidCdUIDFn(p, cdLogger, false) + cdLogger.Error().Err(err).Int("status", uer.StatusCode).Msg("failed to fetch resolver config, the device no longer exists") + cdLogger.Error().Msg(r.failureLine()) + notify() + return + } + if rejection, ok := permanentAPIRejection(err); ok { + r := newProvisionResult(code, apiRejectionSummary(rejection.StatusCode), nil, provisionSecrets()...) + if werr := writeProvisionResult(r); werr != nil { + cdLogger.Warn().Err(werr).Msg("could not persist provision result") + } + cdLogger.Error().Err(err).Int("status", rejection.StatusCode).Msg("failed to fetch resolver config, the API rejected this configuration") + cdLogger.Error().Msg(r.failureLine()) + notify() + return + } + cdLogger.Error().Err(err).Msg("failed to fetch resolver config") + failProvision(newProvisionResult(code, fmt.Sprintf("failed to fetch resolver config: %v", err), nil, provisionSecrets()...), notify) +} + // processCDFlagsFn is the API fetch, indirected so the lifetime binding around it can be // tested without reaching the network. var processCDFlagsFn = processCDFlags @@ -1424,16 +1477,27 @@ func tryUpdateListenerConfigIntercept(cfg *ctrld.Config, notifyFunc func(), fata } } + // bindAttempts feeds the provisioning result detail. newProvisionResult + // caps it, so it grows freely here. + var bindAttempts []provisionBindAttempt + recordBindAttempt := func(addr, proto string, err error) { + if err != nil { + bindAttempts = append(bindAttempts, provisionBindAttempt{Addr: addr, Proto: proto, OSError: err.Error()}) + } + } + tryListen := func(ip string, port int) bool { addr := net.JoinHostPort(ip, strconv.Itoa(port)) udpLn, udpErr := net.ListenPacket("udp", addr) if udpLn != nil { udpLn.Close() } + recordBindAttempt(addr, "udp", udpErr) tcpLn, tcpErr := net.Listen("tcp", addr) if tcpLn != nil { tcpLn.Close() } + recordBindAttempt(addr, "tcp", tcpErr) return udpErr == nil && tcpErr == nil } @@ -1448,8 +1512,10 @@ func tryUpdateListenerConfigIntercept(cfg *ctrld.Config, notifyFunc func(), fata if hasExplicitConfig { // User specified explicit address — don't guess, just fail if fatal { - notifyFunc() - mainLog.Load().Fatal().Msgf("DNS intercept: cannot listen on configured address %s", addr) + msg := fmt.Sprintf("DNS intercept: cannot listen on configured address %s", addr) + mainLog.Load().Error().Msg(msg) + failProvision(newProvisionResult(provisionCodeListenerAddrUnavail, msg, bindAttempts, provisionSecrets()...), notifyFunc) + return updated, false } return updated, false } @@ -1463,8 +1529,10 @@ func tryUpdateListenerConfigIntercept(cfg *ctrld.Config, notifyFunc func(), fata } if fatal { - notifyFunc() - mainLog.Load().Fatal().Msg("DNS intercept: cannot bind 127.0.0.1:53 or 127.0.0.1:5354") + const msg = "DNS intercept: cannot bind 127.0.0.1:53 or 127.0.0.1:5354" + mainLog.Load().Error().Msg(msg) + failProvision(newProvisionResult(provisionCodeListenerBindFailed, msg, bindAttempts, provisionSecrets()...), notifyFunc) + return updated, false } return updated, false } @@ -1566,6 +1634,15 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti _ = closer.Close() } }() + // bindAttempts feeds the provisioning result detail. newProvisionResult + // caps it, so it grows freely here. + var bindAttempts []provisionBindAttempt + recordBindAttempt := func(addr, proto string, err error) { + if err != nil { + bindAttempts = append(bindAttempts, provisionBindAttempt{Addr: addr, Proto: proto, OSError: err.Error()}) + } + } + // tryListen attempts to listen on given udp and tcp address. // Created listeners will be kept in listeners slice above, and close // before function finished. @@ -1574,16 +1651,21 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti if udpLn != nil { closers = append(closers, udpLn) } + recordBindAttempt(addr, "udp", udpErr) tcpLn, tcpErr := net.Listen("tcp", addr) if tcpLn != nil { closers = append(closers, tcpLn) } + recordBindAttempt(addr, "tcp", tcpErr) return errors.Join(udpErr, tcpErr) } + listenerMsg := func(listenerNum int, format string, v ...any) string { + return fmt.Sprintf("listener.%d %s", listenerNum, fmt.Sprintf(format, v...)) + } logMsg := func(e *zerolog.Event, listenerNum int, format string, v ...any) { e.MsgFunc(func() string { - return fmt.Sprintf("listener.%d %s", listenerNum, fmt.Sprintf(format, v...)) + return listenerMsg(listenerNum, format, v...) }) } @@ -1635,8 +1717,10 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti maxAttempts := 10 for { if attempts == maxAttempts { - notifyFunc() - logMsg(mainLog.Load().Fatal(), n, "could not find available listen ip and port") + logMsg(mainLog.Load().Error(), n, "could not find available listen ip and port") + msg := listenerMsg(n, "could not find available listen ip and port") + failProvision(newProvisionResult(provisionCodeListenerBindFailed, msg, bindAttempts, provisionSecrets()...), notifyFunc) + return updated, false } addr := net.JoinHostPort(listener.IP, strconv.Itoa(listener.Port)) err := tryListen(addr) @@ -1648,8 +1732,10 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti if !check.IP && !check.Port { if fatal { - notifyFunc() - logMsg(mainLog.Load().Fatal(), n, "failed to listen: %v", err) + logMsg(mainLog.Load().Error(), n, "failed to listen: %v", err) + msg := listenerMsg(n, "failed to listen: %v", err) + failProvision(newProvisionResult(provisionCodeListenerAddrUnavail, msg, bindAttempts, provisionSecrets()...), notifyFunc) + return updated, false } ok = false break @@ -1716,8 +1802,11 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti } if listener.IP == oldIP && listener.Port == oldPort { if fatal { - notifyFunc() - logMsg(mainLog.Load().Fatal(), n, "could not listen on %s: %v", net.JoinHostPort(listener.IP, strconv.Itoa(listener.Port)), err) + triedAddr := net.JoinHostPort(listener.IP, strconv.Itoa(listener.Port)) + logMsg(mainLog.Load().Error(), n, "could not listen on %s: %v", triedAddr, err) + msg := listenerMsg(n, "could not listen on %s: %v", triedAddr, err) + failProvision(newProvisionResult(provisionCodeListenerBindFailed, msg, bindAttempts, provisionSecrets()...), notifyFunc) + return updated, false } ok = false break @@ -1755,8 +1844,10 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti } } if !found { - notifyFunc() - logMsg(mainLog.Load().Fatal(), n, "could not use %q as DNS nameserver with systemd resolved", listener.IP) + logMsg(mainLog.Load().Error(), n, "could not use %q as DNS nameserver with systemd resolved", listener.IP) + msg := listenerMsg(n, "could not use %q as DNS nameserver with systemd resolved", listener.IP) + failProvision(newProvisionResult(provisionCodeListenerAddrUnavail, msg, bindAttempts, provisionSecrets()...), notifyFunc) + return updated, false } } } @@ -1804,13 +1895,23 @@ func cdUIDFromProvToken() string { Metadata: ctrld.SystemMetadata(context.Background()), } // Process provision token if provided. - resolverConfig, err := controld.FetchResolverUID(context.Background(), req, rootCmd.Version, cdDev) + resolverConfig, err := fetchResolverUIDFn(context.Background(), req, rootCmd.Version, cdDev) if err != nil { - mainLog.Load().Fatal().Err(err).Msgf("failed to fetch resolver uid with provision token: %s", redactToken(cdOrg)) + // The token exchange is the first API call of an org/MDM install, so + // its failure must carry a code like every other bootstrap failure. + code, _ := apiFailureCode(err) + mainLog.Load().Error().Msgf("failed to fetch resolver uid with provision token: %s: %s", + redactToken(cdOrg), redactSecrets(err.Error(), provisionSecrets()...)) + failProvision(newProvisionResult(code, fmt.Sprintf("provision token exchange failed: %v", err), nil, provisionSecrets()...), nil) + return "" } return resolverConfig.UID } +// fetchResolverUIDFn is a var so tests can drive token-exchange failures +// without reaching the network. +var fetchResolverUIDFn = controld.FetchResolverUID + // removeOrgFlagsFromArgs removes organization flags from command line arguments. // The flags are: // diff --git a/cmd/cli/cli_provision_test.go b/cmd/cli/cli_provision_test.go new file mode 100644 index 0000000..34628da --- /dev/null +++ b/cmd/cli/cli_provision_test.go @@ -0,0 +1,329 @@ +package cli + +import ( + "context" + "fmt" + "net" + "net/http" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + + "github.com/rs/zerolog" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// TestApiFailureCode covers the preflight-error mapping: a deleted device +// gets its own code (it drives self-uninstall), other permanent rejections +// are generic, anything else is retryable reachability trouble. +func TestApiFailureCode(t *testing.T) { + rejection := func(status, code int) error { + e := &controld.ErrorResponse{StatusCode: status} + e.ErrorField.Code = code + e.ErrorField.Message = "api said no" + return e + } + + tests := []struct { + name string + err error + wantCode provisionFailureCode + wantOk bool + }{ + {name: "nil error", err: nil, wantCode: "", wantOk: false}, + { + name: "deleted device maps to device invalid", + err: rejection(http.StatusNotFound, controld.InvalidConfigCode), + wantCode: provisionCodeAPIDeviceInvalid, + wantOk: true, + }, + { + name: "revoked credentials map to rejected", + err: rejection(http.StatusUnauthorized, 0), + wantCode: provisionCodeAPIRejected, + wantOk: true, + }, + { + name: "server error maps to unreachable", + err: rejection(http.StatusBadGateway, 0), + wantCode: provisionCodeAPIUnreachable, + wantOk: true, + }, + { + name: "network failure maps to unreachable", + err: retryableNetworkErr(), + wantCode: provisionCodeAPIUnreachable, + wantOk: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + code, ok := apiFailureCode(tc.err) + if ok != tc.wantOk { + t.Fatalf("apiFailureCode() ok = %v, want %v", ok, tc.wantOk) + } + if code != tc.wantCode { + t.Errorf("apiFailureCode() code = %s, want %s", code, tc.wantCode) + } + }) + } +} + +func stubProvisionGlobals(t *testing.T) (exitCode *int, notified *bool) { + t.Helper() + oldCdUID, oldCdOrg := cdUID, cdOrg + oldExit, oldUninstall := provisionExit, uninstallInvalidCdUIDFn + t.Cleanup(func() { + cdUID, cdOrg = oldCdUID, oldCdOrg + provisionExit, uninstallInvalidCdUIDFn = oldExit, oldUninstall + }) + overrideProvisionResultPath(t) + code := -1 + provisionExit = func(c int) { code = c } + n := false + return &code, &n +} + +func TestHandleAPIPreflightFailure(t *testing.T) { + deviceInvalid := func() error { + e := &controld.ErrorResponse{StatusCode: http.StatusNotFound} + e.ErrorField.Code = controld.InvalidConfigCode + e.ErrorField.Message = "device does not exist" + return e + } + rejected := func() error { + e := &controld.ErrorResponse{StatusCode: http.StatusUnauthorized} + e.ErrorField.Message = "bad token" + return e + } + + t.Run("permanent rejection returns cleanly", func(t *testing.T) { + exitCode, notified := stubProvisionGlobals(t) + handleAPIPreflightFailure(&prog{}, rejected(), func() { *notified = true }) + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want a clean return", *exitCode) + } + if !*notified { + t.Error("notify not called") + } + r, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if r.Code != string(provisionCodeAPIRejected) { + t.Errorf("code = %q, want API_REJECTED", r.Code) + } + }) + + t.Run("deleted device self-uninstalls and returns cleanly", func(t *testing.T) { + exitCode, notified := stubProvisionGlobals(t) + uninstalled := false + uninstallInvalidCdUIDFn = func(_ *prog, _ zerolog.Logger, _ bool) bool { + uninstalled = true + return true + } + handleAPIPreflightFailure(&prog{}, deviceInvalid(), func() { *notified = true }) + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want a clean return", *exitCode) + } + if !uninstalled { + t.Error("self-uninstall not attempted") + } + if !*notified { + t.Error("notify not called") + } + r, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if r.Code != string(provisionCodeAPIDeviceInvalid) { + t.Errorf("code = %q, want API_DEVICE_INVALID", r.Code) + } + }) + + t.Run("unreachable exits nonzero", func(t *testing.T) { + exitCode, notified := stubProvisionGlobals(t) + handleAPIPreflightFailure(&prog{}, retryableNetworkErr(), func() { *notified = true }) + if *exitCode != provisionExitCodeForCode[provisionCodeAPIUnreachable] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeAPIUnreachable]) + } + if !*notified { + t.Error("notify not called") + } + r, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if r.Code != string(provisionCodeAPIUnreachable) { + t.Errorf("code = %q, want API_UNREACHABLE", r.Code) + } + }) + + t.Run("bare uid from a composite --cd value is redacted", func(t *testing.T) { + _, _ = stubProvisionGlobals(t) + cdUID = "deviceabc/clientxyz" + cdOrg = "" + err := fmt.Errorf("failed: api says deviceabc is unknown") + handleAPIPreflightFailure(&prog{}, err, func() {}) + r, rerr := readProvisionResult() + if rerr != nil { + t.Fatal(rerr) + } + if strings.Contains(r.Message, "deviceabc") { + t.Errorf("bare uid leaked into message: %q", r.Message) + } + }) +} + +func TestCdUIDFromProvTokenFailureEmitsCode(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldFetch, oldHostname := fetchResolverUIDFn, customHostname + t.Cleanup(func() { fetchResolverUIDFn, customHostname = oldFetch, oldHostname }) + cdUID = "" + cdOrg = "org-secret-token-123" + customHostname = "" + + rejected := &controld.ErrorResponse{StatusCode: http.StatusUnauthorized} + rejected.ErrorField.Message = "bad provision token org-secret-token-123" + fetchResolverUIDFn = func(context.Context, *controld.UtilityOrgRequest, string, bool) (*controld.ResolverConfig, error) { + return nil, rejected + } + + if got := cdUIDFromProvToken(); got != "" { + t.Errorf("cdUIDFromProvToken() = %q, want empty on failure", got) + } + if *exitCode != provisionExitCodeForCode[provisionCodeAPIRejected] { + t.Errorf("exit = %d, want API_REJECTED exit %d", *exitCode, provisionExitCodeForCode[provisionCodeAPIRejected]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeAPIRejected) { + t.Errorf("code = %q, want API_REJECTED", r.Code) + } + if strings.Contains(r.Message, cdOrg) { + t.Errorf("token leaked into result message: %q", r.Message) + } +} + +// Regression test: an explicit ip:port that fails to bind used to die with a +// bare fatal log automation could not tell apart from any other crash. It +// must report a stable code through the provisioning result instead. +func TestTryUpdateListenerConfigConfiguredAddrUnavailable(t *testing.T) { + // Occupy one localhost port on both udp and tcp, and hold both for the + // whole test so ctrld's own bind attempt is guaranteed to fail. + udpConn, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("could not reserve a udp port: %v", err) + } + defer udpConn.Close() + + host, portStr, err := net.SplitHostPort(udpConn.LocalAddr().String()) + if err != nil { + t.Fatalf("could not parse reserved address: %v", err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + t.Fatalf("could not parse reserved port: %v", err) + } + + tcpLn, err := net.Listen("tcp", net.JoinHostPort(host, portStr)) + if err != nil { + t.Fatalf("could not reserve the same port on tcp: %v", err) + } + defer tcpLn.Close() + + oldCdUID, oldCdOrg, oldNextdns, oldIntercept := cdUID, cdOrg, nextdns, interceptMode + oldPath, oldExit := provisionResultPath, provisionExit + t.Cleanup(func() { + cdUID, cdOrg, nextdns, interceptMode = oldCdUID, oldCdOrg, oldNextdns, oldIntercept + provisionResultPath, provisionExit = oldPath, oldExit + }) + // Non-cd, non-nextdns mode with an explicit ip:port: no fallback checks, + // the path that used to reach the fatal exit directly. + cdUID = "" + cdOrg = "" + nextdns = "" + interceptMode = "" + + tmpDir := t.TempDir() + provisionResultPath = func() string { return filepath.Join(tmpDir, "provision_result.json") } + + var exitCode int + var exited bool + provisionExit = func(code int) { exitCode = code; exited = true } + + cfg := &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{ + "0": {IP: host, Port: port}, + }, + } + + notified := false + _, ok := tryUpdateListenerConfig(cfg, nil, func() { notified = true }, true) + + if ok { + t.Error("tryUpdateListenerConfig ok = true, want false") + } + if !notified { + t.Error("expected notifyFunc to run before the recorded exit") + } + if !exited { + t.Fatal("expected provisionExit to be called") + } + if exitCode != 42 { + t.Errorf("exit code = %d, want 42 (LISTENER_CONFIGURED_ADDR_UNAVAILABLE)", exitCode) + } + + result, err := readProvisionResult() + if err != nil { + t.Fatalf("could not read provision result: %v", err) + } + if result.Code != string(provisionCodeListenerAddrUnavail) { + t.Errorf("result code = %s, want %s", result.Code, provisionCodeListenerAddrUnavail) + } + if result.Stage != string(provisionStageListener) { + t.Errorf("result stage = %s, want %s", result.Stage, provisionStageListener) + } + if result.ExitCode != 42 { + t.Errorf("result exit code = %d, want 42", result.ExitCode) + } + if result.Detail == nil || len(result.Detail.Attempts) == 0 { + t.Fatal("expected the occupied address to appear as a recorded bind attempt") + } + + occupiedAddr := net.JoinHostPort(host, portStr) + // Windows words WSAEADDRINUSE differently, so only require the canonical + // message on platforms that produce it. + requireInUseText := runtime.GOOS != "windows" + var sawUDP, sawTCP bool + for _, a := range result.Detail.Attempts { + if a.Addr != occupiedAddr || a.OSError == "" { + continue + } + if requireInUseText && !strings.Contains(strings.ToLower(a.OSError), "address already in use") { + continue + } + switch a.Proto { + case "udp": + sawUDP = true + case "tcp": + sawTCP = true + } + } + if !sawUDP { + t.Error("expected a udp attempt on the occupied address with a bind error") + } + if !sawTCP { + t.Error("expected a tcp attempt on the occupied address with a bind error") + } +} + +// The exhaustion path (exit 41) is not covered: forcing every fallback, +// including a freshly randomized ip/port, to fail has no deterministic seam, +// so a test would race whatever ports are free on the host. diff --git a/cmd/cli/commands.go b/cmd/cli/commands.go index 5b1a968..0cc6542 100644 --- a/cmd/cli/commands.go +++ b/cmd/cli/commands.go @@ -283,6 +283,46 @@ func initRunCmd() *cobra.Command { return runCmd } +// serviceStageFailureCode maps an aborted service-manager task to its +// provisioning code. Other abortOnError tasks (like config validation) keep +// their own error paths. +func serviceStageFailureCode(taskName string) (provisionFailureCode, bool) { + switch taskName { + case "Install": + return provisionCodeServiceInstall, true + case "Start": + return provisionCodeServiceStartFailed, true + default: + return "", false + } +} + +// serviceTaskErrorSummary describes which service-manager task failed and why, +// for use as a provisioning result message. +func serviceTaskErrorSummary(taskName string, err error) string { + return fmt.Sprintf("%s failed: %v", taskName, err) +} + +// resultStalenessTolerance absorbs clock granularity between "ctrld start" +// recording its start time and the daemon writing its result file. +const resultStalenessTolerance = 2 * time.Second + +// reportStartFailure reports why "ctrld start" failed after install/start +// looked fine. A result file the daemon wrote during this attempt names the +// failure better than a generic self-check code, so it wins. +func reportStartFailure(startedAt time.Time, fallbackMsg string) { + if r, err := readProvisionResult(); err == nil && provisionResultTrusted(r) { + if ts, err := time.Parse(time.RFC3339, r.Timestamp); err == nil { + if !ts.Before(startedAt.Add(-resultStalenessTolerance)) { + mainLog.Load().Error().Msg(r.failureLine()) + provisionExit(r.ExitCode) + return + } + } + } + failProvision(newProvisionResult(provisionCodeServiceSelfCheck, fallbackMsg, nil, provisionSecrets()...), nil) +} + func initStartCmd() *cobra.Command { startCmd := &cobra.Command{ PreRun: func(cmd *cobra.Command, args []string) { @@ -524,23 +564,50 @@ NOTE: running "ctrld start" without any arguments will start already installed c {s.Start, true, "Start"}, {noticeWritingControlDConfig, false, "Notice writing ControlD config"}, } + // Any result found later must come from this attempt, not a stale run. + clearProvisionResult() + startAttemptAt := time.Now() mainLog.Load().Notice().Msg("Starting existing ctrld service") - if doTasks(tasks) { - mainLog.Load().Notice().Msg("Service started") - sockDir, err := socketDir() - if err != nil { - mainLog.Load().Warn().Err(err).Msg("Failed to get socket directory") - os.Exit(1) + failedTask, taskErr := doTasksE(tasks) + if taskErr != nil { + if code, ok := serviceStageFailureCode(failedTask); ok { + failProvision(newProvisionResult(code, serviceTaskErrorSummary(failedTask, taskErr), nil, provisionSecrets()...), nil) + return } - reportSetDnsOk(sockDir) - // Verify service registration after successful start. - if err := verifyServiceRegistration(); err != nil { - mainLog.Load().Warn().Err(err).Msg("Service registry verification failed") - } - } else { - mainLog.Load().Error().Err(err).Msg("Failed to start existing ctrld service") os.Exit(1) } + sockDir, err := socketDir() + if err != nil { + mainLog.Load().Warn().Err(err).Msg("Failed to get socket directory") + os.Exit(1) + } + + // The daemon can start and still fail provisioning (for example a + // listener bind conflict). Self-check like a fresh install so this + // path reports the daemon's failure code instead of a false + // "Service started" — but never uninstall an existing service. + time.Sleep(1 * time.Second) + ok, status, err := selfCheckStatus(ctx, s, sockDir) + if !ok || status != service.StatusRunning { + fallbackMsg := "ctrld service did not pass its post-start self-check" + if err != nil { + fallbackMsg = fmt.Sprintf("An error occurred while performing test query: %s", err) + mainLog.Load().Error().Msg(fallbackMsg) + } + if status == service.StatusRunning && err == nil { + fallbackMsg = "ctrld service was running, but a DNS query could not be sent to its listener; check firewall rules blocking/intercepting/redirecting DNS queries" + mainLog.Load().Error().Msg(fallbackMsg) + } + reportStartFailure(startAttemptAt, fallbackMsg) + return + } + mainLog.Load().Notice().Msg("Service started") + clearProvisionResult() + reportSetDnsOk(sockDir) + // Verify service registration after successful start. + if err := verifyServiceRegistration(); err != nil { + mainLog.Load().Warn().Err(err).Msg("Service registry verification failed") + } return } @@ -605,7 +672,7 @@ NOTE: running "ctrld start" without any arguments will start already installed c }) return nil }, false, "Save current DNS"}, - {s.Install, false, "Install"}, + {s.Install, true, "Install"}, {func() error { return ConfigureWindowsServiceFailureActions(ctrldServiceName) }, false, "Configure Windows service failure actions"}, @@ -614,59 +681,77 @@ NOTE: running "ctrld start" without any arguments will start already installed c // generated after s.Start, so we notice users here for consistent with nextdns mode. {noticeWritingControlDConfig, false, "Notice writing ControlD config"}, } + // Any result found later must come from this attempt, not a stale run. + clearProvisionResult() + startAttemptAt := time.Now() mainLog.Load().Notice().Msg("Starting service") - if doTasks(tasks) { - if err := p.router.Install(sc); err != nil { - mainLog.Load().Warn().Err(err).Msg("post installation failed, please check system/service log for details error") + failedTask, taskErr := doTasksE(tasks) + if taskErr != nil { + if code, ok := serviceStageFailureCode(failedTask); ok { + failProvision(newProvisionResult(code, serviceTaskErrorSummary(failedTask, taskErr), nil, provisionSecrets()...), nil) return } + // Not a service-stage task. doTasksE already logged the cause; exit + // non-zero instead of the old silent fall-through that exited 0. + os.Exit(1) + return + } - // add a small delay to ensure the service is started and did not crash - time.Sleep(1 * time.Second) + if err := p.router.Install(sc); err != nil { + mainLog.Load().Warn().Err(err).Msg("post installation failed, please check system/service log for details error") + return + } - ok, status, err := selfCheckStatus(ctx, s, sockDir) - switch { - case ok && status == service.StatusRunning: - mainLog.Load().Notice().Msg("Service started") - default: - marker := bytes.Repeat([]byte("="), 32) - // If ctrld service is not running, emitting log obtained from ctrld process. - if status != service.StatusRunning || ctx.Err() != nil { - mainLog.Load().Error().Msg("ctrld service may not have started due to an error or misconfiguration, service log:") - _, _ = mainLog.Load().Write(marker) - haveLog := false - for msg := range runCmdLogCh { - _, _ = mainLog.Load().Write([]byte(strings.ReplaceAll(msg, msgExit, ""))) - haveLog = true - } - // If we're unable to get log from "ctrld run", notice users about it. - if !haveLog { - mainLog.Load().Write([]byte(`"`)) - } - } - // Report any error if occurred. - if err != nil { - _, _ = mainLog.Load().Write(marker) - msg := fmt.Sprintf("An error occurred while performing test query: %s", err) - mainLog.Load().Write([]byte(msg)) - } - // If ctrld service is running but selfCheckStatus failed, it could be related - // to user's system firewall configuration, notice users about it. - if status == service.StatusRunning && err == nil { - _, _ = mainLog.Load().Write(marker) - mainLog.Load().Write([]byte(`ctrld service was running, but a DNS query could not be sent to its listener`)) - mainLog.Load().Write([]byte(`Please check your system firewall if it is configured to block/intercept/redirect DNS queries`)) - } + // add a small delay to ensure the service is started and did not crash + time.Sleep(1 * time.Second) + ok, status, err := selfCheckStatus(ctx, s, sockDir) + switch { + case ok && status == service.StatusRunning: + mainLog.Load().Notice().Msg("Service started") + clearProvisionResult() + default: + marker := bytes.Repeat([]byte("="), 32) + fallbackMsg := "ctrld service did not pass its post-start self-check" + // If ctrld service is not running, emitting log obtained from ctrld process. + if status != service.StatusRunning || ctx.Err() != nil { + mainLog.Load().Error().Msg("ctrld service may not have started due to an error or misconfiguration, service log:") _, _ = mainLog.Load().Write(marker) - uninstall(p, s) - os.Exit(1) + haveLog := false + for msg := range runCmdLogCh { + _, _ = mainLog.Load().Write([]byte(strings.ReplaceAll(msg, msgExit, ""))) + haveLog = true + } + // If we're unable to get log from "ctrld run", notice users about it. + if !haveLog { + mainLog.Load().Write([]byte(`"`)) + } } - reportSetDnsOk(sockDir) - // Verify service registration after successful start. - if err := verifyServiceRegistration(); err != nil { - mainLog.Load().Warn().Err(err).Msg("Service registry verification failed") + // Report any error if occurred. + if err != nil { + _, _ = mainLog.Load().Write(marker) + msg := fmt.Sprintf("An error occurred while performing test query: %s", err) + mainLog.Load().Write([]byte(msg)) + fallbackMsg = msg } + // If ctrld service is running but selfCheckStatus failed, it could be related + // to user's system firewall configuration, notice users about it. + if status == service.StatusRunning && err == nil { + _, _ = mainLog.Load().Write(marker) + mainLog.Load().Write([]byte(`ctrld service was running, but a DNS query could not be sent to its listener`)) + mainLog.Load().Write([]byte(`Please check your system firewall if it is configured to block/intercept/redirect DNS queries`)) + fallbackMsg = "ctrld service was running, but a DNS query could not be sent to its listener; check firewall rules blocking/intercepting/redirecting DNS queries" + } + + _, _ = mainLog.Load().Write(marker) + uninstall(p, s) + reportStartFailure(startAttemptAt, fallbackMsg) + return + } + reportSetDnsOk(sockDir) + // Verify service registration after successful start. + if err := verifyServiceRegistration(); err != nil { + mainLog.Load().Warn().Err(err).Msg("Service registry verification failed") } }, } diff --git a/cmd/cli/commands_start_test.go b/cmd/cli/commands_start_test.go new file mode 100644 index 0000000..882fac2 --- /dev/null +++ b/cmd/cli/commands_start_test.go @@ -0,0 +1,122 @@ +package cli + +import ( + "testing" + "time" +) + +func TestServiceStageFailureCode(t *testing.T) { + tests := []struct { + taskName string + wantCode provisionFailureCode + wantOK bool + }{ + {"Install", provisionCodeServiceInstall, true}, + {"Start", provisionCodeServiceStartFailed, true}, + {"Checking config", "", false}, + {"", "", false}, + } + for _, tc := range tests { + code, ok := serviceStageFailureCode(tc.taskName) + if code != tc.wantCode || ok != tc.wantOK { + t.Errorf("serviceStageFailureCode(%q) = (%q, %v), want (%q, %v)", tc.taskName, code, ok, tc.wantCode, tc.wantOK) + } + } +} + +func stubProvisionExit(t *testing.T) *int { + t.Helper() + exitCode := -1 + old := provisionExit + provisionExit = func(code int) { exitCode = code } + t.Cleanup(func() { provisionExit = old }) + return &exitCode +} + +func TestReportStartFailureUsesFreshDaemonResult(t *testing.T) { + overrideProvisionResultPath(t) + exitCode := stubProvisionExit(t) + + startedAt := time.Now() + daemonResult := newProvisionResult(provisionCodeAPIUnreachable, "daemon could not reach the API", nil) + if err := writeProvisionResult(daemonResult); err != nil { + t.Fatal(err) + } + + reportStartFailure(startedAt, "generic self-check failure") + + if *exitCode != provisionExitCodeForCode[provisionCodeAPIUnreachable] { + t.Errorf("exit code = %d, want the daemon's own exit code %d", *exitCode, provisionExitCodeForCode[provisionCodeAPIUnreachable]) + } + out, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if out.Code != string(provisionCodeAPIUnreachable) { + t.Errorf("persisted code = %q, want the daemon's own code untouched", out.Code) + } +} + +func TestReportStartFailureFallsBackOnStaleDaemonResult(t *testing.T) { + overrideProvisionResultPath(t) + exitCode := stubProvisionExit(t) + + stale := newProvisionResult(provisionCodeAPIUnreachable, "an old failure", nil) + stale.Timestamp = time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339) + if err := writeProvisionResult(stale); err != nil { + t.Fatal(err) + } + + startedAt := time.Now() + reportStartFailure(startedAt, "test query failed: timeout") + + if *exitCode != provisionExitCodeForCode[provisionCodeServiceSelfCheck] { + t.Errorf("exit code = %d, want SERVICE_SELFCHECK_FAILED exit %d", *exitCode, provisionExitCodeForCode[provisionCodeServiceSelfCheck]) + } + out, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if out.Code != string(provisionCodeServiceSelfCheck) { + t.Errorf("persisted code = %q, want %q", out.Code, provisionCodeServiceSelfCheck) + } + if out.Message != "test query failed: timeout" { + t.Errorf("persisted message = %q, want the fallback message", out.Message) + } +} + +func TestReportStartFailureRejectsUntrustedFile(t *testing.T) { + overrideProvisionResultPath(t) + exitCode := stubProvisionExit(t) + + planted := newProvisionResult(provisionCodeAPIUnreachable, "planted", nil) + planted.Code = "FAKE_CODE" + planted.ExitCode = 99 + if err := writeProvisionResult(planted); err != nil { + t.Fatal(err) + } + + reportStartFailure(time.Now().Add(-time.Minute), "self-check failed") + + if *exitCode != provisionExitCodeForCode[provisionCodeServiceSelfCheck] { + t.Errorf("exit = %d, want the fallback %d, never the planted 99", *exitCode, provisionExitCodeForCode[provisionCodeServiceSelfCheck]) + } +} + +func TestReportStartFailureFallsBackWhenResultFileMissing(t *testing.T) { + overrideProvisionResultPath(t) + exitCode := stubProvisionExit(t) + + reportStartFailure(time.Now(), "firewall hint") + + if *exitCode != provisionExitCodeForCode[provisionCodeServiceSelfCheck] { + t.Errorf("exit code = %d, want SERVICE_SELFCHECK_FAILED exit %d", *exitCode, provisionExitCodeForCode[provisionCodeServiceSelfCheck]) + } + out, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if out.Message != "firewall hint" { + t.Errorf("persisted message = %q, want the fallback message", out.Message) + } +} diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go index e828103..a894886 100644 --- a/cmd/cli/dns_intercept_darwin.go +++ b/cmd/cli/dns_intercept_darwin.go @@ -1683,12 +1683,17 @@ func (p *prog) dnsInterceptIgnoredChangeReconcileDue(now time.Time) bool { } func (p *prog) scheduleDNSAfterVPNSettleRefresh(reason string, delay time.Duration) { - time.AfterFunc(delay, func() { + timer := time.AfterFunc(delay, func() { if p.dnsInterceptState == nil { return } p.refreshDNSAfterVPNSettle(reason) }) + // Track the timer like the other delayed rechecks, so intercept teardown + // (and test cleanup) can stop it instead of letting it fire afterwards. + p.pfDelayedRecheckMu.Lock() + p.pfDelayedRecheckTimers = append(p.pfDelayedRecheckTimers, timer) + p.pfDelayedRecheckMu.Unlock() } func (p *prog) pfExecBackoffActive() bool { diff --git a/cmd/cli/provision_result.go b/cmd/cli/provision_result.go new file mode 100644 index 0000000..bd513be --- /dev/null +++ b/cmd/cli/provision_result.go @@ -0,0 +1,247 @@ +package cli + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + "unicode/utf8" +) + +// A terminal provisioning failure reports the same stable code on three +// surfaces: a persisted result file, one fixed-format output line, and a +// stage-scoped process exit code. docs/provisioning-failure-codes.md maps +// each code to its scenario and must stay in sync with the constants below. +// Codes are append-only once released; renaming or reusing one breaks the +// support contract. + +type provisionStage string + +const ( + provisionStageBootstrap provisionStage = "bootstrap" + provisionStageListener provisionStage = "listener" + provisionStageService provisionStage = "service" +) + +type provisionFailureCode string + +const ( + provisionCodeAPIUnreachable provisionFailureCode = "API_UNREACHABLE" + provisionCodeAPIRejected provisionFailureCode = "API_REJECTED" + provisionCodeAPIDeviceInvalid provisionFailureCode = "API_DEVICE_INVALID" + provisionCodeListenerBindFailed provisionFailureCode = "LISTENER_BIND_FAILED" + provisionCodeListenerAddrUnavail provisionFailureCode = "LISTENER_CONFIGURED_ADDR_UNAVAILABLE" + provisionCodeServiceInstall provisionFailureCode = "SERVICE_INSTALL_FAILED" + provisionCodeServiceStartFailed provisionFailureCode = "SERVICE_START_FAILED" + provisionCodeServiceSelfCheck provisionFailureCode = "SERVICE_SELFCHECK_FAILED" +) + +var allProvisionFailureCodes = []provisionFailureCode{ + provisionCodeAPIUnreachable, + provisionCodeAPIRejected, + provisionCodeAPIDeviceInvalid, + provisionCodeListenerBindFailed, + provisionCodeListenerAddrUnavail, + provisionCodeServiceInstall, + provisionCodeServiceStartFailed, + provisionCodeServiceSelfCheck, +} + +var provisionStageForCode = map[provisionFailureCode]provisionStage{ + provisionCodeAPIUnreachable: provisionStageBootstrap, + provisionCodeAPIRejected: provisionStageBootstrap, + provisionCodeAPIDeviceInvalid: provisionStageBootstrap, + provisionCodeListenerBindFailed: provisionStageListener, + provisionCodeListenerAddrUnavail: provisionStageListener, + provisionCodeServiceInstall: provisionStageService, + provisionCodeServiceStartFailed: provisionStageService, + provisionCodeServiceSelfCheck: provisionStageService, +} + +// Exit codes are grouped by stage (bootstrap 30-39, listener 40-49, service +// 50-59) so the exit code alone names the failed stage. 0-3 belong to +// "ctrld status" and 126 to the deactivation pin check; never reuse those. +var provisionExitCodeForCode = map[provisionFailureCode]int{ + provisionCodeAPIUnreachable: 30, + provisionCodeAPIRejected: 31, + provisionCodeAPIDeviceInvalid: 32, + provisionCodeListenerBindFailed: 41, + provisionCodeListenerAddrUnavail: 42, + provisionCodeServiceInstall: 51, + provisionCodeServiceStartFailed: 52, + provisionCodeServiceSelfCheck: 53, +} + +const ( + provisionResultFileName = "provision_result.json" + // Detail identifies a failure, it is not a log. Caps keep the artifact + // small and predictable. + maxProvisionBindAttempts = 12 + maxProvisionStringLen = 256 +) + +type provisionBindAttempt struct { + Addr string `json:"addr"` + Proto string `json:"proto"` + OSError string `json:"os_error"` +} + +type provisionDetail struct { + Attempts []provisionBindAttempt `json:"attempts,omitempty"` +} + +type provisionResult struct { + Version int `json:"version"` + Timestamp string `json:"timestamp"` + Stage string `json:"stage"` + Code string `json:"code"` + ExitCode int `json:"exit_code"` + Message string `json:"message"` + Detail *provisionDetail `json:"detail,omitempty"` +} + +// provisionResultPath is a var so tests can point it at a temp dir. +var provisionResultPath = func() string { + return absHomeDir(provisionResultFileName) +} + +// provisionExit is a var so tests can observe the exit code instead of dying. +var provisionExit = os.Exit + +// newProvisionResult builds a result with every field bounded and the given +// secrets stripped. The artifact reaches installer logs and support tickets, +// so callers pass every secret in scope (provision token, cd UID). +func newProvisionResult(code provisionFailureCode, message string, attempts []provisionBindAttempt, secrets ...string) *provisionResult { + sanitize := func(s string) string { + s = redactSecrets(s, secrets...) + if len(s) > maxProvisionStringLen { + // Cut on a rune boundary so a localized OS error does not end in + // a broken multi-byte sequence. + cut := maxProvisionStringLen + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + s = s[:cut] + } + return s + } + r := &provisionResult{ + Version: 1, + Timestamp: time.Now().UTC().Format(time.RFC3339), + Stage: string(provisionStageForCode[code]), + Code: string(code), + ExitCode: provisionExitCodeForCode[code], + Message: sanitize(message), + } + if len(attempts) > 0 { + if len(attempts) > maxProvisionBindAttempts { + attempts = attempts[:maxProvisionBindAttempts] + } + detail := &provisionDetail{Attempts: make([]provisionBindAttempt, 0, len(attempts))} + for _, a := range attempts { + detail.Attempts = append(detail.Attempts, provisionBindAttempt{ + Addr: sanitize(a.Addr), + Proto: sanitize(a.Proto), + OSError: sanitize(a.OSError), + }) + } + r.Detail = detail + } + return r +} + +// redactSecrets removes every non-empty secret from s. +func redactSecrets(s string, secrets ...string) string { + for _, secret := range secrets { + if secret == "" { + continue + } + s = strings.ReplaceAll(s, secret, "[redacted]") + } + return s +} + +// provisionResultTrusted rejects a result whose code, stage, or exit code is +// not part of the known contract, so a corrupt or planted file cannot drive +// what "ctrld start" logs and exits with. +func provisionResultTrusted(r *provisionResult) bool { + code := provisionFailureCode(r.Code) + stage, ok := provisionStageForCode[code] + if !ok { + return false + } + return r.Stage == string(stage) && r.ExitCode == provisionExitCodeForCode[code] +} + +func (r *provisionResult) failureLine() string { + return fmt.Sprintf("provisioning failed: stage=%s code=%s (exit %d)", r.Stage, r.Code, r.ExitCode) +} + +// writeProvisionResult persists the result atomically (temp file + rename in +// the same directory) so a reader never sees a partial file. +func writeProvisionResult(r *provisionResult) error { + path := provisionResultPath() + buf, err := json.MarshalIndent(r, "", " ") + if err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(path), provisionResultFileName+".tmp*") + if err != nil { + return err + } + tmpName := tmp.Name() + if _, err := tmp.Write(buf); err != nil { + _ = tmp.Close() + _ = os.Remove(tmpName) + return err + } + if err := tmp.Close(); err != nil { + _ = os.Remove(tmpName) + return err + } + if err := os.Chmod(tmpName, 0o600); err != nil { + _ = os.Remove(tmpName) + return err + } + if err := os.Rename(tmpName, path); err != nil { + _ = os.Remove(tmpName) + return err + } + return nil +} + +func readProvisionResult() (*provisionResult, error) { + buf, err := os.ReadFile(provisionResultPath()) + if err != nil { + return nil, err + } + r := &provisionResult{} + if err := json.Unmarshal(buf, r); err != nil { + return nil, err + } + return r, nil +} + +// clearProvisionResult removes a stale result once provisioning succeeds, so +// support never diagnoses a healthy install from an old failure. +func clearProvisionResult() { + if err := os.Remove(provisionResultPath()); err != nil && !os.IsNotExist(err) { + mainLog.Load().Debug().Err(err).Msg("could not remove provision result file") + } +} + +// failProvision persists the result, prints the identifier line, unblocks a +// waiting "ctrld start" via notify, then exits with the stage code. The write +// comes first so the file survives even if logging or notify misbehaves. +func failProvision(r *provisionResult, notify func()) { + if err := writeProvisionResult(r); err != nil { + mainLog.Load().Warn().Err(err).Msg("could not persist provision result") + } + mainLog.Load().Error().Msg(r.failureLine()) + if notify != nil { + notify() + } + provisionExit(r.ExitCode) +} diff --git a/cmd/cli/provision_result_test.go b/cmd/cli/provision_result_test.go new file mode 100644 index 0000000..3003904 --- /dev/null +++ b/cmd/cli/provision_result_test.go @@ -0,0 +1,278 @@ +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + "unicode/utf8" +) + +func overrideProvisionResultPath(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), provisionResultFileName) + old := provisionResultPath + provisionResultPath = func() string { return path } + t.Cleanup(func() { provisionResultPath = old }) + return path +} + +func TestProvisionCodesMapToOneStageAndInRangeExit(t *testing.T) { + stageRanges := map[provisionStage][2]int{ + provisionStageBootstrap: {30, 39}, + provisionStageListener: {40, 49}, + provisionStageService: {50, 59}, + } + reservedExits := map[int]string{ + statusExitRunning: "ctrld status running", + statusExitStopped: "ctrld status stopped", + statusExitUnknown: "ctrld status unknown", + statusExitNotReady: "ctrld status not ready", + deactivationPinInvalidExitCode: "deactivation pin invalid", + } + seenExits := make(map[int]provisionFailureCode) + for _, code := range allProvisionFailureCodes { + stage, ok := provisionStageForCode[code] + if !ok { + t.Fatalf("code %s has no stage", code) + } + exit, ok := provisionExitCodeForCode[code] + if !ok { + t.Fatalf("code %s has no exit code", code) + } + r := stageRanges[stage] + if exit < r[0] || exit > r[1] { + t.Errorf("code %s exit %d outside stage %s range %v", code, exit, stage, r) + } + if owner, ok := reservedExits[exit]; ok { + t.Errorf("code %s exit %d collides with %s", code, exit, owner) + } + if prev, dup := seenExits[exit]; dup { + t.Errorf("codes %s and %s share exit %d", prev, code, exit) + } + seenExits[exit] = code + } + if len(allProvisionFailureCodes) != 8 { + t.Errorf("expected 8 codes, got %d", len(allProvisionFailureCodes)) + } +} + +func TestNewProvisionResultRedactsSecrets(t *testing.T) { + token := "org-secret-token-12345" + cdUIDValue := "abcdef123456" + attempts := []provisionBindAttempt{ + {Addr: "127.0.0.1:53", Proto: "udp", OSError: "bind failed for " + token}, + } + r := newProvisionResult( + provisionCodeListenerBindFailed, + "could not bind, token="+token+" uid="+cdUIDValue, + attempts, + token, cdUIDValue, + ) + raw, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{token, cdUIDValue} { + if strings.Contains(string(raw), secret) { + t.Errorf("serialized result contains secret %q: %s", secret, raw) + } + } +} + +func TestNewProvisionResultBoundsDetail(t *testing.T) { + long := strings.Repeat("x", 1000) + var attempts []provisionBindAttempt + for i := 0; i < 50; i++ { + attempts = append(attempts, provisionBindAttempt{Addr: long, Proto: "udp", OSError: long}) + } + r := newProvisionResult(provisionCodeListenerBindFailed, long, attempts) + if got := len(r.Detail.Attempts); got > maxProvisionBindAttempts { + t.Errorf("attempts not capped: %d > %d", got, maxProvisionBindAttempts) + } + if len(r.Message) > maxProvisionStringLen { + t.Errorf("message not capped: %d", len(r.Message)) + } + for _, a := range r.Detail.Attempts { + if len(a.Addr) > maxProvisionStringLen || len(a.OSError) > maxProvisionStringLen { + t.Error("attempt fields not capped") + } + } +} + +func TestProvisionResultFields(t *testing.T) { + r := newProvisionResult(provisionCodeAPIRejected, "the API rejected this configuration", nil) + if r.Version != 1 { + t.Errorf("version = %d, want 1", r.Version) + } + if r.Stage != string(provisionStageBootstrap) { + t.Errorf("stage = %q, want bootstrap", r.Stage) + } + if r.ExitCode != provisionExitCodeForCode[provisionCodeAPIRejected] { + t.Errorf("exit = %d", r.ExitCode) + } + if _, err := time.Parse(time.RFC3339, r.Timestamp); err != nil { + t.Errorf("timestamp %q not RFC3339: %v", r.Timestamp, err) + } + if r.Detail != nil { + t.Error("nil attempts should give nil detail") + } +} + +func TestProvisionResultTrusted(t *testing.T) { + good := newProvisionResult(provisionCodeListenerBindFailed, "x", nil) + if !provisionResultTrusted(good) { + t.Error("constructor-built result must be trusted") + } + bogusCode := newProvisionResult(provisionCodeListenerBindFailed, "x", nil) + bogusCode.Code = "TOTALLY_MADE_UP" + if provisionResultTrusted(bogusCode) { + t.Error("unknown code must not be trusted") + } + wrongExit := newProvisionResult(provisionCodeListenerBindFailed, "x", nil) + wrongExit.ExitCode = 126 + if provisionResultTrusted(wrongExit) { + t.Error("exit code not matching the contract must not be trusted") + } + wrongStage := newProvisionResult(provisionCodeListenerBindFailed, "x", nil) + wrongStage.Stage = string(provisionStageService) + if provisionResultTrusted(wrongStage) { + t.Error("stage not matching the code must not be trusted") + } +} + +func TestNewProvisionResultTruncatesOnRuneBoundary(t *testing.T) { + msg := strings.Repeat("é", maxProvisionStringLen) // 2 bytes per rune + r := newProvisionResult(provisionCodeListenerBindFailed, msg, nil) + if len(r.Message) > maxProvisionStringLen { + t.Errorf("message not capped: %d bytes", len(r.Message)) + } + if !utf8.ValidString(r.Message) { + t.Error("truncation split a multi-byte rune") + } +} + +func TestFailureCodeDocTableMatchesConstants(t *testing.T) { + buf, err := os.ReadFile(filepath.Join("..", "..", "docs", "provisioning-failure-codes.md")) + if os.IsNotExist(err) { + // The Windows CI runner executes prebuilt test binaries outside the + // repo; the sync guarantee is still enforced on runners with a checkout. + t.Skip("failure-code doc not available in this test environment") + } + if err != nil { + t.Fatalf("could not read the failure-code doc: %v", err) + } + doc := string(buf) + rows := 0 + for _, line := range strings.Split(doc, "\n") { + if strings.HasPrefix(line, "| `") { + rows++ + } + } + if rows != len(allProvisionFailureCodes) { + t.Errorf("doc table has %d code rows, want %d", rows, len(allProvisionFailureCodes)) + } + for _, code := range allProvisionFailureCodes { + row := "| `" + string(code) + "` | " + string(provisionStageForCode[code]) + " | " + strconv.Itoa(provisionExitCodeForCode[code]) + " |" + if !strings.Contains(doc, row) { + t.Errorf("doc table missing row for %s (want prefix %q)", code, row) + } + } +} + +func TestProvisionFailureLineFormat(t *testing.T) { + r := newProvisionResult(provisionCodeListenerBindFailed, "could not find available listen ip and port", nil) + want := "provisioning failed: stage=listener code=LISTENER_BIND_FAILED (exit 41)" + if got := r.failureLine(); got != want { + t.Errorf("failureLine() = %q, want %q", got, want) + } +} + +func TestProvisionResultRoundTrip(t *testing.T) { + overrideProvisionResultPath(t) + in := newProvisionResult(provisionCodeServiceStartFailed, "service failed to start", nil) + if err := writeProvisionResult(in); err != nil { + t.Fatal(err) + } + out, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if out.Code != in.Code || out.Stage != in.Stage || out.ExitCode != in.ExitCode || out.Message != in.Message { + t.Errorf("round trip mismatch: in=%+v out=%+v", in, out) + } +} + +func TestWriteProvisionResultOverwritesAtomically(t *testing.T) { + path := overrideProvisionResultPath(t) + first := newProvisionResult(provisionCodeAPIUnreachable, "first", nil) + if err := writeProvisionResult(first); err != nil { + t.Fatal(err) + } + second := newProvisionResult(provisionCodeListenerBindFailed, "second", nil) + if err := writeProvisionResult(second); err != nil { + t.Fatal(err) + } + out, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if out.Code != string(provisionCodeListenerBindFailed) || out.Message != "second" { + t.Errorf("overwrite failed: %+v", out) + } + entries, err := os.ReadDir(filepath.Dir(path)) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Errorf("temp files left behind: %v", entries) + } +} + +func TestClearProvisionResult(t *testing.T) { + path := overrideProvisionResultPath(t) + clearProvisionResult() // missing file must not panic or error loudly + if err := writeProvisionResult(newProvisionResult(provisionCodeAPIUnreachable, "x", nil)); err != nil { + t.Fatal(err) + } + clearProvisionResult() + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Errorf("result file still present after clear: %v", err) + } +} + +func TestReadProvisionResultMissing(t *testing.T) { + overrideProvisionResultPath(t) + if _, err := readProvisionResult(); err == nil { + t.Error("expected error reading missing result file") + } +} + +func TestFailProvisionWritesLogsNotifiesAndExits(t *testing.T) { + overrideProvisionResultPath(t) + exitCode := -1 + oldExit := provisionExit + provisionExit = func(code int) { exitCode = code } + t.Cleanup(func() { provisionExit = oldExit }) + + notified := false + r := newProvisionResult(provisionCodeListenerBindFailed, "no listen addr", nil) + failProvision(r, func() { notified = true }) + + if !notified { + t.Error("notify func not called") + } + if exitCode != provisionExitCodeForCode[provisionCodeListenerBindFailed] { + t.Errorf("exit code = %d", exitCode) + } + out, err := readProvisionResult() + if err != nil { + t.Fatalf("result not persisted: %v", err) + } + if out.Code != string(provisionCodeListenerBindFailed) { + t.Errorf("persisted code = %q", out.Code) + } +} diff --git a/cmd/cli/service.go b/cmd/cli/service.go index f75ee55..39b41db 100644 --- a/cmd/cli/service.go +++ b/cmd/cli/service.go @@ -216,22 +216,30 @@ type task struct { Name string } -func doTasks(tasks []task) bool { - for _, task := range tasks { - mainLog.Load().Debug().Msgf("Running task %s", task.Name) - if err := task.f(); err != nil { - if task.abortOnError { - mainLog.Load().Error().Msgf("error running task %s: %v", task.Name, err) - return false +// doTasksE runs tasks in order and reports which abortOnError task, if any, +// stopped the run. Use it over doTasks when the failure must be attributed +// to a specific task. +func doTasksE(tasks []task) (failedTaskName string, err error) { + for _, t := range tasks { + mainLog.Load().Debug().Msgf("Running task %s", t.Name) + if taskErr := t.f(); taskErr != nil { + if t.abortOnError { + mainLog.Load().Error().Msgf("error running task %s: %v", t.Name, taskErr) + return t.Name, taskErr } // if this is darwin stop command, dont print debug // since launchctl complains on every start - if runtime.GOOS != "darwin" || task.Name != "Stop" { - mainLog.Load().Debug().Msgf("error running task %s: %v", task.Name, err) + if runtime.GOOS != "darwin" || t.Name != "Stop" { + mainLog.Load().Debug().Msgf("error running task %s: %v", t.Name, taskErr) } } } - return true + return "", nil +} + +func doTasks(tasks []task) bool { + _, err := doTasksE(tasks) + return err == nil } func checkHasElevatedPrivilege() { diff --git a/cmd/cli/service_test.go b/cmd/cli/service_test.go index 155bd3e..5047797 100644 --- a/cmd/cli/service_test.go +++ b/cmd/cli/service_test.go @@ -1,6 +1,7 @@ package cli import ( + "errors" "strings" "testing" ) @@ -26,3 +27,59 @@ func Test_ensureSystemdKillMode(t *testing.T) { }) } } + +func TestDoTasksESuccess(t *testing.T) { + var ran []string + tasks := []task{ + {func() error { ran = append(ran, "a"); return nil }, false, "a"}, + {func() error { ran = append(ran, "b"); return nil }, true, "b"}, + } + failedTask, err := doTasksE(tasks) + if failedTask != "" || err != nil { + t.Errorf("doTasksE() = (%q, %v), want (\"\", nil)", failedTask, err) + } + if got := strings.Join(ran, ","); got != "a,b" { + t.Errorf("ran tasks %q, want all tasks run in order", got) + } +} + +func TestDoTasksEAbortsOnAbortOnErrorTask(t *testing.T) { + wantErr := errors.New("install failed") + var ran []string + tasks := []task{ + {func() error { ran = append(ran, "Stop"); return nil }, false, "Stop"}, + {func() error { ran = append(ran, "Install"); return wantErr }, true, "Install"}, + {func() error { ran = append(ran, "Start"); return nil }, true, "Start"}, + } + failedTask, err := doTasksE(tasks) + if failedTask != "Install" || !errors.Is(err, wantErr) { + t.Errorf("doTasksE() = (%q, %v), want (\"Install\", %v)", failedTask, err, wantErr) + } + if got := strings.Join(ran, ","); got != "Stop,Install" { + t.Errorf("ran tasks %q, want the run to stop right after the abort", got) + } +} + +func TestDoTasksENonAbortFailureContinues(t *testing.T) { + var ran []string + tasks := []task{ + {func() error { ran = append(ran, "a"); return errors.New("a failed") }, false, "a"}, + {func() error { ran = append(ran, "b"); return nil }, true, "b"}, + } + failedTask, err := doTasksE(tasks) + if failedTask != "" || err != nil { + t.Errorf("doTasksE() = (%q, %v), want (\"\", nil) since the failing task did not abort", failedTask, err) + } + if got := strings.Join(ran, ","); got != "a,b" { + t.Errorf("ran tasks %q, want the run to continue past the non-abort failure", got) + } +} + +func TestDoTasksDelegatesToDoTasksE(t *testing.T) { + if !doTasks([]task{{func() error { return nil }, true, "ok"}}) { + t.Error("doTasks() = false, want true on success") + } + if doTasks([]task{{func() error { return errors.New("boom") }, true, "boom"}}) { + t.Error("doTasks() = true, want false when an abortOnError task fails") + } +} diff --git a/docs/provisioning-failure-codes.md b/docs/provisioning-failure-codes.md new file mode 100644 index 0000000..6d8b299 --- /dev/null +++ b/docs/provisioning-failure-codes.md @@ -0,0 +1,61 @@ +# Provisioning failure codes + +When ctrld hits a terminal failure during provisioning, it reports the same +stable code on three surfaces: + +- **Result file** — `provision_result.json` in the ctrld home directory + (next to the persisted internal `ctrld.log`). JSON with `stage`, `code`, + `exit_code`, `message`, and for listener failures a bounded + `detail.attempts` list of `{addr, proto, os_error}`. Written atomically, + removed on the next successful provisioning. Never contains provision + tokens, resolver/device IDs, or configuration contents. +- **Output line** — one fixed-format line on the CLI output: + `provisioning failed: stage= code= (exit )`. + The macOS pkg `postinstall` extracts exactly this line into the installer + log, so MDM consoles see it without any ctrld log configuration. +- **Exit code** — stage-scoped: bootstrap 30–39, listener 40–49, + service 50–59. Unrelated existing contracts are unchanged + (`ctrld status` exits 0–3; invalid deactivation pin exits 126). + +A customer or administrator only needs to report the code (or the whole +output line). The table below is the maintained support mapping; it must +stay in sync with `cmd/cli/provision_result.go` and changes in the same MR. + +## Codes + +| Code | Stage | Exit | Failure scenario | Next action / evidence | +|---|---|---|---|---| +| `API_UNREACHABLE` | bootstrap | 30 | The Control D API could not be reached or answered with a retryable error (network failure, proxy interference, 5xx, timeout) and retries ran out. The service manager may retry the service later. | Check the device's network path to `api.controld.com` (DNS, proxy, firewall, captive portal). Ask for the result file's `message` and whether other TLS traffic works. | +| `API_REJECTED` | bootstrap | 31 | The API answered and permanently rejected the configuration (4xx other than 408/429): bad or revoked token, malformed request. ctrld exits without burning service-manager restarts because retrying cannot change the answer. | Verify the provision token / org configuration in the Control D dashboard. Re-push after fixing credentials. Evidence: HTTP status in the result file `message`. | +| `API_DEVICE_INVALID` | bootstrap | 32 | The API reports the device/resolver no longer exists (error code 40402). ctrld self-uninstalls its service because the identity is gone server-side. | Confirm the device was deleted or re-provisioned in the dashboard; re-provision with a current token. No local evidence needed beyond the code. | +| `LISTENER_BIND_FAILED` | listener | 41 | No listen address could be bound after all fallbacks (configured address, 0.0.0.0:53, localhost:53, port 5354, random) were exhausted. `detail.attempts` records each tried address with the UDP/TCP OS error, e.g. `address already in use` (another DNS service owns the port) or `can't assign requested address` (address not on any interface). | Read `detail.attempts`: `address already in use` → find the process owning the port (`sudo lsof -i :53 -nP`); `can't assign requested address` → the configured IP is not present on the device. Then fix the conflict or the listener config. | +| `LISTENER_CONFIGURED_ADDR_UNAVAILABLE` | listener | 42 | An explicitly configured listener address could not be bound and configuration checks forbid falling back to another address, or (macOS intercept mode) the required explicit address is unavailable. | The configured `ip:port` in the listener config is wrong for this device or occupied. Verify the address exists on an interface and nothing else binds it; correct the config rather than expecting fallback. | +| `SERVICE_INSTALL_FAILED` | service | 51 | The OS service manager refused to install the service (launchd/systemd/SCM registration failed). | Check OS-level constraints: permissions/elevation, MDM policy blocking daemon installation, corrupted previous install. Evidence: result file `message` (service manager error), plus `launchctl print system/ctrld` / `systemctl status ctrld` / SCM state. | +| `SERVICE_START_FAILED` | service | 52 | The service installed but the service manager could not start it. | Check the service manager's own log for the start error, then the ctrld home dir `ctrld.log`. Often permissions or a binary quarantined by security tooling. | +| `SERVICE_SELFCHECK_FAILED` | service | 53 | The service started but never became healthy: no fresher failure was reported by the daemon, and the post-install DNS self-check failed. The just-installed service is rolled back (uninstalled). If the daemon itself recorded a more specific failure (e.g. a listener code), that code is reported instead of this one. | Ask for the drained service log printed by `ctrld start` and the result file. If the service was running but unreachable, check host firewall rules intercepting DNS to the listener. | + +## Reading the result file + +macOS and Linux (default service home is `/etc/controld`): + +```sh +sudo cat /etc/controld/provision_result.json +``` + +On Windows the file sits next to `ctrld.exe` in the install directory. A +custom `homedir` config moves it accordingly; routers and mobile use their +platform home directory. + +The file sits in the same directory as the persisted internal log +(`ctrld.log`) for the user the service runs as. On a healthy install the +file is absent. + +## Rules for maintainers + +- Codes are append-only once released. Never rename, renumber, or reuse a + code or exit number; add a new one and note the deprecation here. +- Every code added in `cmd/cli/provision_result.go` needs a row here in the + same MR. Tests enforce the code/stage/exit maps and that this table has + exactly one row per code. +- Detail must stay bounded and free of secrets: the constructor strips the + provision token and cd UID and caps sizes; do not bypass it. diff --git a/tasks/plan.md b/tasks/plan.md new file mode 100644 index 0000000..e1303fc --- /dev/null +++ b/tasks/plan.md @@ -0,0 +1,213 @@ +# Plan: provisioning failure codes (issue #586) + +Spec: SPEC.md. Baseline: `ac0e6aed` on `v1.0`. +Branches: `issue-586` (off `v1.0`), `issue-586-master` (off `master`). +Two MRs, both referencing #586; the `v1.0` MR carries `Closes #586`. + +## Shared contract (fixed here so parallel tasks cannot diverge) + +| Code | Stage | Exit | +|---|---|---| +| `API_UNREACHABLE` | bootstrap | 30 | +| `API_REJECTED` | bootstrap | 31 | +| `API_DEVICE_INVALID` | bootstrap | 32 | +| `LISTENER_BIND_FAILED` | listener | 41 | +| `LISTENER_CONFIGURED_ADDR_UNAVAILABLE` | listener | 42 | +| `SERVICE_INSTALL_FAILED` | service | 51 | +| `SERVICE_START_FAILED` | service | 52 | +| `SERVICE_SELFCHECK_FAILED` | service | 53 | + +- Result file: `provision_result.json` in the ctrld home dir (same + resolution as the persisted internal log: `absHomeDir` on v1.0, the + `userHomeDir`-based equivalent on master). Atomic write (temp + + rename in the same dir). Cleared when provisioning succeeds. +- Result schema (version 1): `version`, `timestamp` (RFC3339, UTC), + `stage`, `code`, `exit_code`, `message`, optional `detail.attempts[]` + of `{addr, proto, os_error}`; attempts capped at 12 entries, every + string capped at 256 chars. +- Identifier line, exact format (greppable, token-free by + construction): `provisioning failed: stage= code= (exit )`. +- Redaction: results are built through a constructor that takes the + secrets in scope (cd UID, provision token) and strips them from every + field. Messages come from our own summaries plus OS error strings, + never raw config or API bodies. +- Exit seam: `provisionExit = os.Exit` package var so tests can stub + process exit. Emission helper `failProvision(...)` writes the file, + logs the identifier line, calls the notify func, then exits with the + stage code. Nonzero exit is preserved everywhere the daemon exits + nonzero today; the deliberate clean return on permanent API rejection + stays a clean return (result file only). + +## Dependency graph + +``` +A1 (contract module + doc, v1.0) D1 (master port) + ├─► B1 daemon emissions (cli.go) depends on: contract table + ├─► B2 start-side (commands.go+service.go) (from A1) + C1 verified + └─► B3 postinstall (scripts, tests) implementation as reference + └─► C1 v1.0 checkpoint ────────────► D1 ─► E1 final checkpoint +``` + +## Group A — serial, runs inline (1 task) + +### A1. Contract foundation on `issue-586` +Create branch `issue-586` from `v1.0`. New files: +`cmd/cli/provision_result.go`, `cmd/cli/provision_result_test.go`, +`docs/provisioning-failure-codes.md`. + +Module contents: stage type + the 8 code constants + exit-code map; +`ProvisionResult` struct per schema; bounded/redacting constructor; +atomic `writeProvisionResult` / `readProvisionResult` / +`clearProvisionResult`; identifier-line formatter; `provisionExit` +seam; `failProvision` helper. Doc: full table — code, stage, exit, +failure scenario, next safe troubleshooting action / evidence request. + +Tests (RED first): every code maps to exactly one stage and one +in-range exit code (30–39/40–49/50–59); no collision with 0–3 +(`ctrld status`) or 126 (pin); file round-trip; atomic overwrite; +clear; redaction (a result built from inputs containing a fake token +and cd UID serializes without them); attempts/string caps enforced; +identifier line matches the exact format. + +Acceptance: `go build ./...` and `go test ./cmd/cli/` green; doc rows +exactly match the constants. + +## Group B — parallel Workflow fan-out, one subagent per task, worktree isolation, branched from `issue-586` after A1 + +### B1. Daemon emissions in `cli.go` +- Bootstrap branches in `run()` (`cli.go:339-372`): + - permanent rejection (`permanentAPIRejection`): write `API_REJECTED` + result (HTTP status + our own summary, no raw API body), keep the + existing clean return and its comment. + - invalid device (`controld.InvalidConfigCode`): write + `API_DEVICE_INVALID` before `uninstallInvalidCdUID`. + - fatal fetch: write `API_UNREACHABLE`, replace + `cdLogger.Fatal()` with error log + identifier line + + `failProvision` exit 30 (still nonzero for the service manager). +- Listener (`tryUpdateListenerConfig`, `tryUpdateListenerConfigIntercept`): + - record every failed bind attempt `{addr, proto, os_error}` — + capture UDP and TCP errors separately in `tryListen` (keep + `errors.Join` for control flow), cap per contract. + - exhaustion fatal (`cli.go:1639`) and converged-random fatal + (`cli.go:1720`) → `LISTENER_BIND_FAILED` exit 41 with attempts. + - no-fallback-allowed fatal (`cli.go:1652`) and intercept-mode fatals + (`cli.go:1452,1467`) → `LISTENER_CONFIGURED_ADDR_UNAVAILABLE` + exit 42 (fallback-exhausted intercept fatal stays + `LISTENER_BIND_FAILED`). + - all fatals keep calling the notify func first; final message + includes the code string. +- Clear the result file at the point provisioning is known good + (after `updateListenerConfig` succeeds in `run()`). + +Tests (RED first): occupy a UDP+TCP port, drive the listener path to +exhaustion with the exit seam stubbed, assert the result file has +`LISTENER_BIND_FAILED`, the attempted address, both protocols' +`os_error` (`address already in use` class); pure mapping test +API error → code (permanent 4xx → `API_REJECTED`, 40402 → +`API_DEVICE_INVALID`, network error → `API_UNREACHABLE`) following +`cli_preflight_test.go` patterns. + +Acceptance: only `cmd/cli/cli.go` + new/extended tests touched; +`go build ./... && go test ./cmd/cli/` green. + +### B2. `ctrld start` reporting in `commands.go` + `service.go` +- Add `doTasksE` (returns failed task name + error; `doTasks` keeps + its signature and delegates). +- Fresh-install path (`commands.go:618`): failed `Install` task → + `SERVICE_INSTALL_FAILED` (write result, print identifier line, exit + 51); failed `Start` task → `SERVICE_START_FAILED` (exit 52). This + fixes the current fall-through that exits 0 on install failure. +- Existing-service path (`commands.go:528-543`): failure → 52 with the + same reporting (replaces bare `os.Exit(1)`). +- Self-check failure branch (`commands.go:627-664`): keep the log + drain and `uninstall(p, s)`; then read the daemon's result file — + if present and stamped after this start attempt began, report its + stage/code/exit (daemon identity wins: e.g. `LISTENER_BIND_FAILED`); + otherwise write and report `SERVICE_SELFCHECK_FAILED` exit 53. + Extract this into a testable helper (fabricated result files + + stubbed exit seam). +- On successful start (self-check ok), clear any stale result file. + +Tests (RED first): `doTasksE` failure attribution; helper precedence +(fresh daemon result wins; stale/missing falls back to 53); exit-code +selection per failed task. + +Acceptance: only `cmd/cli/commands.go`, `cmd/cli/service.go` + tests +touched; build and package tests green. + +### B3. postinstall MDM surface +- `scripts/pkg/postinstall`: capture `ctrld start` output to a + `mktemp` file (chmod 600) instead of `/dev/null`; keep the plist + check as the success gate; on failure, `grep -m1 '^provisioning + failed: '` from the capture into the install log together with the + exit code; delete the capture file always; never echo any other + output line (token safety preserved by extracting only the + fixed-format line). +- Shell test `test-scripts/darwin/test-postinstall-provision-failure.sh` + (matching existing script conventions): stub `$CTRLD` that prints a + fake token plus a valid identifier line and exits 41; assert the + logged output contains stage/code/exit and not the token; assert + success path unchanged. Runs without root. +- Update `docs/macos-pkg-mdm.md` where it documents the discard + behavior/failure triage, and add the failure-code doc link. + +Acceptance: shell test passes locally (`sh test-scripts/darwin/...`); +only `scripts/pkg/postinstall`, `test-scripts/darwin/`, `docs/` +touched. + +## Checkpoint C1 — serial, after Group B merges + +Merge order: B1, B2, B3 into `issue-586`. Then: `go build ./...`, +`go vet ./...`, `go test ./cmd/cli/...` (and full `./...`), run the B3 +shell test, verify doc table == constants, and verify each spec +acceptance criterion has an implementation + test. Fix-forward any +merge fallout before Group D starts. + +## Group D — serial (1 task, own worktree off `master`) + +### D1. Master port on `issue-586-master` +Create `git worktree` with branch `issue-586-master` from +`origin/master`/`master`. Port with the v1.0 implementation as +reference, adapted to master's structure (zap-shaped logging idiom, +no `commands.go`): + +- `cmd/cli/provision_result.go` + tests: identical contract table. +- Bootstrap: `run()` branches at master `cli.go:340-374` (same + permanent-rejection clean return, invalid-device, fatal fetch). +- Listener: `tryUpdateListenerConfig` fatals at master + `cli.go:1657/1667/1719`; intercept variant at `cli.go:1487/1502`; + per-attempt capture (bind errors currently logged at Debug, + `cli.go:1665`). +- Start side: `commands_service_start.go` — both `doTasks` call sites, + self-check `default:` arm (`os.Exit(1)` ~line 370), same fall-through + audit, same precedence logic; `doTasksE` in `service.go`. +- `docs/provisioning-failure-codes.md`: same table (omit + pkg/postinstall-specific notes; master has no `scripts/pkg`). +- No postinstall work on master. + +Tests mirrored from v1.0 where the structure allows. + +Acceptance: in the master worktree, `go build ./...`, +`go test ./cmd/cli/...` green; constants table semantically identical +to `issue-586`. + +## Checkpoint E1 — serial, final + +- Cross-branch contract equality: compare code constants, exit codes, + identifier-line format, result schema between the two branches. +- Full test suites on both branches. +- Both branches committed (per-task Conventional Commits); no pushes, + no MRs yet — `/draft-review` is the next pipeline step. + +## Execution notes + +- Each parallel group runs as one Workflow fan-out, one subagent per + task, `isolation: 'worktree'` so parallel edits never conflict; + serial tasks (A1, C1, D1, E1) run inline (D1 manages its own + master-based worktree). +- Every subagent follows RED → GREEN → regression → build and commits + in its worktree; the orchestrator merges in dependency order and + runs the full suite before the next group. +- Subagent prompts are self-contained: they carry the contract table + and file anchors from this plan, not references to SPEC.md (worktree + copies may not include untracked files). diff --git a/tasks/todo.md b/tasks/todo.md new file mode 100644 index 0000000..10a2693 --- /dev/null +++ b/tasks/todo.md @@ -0,0 +1,21 @@ +# TODO: issue #586 provisioning failure codes + +Groups run in order; tasks inside a parallel group run as one Workflow +fan-out (one subagent per task, worktree isolation). + +## Group A (serial) +- [x] A1: contract module `cmd/cli/provision_result.go` + tests + `docs/provisioning-failure-codes.md` on branch `issue-586` + +## Group B (parallel after A1) +- [x] B1: daemon emissions — bootstrap + listener paths in `cmd/cli/cli.go` + tests +- [x] B2: `ctrld start` reporting — `cmd/cli/commands.go`, `cmd/cli/service.go` (doTasksE, exit-0 fall-through fix, self-check precedence) + tests +- [x] B3: postinstall MDM surface — `scripts/pkg/postinstall`, shell test, `docs/macos-pkg-mdm.md` + +## Checkpoint C1 (serial) +- [x] C1: merge B1→B2→B3 into `issue-586`, full build/vet/test, shell test, doc/constants parity, spec AC audit + +## Group D (serial) +- [x] D1: master port on `issue-586-master` (contract module, cli.go emissions, commands_service_start.go, docs) + tests + +## Checkpoint E1 (serial) +- [x] E1: cross-branch contract equality, full suites on both branches, commits tidy — stop before push/MR (`/draft-review` next) diff --git a/test-scripts/darwin/test-postinstall-provision-failure.sh b/test-scripts/darwin/test-postinstall-provision-failure.sh new file mode 100755 index 0000000..d9f6eed --- /dev/null +++ b/test-scripts/darwin/test-postinstall-provision-failure.sh @@ -0,0 +1,148 @@ +#!/bin/sh +# Test: the MDM pkg postinstall script surfaces ctrld's provisioning +# failure identifier in its output without leaking the provision token, +# and still reports success once the plist exists. +# +# Self-contained and root-free: every path postinstall touches is +# redirected into a throwaway temp directory via the +# CTRLD_POSTINSTALL_{PLIST,CTRLD,PREFS} overrides, and 'defaults' is +# stubbed on PATH so the profile-wait loop resolves on its first attempt. +# +# Out of scope: the upgrade path (plist already exists) calls the real +# launchctl and is not exercised here. +# +# Run: sh test-postinstall-provision-failure.sh + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +POSTINSTALL="$SCRIPT_DIR/../../scripts/pkg/postinstall" + +WORKDIR=$(mktemp -d -t ctrld-postinstall-test) || { + echo "FAIL: could not create test work directory" >&2 + exit 1 +} +trap 'rm -rf "$WORKDIR"' EXIT + +FAKE_TOKEN="FAKE-PROVISION-TOKEN-DO-NOT-LEAK-93af0c" +export FAKE_TOKEN + +STUBBIN="$WORKDIR/stubbin" +mkdir -p "$STUBBIN" + +cat > "$STUBBIN/defaults" <<'STUB' +#!/bin/sh +# Stand-in for macOS 'defaults read ': answers ProvisionToken +# immediately, like a profile that only sets that one key, so the +# postinstall wait loop never has to sleep. +if [ "$1" = "read" ] && [ "$3" = "ProvisionToken" ]; then + echo "$FAKE_TOKEN" + exit 0 +fi +exit 1 +STUB +chmod +x "$STUBBIN/defaults" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +assert_eq() { + # assert_eq + if [ "$1" != "$2" ]; then + fail "$3 (expected '$2', got '$1')" + fi +} + +assert_contains() { + # assert_contains + case "$1" in + *"$2"*) ;; + *) fail "$3 (expected to find '$2')" ;; + esac +} + +assert_not_contains() { + # assert_not_contains + case "$1" in + *"$2"*) fail "$3 (must not contain '$2')" ;; + *) ;; + esac +} + +# run_postinstall runs postinstall with the given plist/ctrld overrides and +# a private TMPDIR, so the caller can check that the postinstall's own +# capture file (created inside that TMPDIR via mktemp) does not survive. +run_postinstall() { + plist_override=$1 + ctrld_override=$2 + capture_tmpdir=$3 + output=$(PATH="$STUBBIN:$PATH" \ + TMPDIR="$capture_tmpdir" \ + CTRLD_POSTINSTALL_PLIST="$plist_override" \ + CTRLD_POSTINSTALL_CTRLD="$ctrld_override" \ + CTRLD_POSTINSTALL_PREFS="/does/not/matter" \ + sh "$POSTINSTALL" 2>&1) + exit_code=$? +} + +# --- Failure case: ctrld reports a listener bind failure --------------- + +failure_dir="$WORKDIR/failure" +failure_tmp="$failure_dir/tmp" +mkdir -p "$failure_tmp" +failure_plist="$failure_dir/ctrld.plist" +failure_ctrld="$failure_dir/ctrld" + +cat > "$failure_ctrld" <<'STUB' +#!/bin/sh +# Stands in for a ctrld that fails to bind its listener: echoes the raw +# token (as ctrld's own error output may) plus the fixed-format failure +# identifier behind a log-style prefix, then exits with the stage code. +echo "$FAKE_TOKEN" +echo "2024-01-01T00:00:00Z ERR ctrld: provisioning failed: stage=listener code=LISTENER_BIND_FAILED (exit 41)" +exit 41 +STUB +chmod +x "$failure_ctrld" + +run_postinstall "$failure_plist" "$failure_ctrld" "$failure_tmp" + +assert_eq "$exit_code" "1" "failure case: postinstall exit code" +assert_contains "$output" "stage=listener" "failure case: output names the stage" +assert_contains "$output" "LISTENER_BIND_FAILED" "failure case: output names the code" +assert_contains "$output" "41" "failure case: output names the exit code" +assert_not_contains "$output" "$FAKE_TOKEN" "failure case: output must not contain the provision token" + +leftover=$(ls -A "$failure_tmp" 2>/dev/null) +assert_eq "$leftover" "" "failure case: capture temp file removed" + +# --- Success case: ctrld provisions and writes the plist ---------------- + +success_dir="$WORKDIR/success" +success_tmp="$success_dir/tmp" +mkdir -p "$success_tmp" +success_plist="$success_dir/ctrld.plist" +success_ctrld="$success_dir/ctrld" + +cat > "$success_ctrld" < "$success_plist" +exit 0 +STUB +chmod +x "$success_ctrld" + +run_postinstall "$success_plist" "$success_ctrld" "$success_tmp" + +assert_eq "$exit_code" "0" "success case: postinstall exit code" +assert_contains "$output" "provisioning complete" "success case: output mentions success" + +if [ "$FAILURES" -gt 0 ]; then + echo "$FAILURES assertion(s) failed" >&2 + exit 1 +fi + +echo "OK: postinstall surfaces provisioning failure codes without leaking the token" +exit 0 From 6615e431dca441fd5a923fcb067a895db0923e35 Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Fri, 21 Aug 2026 08:07:19 +0000 Subject: [PATCH 14/16] Apply managed DNS mode in the macOS package --- cmd/cli/cli.go | 33 +++-- cmd/cli/commands.go | 24 ++-- cmd/cli/intercept_mode_config_test.go | 38 ++++++ cmd/cli/main_test.go | 24 +++- cmd/cli/prog.go | 5 +- cmd/cli/prog_intercept_fallback_test.go | 16 +++ cmd/cli/service.go | 5 + cmd/cli/service_args_darwin.go | 48 ++++--- cmd/cli/service_args_darwin_test.go | 58 +++++++++ cmd/cli/service_args_others.go | 10 +- cmd/cli/service_args_windows.go | 58 ++++++--- cmd/cli/service_args_windows_test.go | 54 ++++++++ .../darwin/test-pkg-intercept-mode.sh | 123 ++++++++++++++++++ 13 files changed, 432 insertions(+), 64 deletions(-) create mode 100644 cmd/cli/intercept_mode_config_test.go create mode 100644 cmd/cli/service_args_darwin_test.go create mode 100644 cmd/cli/service_args_windows_test.go create mode 100755 test-scripts/darwin/test-pkg-intercept-mode.sh diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 1f495a3..cf8c943 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -147,6 +147,25 @@ func isMobile() bool { return runtime.GOOS == "android" || runtime.GOOS == "ios" } +func updateConfigInterceptMode(cfg *ctrld.Config, mode string) bool { + desired := "" + switch mode { + case "dns", "hard": + desired = mode + case "off": + desired = "" + case "": + return false + default: + return false + } + if cfg.Service.InterceptMode == desired { + return false + } + cfg.Service.InterceptMode = desired + return true +} + // isAndroid reports whether the current OS is Android. func isAndroid() bool { return runtime.GOOS == "android" @@ -361,14 +380,12 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { processLogAndCacheFlags(v, &cfg) } - // Persist intercept_mode to config when provided via CLI flag on full install. - // This ensures the config file reflects the actual running mode for RMM/MDM visibility. - if interceptMode == "dns" || interceptMode == "hard" { - if cfg.Service.InterceptMode != interceptMode { - cfg.Service.InterceptMode = interceptMode - updated = true - mainLog.Load().Info().Msgf("writing intercept_mode = %q to config", interceptMode) - } + // Keep config and the explicit CLI/service mode in sync. In particular, "off" + // must clear a previously persisted dns/hard value or the next service start + // would silently re-enable interception from config. + if updateConfigInterceptMode(&cfg, interceptMode) { + updated = true + mainLog.Load().Info().Msgf("writing intercept_mode = %q to config", cfg.Service.InterceptMode) } if updated { diff --git a/cmd/cli/commands.go b/cmd/cli/commands.go index 0cc6542..b58ae78 100644 --- a/cmd/cli/commands.go +++ b/cmd/cli/commands.go @@ -394,21 +394,23 @@ NOTE: running "ctrld start" without any arguments will start already installed c svcExists := serviceConfigFileExists() mainLog.Load().Debug().Msgf("intercept upgrade check: args=%v interceptOnly=%v svcConfigExists=%v interceptMode=%q", osArgsEarly, interceptOnly, svcExists, interceptMode) if interceptOnly && svcExists { - // Remove any existing intercept flags before applying the new value. - _ = removeServiceFlag("--intercept-mode") + // Replace any existing split or --intercept-mode= form. Keep an + // explicit "off" argument so it overrides a previously persisted config + // value while the service clears that value on startup. + if err := removeServiceFlag("--intercept-mode"); err != nil { + mainLog.Load().Fatal().Err(err).Msg("failed to remove existing intercept mode from service arguments") + } if interceptMode == "off" { - // "off" = remove intercept mode entirely (just the removal above). - mainLog.Load().Notice().Msg("Existing service detected — removing --intercept-mode from service arguments") + mainLog.Load().Notice().Msg("Existing service detected — disabling intercept mode") } else { - // Add the new mode value. mainLog.Load().Notice().Msgf("Existing service detected — appending --intercept-mode %s to service arguments", interceptMode) - if err := appendServiceFlag("--intercept-mode"); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to append intercept flag to service arguments") - } - if err := appendServiceFlag(interceptMode); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to append intercept mode value to service arguments") - } + } + if err := appendServiceFlag("--intercept-mode"); err != nil { + mainLog.Load().Fatal().Err(err).Msg("failed to append intercept flag to service arguments") + } + if err := appendServiceFlag(interceptMode); err != nil { + mainLog.Load().Fatal().Err(err).Msg("failed to append intercept mode value to service arguments") } // Stop the service if running (bypasses ctrld pin — this is an diff --git a/cmd/cli/intercept_mode_config_test.go b/cmd/cli/intercept_mode_config_test.go new file mode 100644 index 0000000..a3167eb --- /dev/null +++ b/cmd/cli/intercept_mode_config_test.go @@ -0,0 +1,38 @@ +package cli + +import ( + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func TestUpdateConfigInterceptMode(t *testing.T) { + tests := []struct { + name string + current string + mode string + want string + wantUpdated bool + }{ + {name: "empty flag preserves config", current: "dns", mode: "", want: "dns"}, + {name: "dns is persisted", mode: "dns", want: "dns", wantUpdated: true}, + {name: "hard is persisted", current: "dns", mode: "hard", want: "hard", wantUpdated: true}, + {name: "off clears persisted mode", current: "dns", mode: "off", want: "", wantUpdated: true}, + {name: "off is idempotent", mode: "off", want: ""}, + {name: "invalid flag preserves config", current: "hard", mode: "invalid", want: "hard"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := &ctrld.Config{} + cfg.Service.InterceptMode = tc.current + updated := updateConfigInterceptMode(cfg, tc.mode) + if updated != tc.wantUpdated { + t.Fatalf("updateConfigInterceptMode() updated = %v, want %v", updated, tc.wantUpdated) + } + if cfg.Service.InterceptMode != tc.want { + t.Fatalf("service.intercept_mode = %q, want %q", cfg.Service.InterceptMode, tc.want) + } + }) + } +} diff --git a/cmd/cli/main_test.go b/cmd/cli/main_test.go index 6e2257f..10553e4 100644 --- a/cmd/cli/main_test.go +++ b/cmd/cli/main_test.go @@ -5,12 +5,34 @@ import ( "os" "os/exec" "strings" + "sync" "testing" "github.com/rs/zerolog" ) -var logOutput strings.Builder +// logOutput is the log sink for the whole test binary. Tests share it with any +// background goroutine the code under test starts (watchdogs, timers), so it +// must tolerate concurrent writes. +var logOutput syncBuffer + +// syncBuffer is a strings.Builder guarded by a mutex. +type syncBuffer struct { + mu sync.Mutex + sb strings.Builder +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.String() +} // envFakeVersionOutput makes this test binary impersonate a ctrld executable: when // set, the process writes the value to stdout and exits without running any test, so diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 7a0c3b0..a51417c 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -930,11 +930,12 @@ func (p *prog) setDNS() { // Validate and resolve intercept mode. // CLI flag (--intercept-mode) takes priority over config file. - // Valid values: "" (off), "dns" (with VPN split routing), "hard" (all DNS through ctrld). + // Valid values: "" (use config), "off" (explicitly disable), "dns" (with VPN + // split routing), and "hard" (all DNS through ctrld). if interceptMode != "" && !validInterceptMode(interceptMode) { mainLog.Load().Fatal().Msgf("invalid --intercept-mode value %q: must be 'off', 'dns', or 'hard'", interceptMode) } - if interceptMode == "" || interceptMode == "off" { + if interceptMode == "" { interceptMode = p.configuredInterceptMode() if interceptMode != "" && interceptMode != "off" { mainLog.Load().Info().Msgf("Intercept mode enabled via config (intercept_mode = %q)", interceptMode) diff --git a/cmd/cli/prog_intercept_fallback_test.go b/cmd/cli/prog_intercept_fallback_test.go index 2c5078c..568e788 100644 --- a/cmd/cli/prog_intercept_fallback_test.go +++ b/cmd/cli/prog_intercept_fallback_test.go @@ -146,6 +146,22 @@ func (h *interceptFallbackHarness) run(t *testing.T) { p.setDNS() } +func TestSetDNSExplicitOffOverridesConfig(t *testing.T) { + h := newInterceptFallbackHarness(t, &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 53}) + interceptMode = "off" + dnsIntercept = false + hardIntercept = false + + h.run(t) + + if h.interceptCalls != 0 { + t.Fatalf("intercept start called %d time(s), want 0: explicit off must override service.intercept_mode", h.interceptCalls) + } + if h.installCalls != 1 { + t.Fatalf("interface DNS installed %d time(s), want 1", h.installCalls) + } +} + // TestSetDNSRefusesUnreachableFallback is the behaviour test for the reported outage: it // drives the real setDNS() lifecycle rather than the classification helper alone. // diff --git a/cmd/cli/service.go b/cmd/cli/service.go index 39b41db..3611b68 100644 --- a/cmd/cli/service.go +++ b/cmd/cli/service.go @@ -162,6 +162,11 @@ func (s *systemd) Start() error { // This is necessary for running self-upgrade flow. func ensureSystemdKillMode(r io.Reader) (opts []*unit.UnitOption, change bool) { opts, err := unit.DeserializeOptions(r) + // staticcheck sees only the explicit non-nil sends on the lexer's error + // channel, so it reports this comparison as always true. On success the + // lexer sends nothing and closes the channel, so the receive yields a nil + // error and this branch is not taken. + //lint:ignore SA4023 upstream delivers a nil error by closing the channel if err != nil { mainLog.Load().Error().Err(err).Msg("failed to deserialize options") return diff --git a/cmd/cli/service_args_darwin.go b/cmd/cli/service_args_darwin.go index d588960..5bc1823 100644 --- a/cmd/cli/service_args_darwin.go +++ b/cmd/cli/service_args_darwin.go @@ -24,19 +24,19 @@ func serviceConfigFileExists() bool { // to intercept mode without losing the existing --cd flag and other arguments. // // On macOS, this modifies the launchd plist at /Library/LaunchDaemons/ctrld.plist -// using the "defaults" command, which is the standard way to edit plists. +// using PlistBuddy for exact array reads and writes. // // The function is idempotent: if the flag already exists, it's a no-op. func appendServiceFlag(flag string) error { // Read current ProgramArguments from plist. - out, err := exec.Command("defaults", "read", launchdPlistPath, "ProgramArguments").CombinedOutput() + out, err := exec.Command("/usr/libexec/PlistBuddy", "-c", "Print :ProgramArguments", launchdPlistPath).CombinedOutput() if err != nil { return fmt.Errorf("failed to read plist ProgramArguments: %w (output: %s)", err, strings.TrimSpace(string(out))) } - // Check if the flag is already present (idempotent). - args := string(out) - if strings.Contains(args, flag) { + // Check exact array entries. A substring match can confuse a mode such as "off" + // with an unrelated path or argument and leave the flag without its value. + if serviceArgumentPresent(out, flag) { mainLog.Load().Debug().Msgf("Service flag %q already present in plist, skipping", flag) return nil } @@ -61,9 +61,8 @@ func verifyServiceRegistration() error { return nil } -// removeServiceFlag removes a CLI flag (and its value, if the next argument is not -// a flag) from the installed service's launch arguments. For example, removing -// "--intercept-mode" also removes the following "dns" or "hard" value argument. +// removeServiceFlag removes both "--flag value" and "--flag=value" forms from the +// installed service's launch arguments. // // The function is idempotent: if the flag doesn't exist, it's a no-op. func removeServiceFlag(flag string) error { @@ -92,22 +91,14 @@ func removeServiceFlag(flag string) error { entries = append(entries, trimmed) } - index := -1 - for i, entry := range entries { - if entry == flag { - index = i - break - } - } + index, hasValue := serviceFlagPosition(entries, flag) if index < 0 { mainLog.Load().Debug().Msgf("Service flag %q not present in plist, skipping removal", flag) return nil } - // Check if the next entry is a value (not a flag). If so, delete it first - // (deleting by index shifts subsequent entries down, so delete value before flag). - hasValue := index+1 < len(entries) && !strings.HasPrefix(entries[index+1], "-") + // Delete a separate value first. An inline --flag=value entry is one array item. if hasValue { delVal := exec.Command( "/usr/libexec/PlistBuddy", @@ -132,3 +123,24 @@ func removeServiceFlag(flag string) error { mainLog.Load().Info().Msgf("Removed %q from service launch arguments", flag) return nil } + +func serviceArgumentPresent(out []byte, argument string) bool { + for _, line := range strings.Split(string(out), "\n") { + if strings.TrimSpace(line) == argument { + return true + } + } + return false +} + +func serviceFlagPosition(entries []string, flag string) (index int, hasValue bool) { + for i, entry := range entries { + switch { + case entry == flag: + return i, i+1 < len(entries) && !strings.HasPrefix(entries[i+1], "-") + case strings.HasPrefix(entry, flag+"="): + return i, false + } + } + return -1, false +} diff --git a/cmd/cli/service_args_darwin_test.go b/cmd/cli/service_args_darwin_test.go new file mode 100644 index 0000000..dc23b3b --- /dev/null +++ b/cmd/cli/service_args_darwin_test.go @@ -0,0 +1,58 @@ +//go:build darwin + +package cli + +import "testing" + +func TestServiceArgumentPresent(t *testing.T) { + out := []byte("Array {\n /usr/local/bin/ctrld\n run\n --config=/Users/officer/ctrld.toml\n --intercept-mode=dns\n}\n") + if !serviceArgumentPresent(out, "--intercept-mode=dns") { + t.Fatal("exact inline argument was not found") + } + if serviceArgumentPresent(out, "--intercept-mode") { + t.Fatal("inline flag was mistaken for a separate flag argument") + } + if serviceArgumentPresent(out, "off") { + t.Fatal("substring in an unrelated path was mistaken for the off argument") + } +} + +func TestServiceFlagPosition(t *testing.T) { + tests := []struct { + name string + entries []string + wantIndex int + wantHasValue bool + }{ + { + name: "split form", + entries: []string{"run", "--cd=uid", "--intercept-mode", "dns"}, + wantIndex: 2, + wantHasValue: true, + }, + { + name: "inline form", + entries: []string{"run", "--cd=uid", "--intercept-mode=dns"}, + wantIndex: 2, + }, + { + name: "flag followed by another flag", + entries: []string{"run", "--intercept-mode", "--config=/etc/ctrld.toml"}, + wantIndex: 1, + }, + { + name: "absent", + entries: []string{"run", "--cd=uid"}, + wantIndex: -1, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + index, hasValue := serviceFlagPosition(tc.entries, "--intercept-mode") + if index != tc.wantIndex || hasValue != tc.wantHasValue { + t.Fatalf("serviceFlagPosition() = (%d, %v), want (%d, %v)", index, hasValue, tc.wantIndex, tc.wantHasValue) + } + }) + } +} diff --git a/cmd/cli/service_args_others.go b/cmd/cli/service_args_others.go index 07edda2..b6eb688 100644 --- a/cmd/cli/service_args_others.go +++ b/cmd/cli/service_args_others.go @@ -3,10 +3,14 @@ package cli import ( - "fmt" + "errors" "os" ) +// errServiceFlagsUnsupported is returned by the service-argument helpers on +// platforms that do not store service arguments in a file ctrld can rewrite. +var errServiceFlagsUnsupported = errors.New("modifying service flags is not supported on this platform; use intercept_mode in config instead") + // serviceConfigFileExists checks common service config file locations on Linux. func serviceConfigFileExists() bool { // systemd unit file @@ -24,7 +28,7 @@ func serviceConfigFileExists() bool { // Linux services (systemd) store args in unit files; intercept mode // should be set via the config file (intercept_mode) on these platforms. func appendServiceFlag(flag string) error { - return fmt.Errorf("appending service flags is not supported on this platform; use intercept_mode in config instead") + return errServiceFlagsUnsupported } // verifyServiceRegistration is a no-op on this platform. @@ -34,5 +38,5 @@ func verifyServiceRegistration() error { // removeServiceFlag is not yet implemented on this platform. func removeServiceFlag(flag string) error { - return fmt.Errorf("removing service flags is not supported on this platform; use intercept_mode in config instead") + return errServiceFlagsUnsupported } diff --git a/cmd/cli/service_args_windows.go b/cmd/cli/service_args_windows.go index 246a009..1eed7da 100644 --- a/cmd/cli/service_args_windows.go +++ b/cmd/cli/service_args_windows.go @@ -47,8 +47,9 @@ func appendServiceFlag(flag string) error { return fmt.Errorf("failed to read service config: %w", err) } - // Check if flag already present (idempotent). - if strings.Contains(config.BinaryPathName, flag) { + // Check exact arguments so a short mode such as "off" is not confused with + // an unrelated path or value. + if binaryPathArgumentPresent(config.BinaryPathName, flag) { mainLog.Load().Debug().Msgf("Service flag %q already present in BinPath, skipping", flag) return nil } @@ -103,9 +104,8 @@ func verifyServiceRegistration() error { return nil } -// removeServiceFlag removes a CLI flag (and its value, if present) from the installed -// Windows service's BinPath. For example, removing "--intercept-mode" also removes -// the following "dns" or "hard" value. The function is idempotent. +// removeServiceFlag removes both "--flag value" and "--flag=value" forms from the +// installed Windows service's BinPath. The function is idempotent. func removeServiceFlag(flag string) error { m, err := mgr.Connect() if err != nil { @@ -124,25 +124,12 @@ func removeServiceFlag(flag string) error { return fmt.Errorf("failed to read service config: %w", err) } - if !strings.Contains(config.BinaryPathName, flag) { + updatedPath, removed := removeBinaryPathFlag(config.BinaryPathName, flag) + if !removed { mainLog.Load().Debug().Msgf("Service flag %q not present in BinPath, skipping removal", flag) return nil } - - // Split BinPath into parts, find and remove the flag + its value (if any). - parts := strings.Fields(config.BinaryPathName) - var newParts []string - for i := 0; i < len(parts); i++ { - if parts[i] == flag { - // Skip the flag. Also skip the next part if it's a value (not a flag). - if i+1 < len(parts) && !strings.HasPrefix(parts[i+1], "-") { - i++ // skip value too - } - continue - } - newParts = append(newParts, parts[i]) - } - config.BinaryPathName = strings.Join(newParts, " ") + config.BinaryPathName = updatedPath if err := s.UpdateConfig(config); err != nil { return fmt.Errorf("failed to update service config: %w", err) @@ -151,3 +138,32 @@ func removeServiceFlag(flag string) error { mainLog.Load().Info().Msgf("Removed %q from service BinPath", flag) return nil } + +func binaryPathArgumentPresent(binaryPath, argument string) bool { + for _, part := range strings.Fields(binaryPath) { + if part == argument { + return true + } + } + return false +} + +func removeBinaryPathFlag(binaryPath, flag string) (string, bool) { + parts := strings.Fields(binaryPath) + newParts := make([]string, 0, len(parts)) + removed := false + for i := 0; i < len(parts); i++ { + switch { + case parts[i] == flag: + removed = true + if i+1 < len(parts) && !strings.HasPrefix(parts[i+1], "-") { + i++ + } + case strings.HasPrefix(parts[i], flag+"="): + removed = true + default: + newParts = append(newParts, parts[i]) + } + } + return strings.Join(newParts, " "), removed +} diff --git a/cmd/cli/service_args_windows_test.go b/cmd/cli/service_args_windows_test.go new file mode 100644 index 0000000..ed8e915 --- /dev/null +++ b/cmd/cli/service_args_windows_test.go @@ -0,0 +1,54 @@ +//go:build windows + +package cli + +import "testing" + +func TestBinaryPathArgumentPresent(t *testing.T) { + path := `C:\ControlD\ctrld.exe run --config=C:\Users\officer\ctrld.toml --intercept-mode=dns` + if !binaryPathArgumentPresent(path, "--intercept-mode=dns") { + t.Fatal("exact inline argument was not found") + } + if binaryPathArgumentPresent(path, "--intercept-mode") { + t.Fatal("inline flag was mistaken for a separate flag argument") + } + if binaryPathArgumentPresent(path, "off") { + t.Fatal("substring in an unrelated path was mistaken for the off argument") + } +} + +func TestRemoveBinaryPathFlag(t *testing.T) { + tests := []struct { + name string + binaryPath string + wantPath string + wantRemoved bool + }{ + { + name: "split form", + binaryPath: `ctrld.exe run --cd=uid --intercept-mode dns --config=ctrld.toml`, + wantPath: `ctrld.exe run --cd=uid --config=ctrld.toml`, + wantRemoved: true, + }, + { + name: "inline form", + binaryPath: `ctrld.exe run --cd=uid --intercept-mode=dns --config=ctrld.toml`, + wantPath: `ctrld.exe run --cd=uid --config=ctrld.toml`, + wantRemoved: true, + }, + { + name: "absent", + binaryPath: `ctrld.exe run --cd=uid`, + wantPath: `ctrld.exe run --cd=uid`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + path, removed := removeBinaryPathFlag(tc.binaryPath, "--intercept-mode") + if path != tc.wantPath || removed != tc.wantRemoved { + t.Fatalf("removeBinaryPathFlag() = (%q, %v), want (%q, %v)", path, removed, tc.wantPath, tc.wantRemoved) + } + }) + } +} diff --git a/test-scripts/darwin/test-pkg-intercept-mode.sh b/test-scripts/darwin/test-pkg-intercept-mode.sh new file mode 100755 index 0000000..eb06e45 --- /dev/null +++ b/test-scripts/darwin/test-pkg-intercept-mode.sh @@ -0,0 +1,123 @@ +#!/bin/sh +set -eu + +repo_root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) +postinstall="$repo_root/scripts/pkg/postinstall" +fixture=$(mktemp -d "${TMPDIR:-/tmp}/ctrld-pkg-intercept.XXXXXX") +trap 'rm -rf "$fixture"' EXIT HUP INT TERM + +bin="$fixture/bin" +mkdir -p "$bin" + +cat >"$bin/defaults" <<'EOF' +#!/bin/sh +key=${3:-} +case "$key" in + ProvisionToken) + [ "${FAKE_TOKEN_PRESENT:-0}" = "1" ] || exit 1 + printf '%s\n' "${FAKE_TOKEN:-test-token}" + ;; + InterceptMode) + [ "${FAKE_MODE_PRESENT:-0}" = "1" ] || exit 1 + printf '%s\n' "${FAKE_MODE:-}" + ;; + CustomHostname|UseDevEnvironment) + exit 1 + ;; + *) + exit 1 + ;; +esac +EOF + +cat >"$bin/launchctl" <<'EOF' +#!/bin/sh +printf 'launchctl %s\n' "$*" >>"$CALLS" +exit 0 +EOF + +cat >"$bin/ctrld" <<'EOF' +#!/bin/sh +printf 'ctrld %s\n' "$*" >>"$CALLS" +case " $* " in + *" --cd-org="*) : >"$CTRLD_POSTINSTALL_PLIST" ;; +esac +exit 0 +EOF + +chmod +x "$bin/defaults" "$bin/launchctl" "$bin/ctrld" + +assert_contains() { + expected=$1 + file=$2 + if ! grep -Fq -- "$expected" "$file"; then + printf 'FAIL: expected %s in %s\n' "$expected" "$file" >&2 + sed -n '1,120p' "$file" >&2 + exit 1 + fi +} + +assert_not_contains() { + unexpected=$1 + file=$2 + if grep -Fq -- "$unexpected" "$file"; then + printf 'FAIL: did not expect %s in %s\n' "$unexpected" "$file" >&2 + sed -n '1,120p' "$file" >&2 + exit 1 + fi +} + +run_case() { + name=$1 + existing=$2 + mode_present=$3 + mode=$4 + case_dir="$fixture/$name" + mkdir -p "$case_dir" + plist="$case_dir/ctrld.plist" + prefs="$case_dir/preferences" + calls="$case_dir/calls" + output="$case_dir/output" + : >"$calls" + if [ "$existing" = "1" ]; then + : >"$plist" + fi + + PATH="$bin:$PATH" \ + CALLS="$calls" \ + FAKE_TOKEN_PRESENT=1 \ + FAKE_TOKEN=test-token \ + FAKE_MODE_PRESENT="$mode_present" \ + FAKE_MODE="$mode" \ + CTRLD_POSTINSTALL_PLIST="$plist" \ + CTRLD_POSTINSTALL_CTRLD="$bin/ctrld" \ + CTRLD_POSTINSTALL_PREFS="$prefs" \ + "$postinstall" >"$output" 2>&1 + + printf '%s\n' "$case_dir" +} + +case_dir=$(run_case fresh-legacy 0 0 '') +assert_contains 'ctrld start --cd-org=test-token' "$case_dir/calls" +assert_not_contains '--intercept-mode' "$case_dir/calls" + +case_dir=$(run_case fresh-standard 0 1 standard) +assert_contains 'ctrld start --cd-org=test-token' "$case_dir/calls" +assert_not_contains '--intercept-mode' "$case_dir/calls" + +case_dir=$(run_case fresh-intercept 0 1 intercept-dns) +assert_contains 'ctrld start --cd-org=test-token --intercept-mode dns' "$case_dir/calls" + +case_dir=$(run_case upgrade-legacy 1 0 '') +assert_contains 'launchctl load' "$case_dir/calls" +assert_not_contains 'ctrld start' "$case_dir/calls" + +case_dir=$(run_case upgrade-standard 1 1 standard) +assert_contains 'ctrld start --intercept-mode off' "$case_dir/calls" +assert_not_contains 'launchctl load' "$case_dir/calls" + +case_dir=$(run_case upgrade-intercept 1 1 intercept-dns) +assert_contains 'ctrld start --intercept-mode dns' "$case_dir/calls" +assert_not_contains 'launchctl load' "$case_dir/calls" + +printf 'PASS: pkg postinstall preserves legacy mode and applies standard/intercept-dns policy\n' From 30acb846caf0dc3e7eb1f97782c91853f48d8406 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 21 Aug 2026 15:17:43 +0700 Subject: [PATCH 15/16] Bump staticcheck-action to v1.4.1 While at it, also removing the unmatched //lint line. --- .github/workflows/ci.yml | 6 +++--- cmd/cli/service.go | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a81bc7b..2bcc0dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: fail-fast: false matrix: os: ["windows-latest", "ubuntu-latest", "macOS-latest"] - go: ["1.25.x"] + go: ["1.26.x"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -19,8 +19,8 @@ jobs: with: go-version: ${{ matrix.go }} - run: "go test -race ./..." - - uses: dominikh/staticcheck-action@v1.4.0 + - uses: dominikh/staticcheck-action@v1.4.1 with: - version: "2026.1" + version: "2026.2" install-go: false cache-key: ${{ matrix.go }} diff --git a/cmd/cli/service.go b/cmd/cli/service.go index 3611b68..0cad94f 100644 --- a/cmd/cli/service.go +++ b/cmd/cli/service.go @@ -166,7 +166,6 @@ func ensureSystemdKillMode(r io.Reader) (opts []*unit.UnitOption, change bool) { // channel, so it reports this comparison as always true. On success the // lexer sends nothing and closes the channel, so the receive yields a nil // error and this branch is not taken. - //lint:ignore SA4023 upstream delivers a nil error by closing the channel if err != nil { mainLog.Load().Error().Err(err).Msg("failed to deserialize options") return From d78e9bcf5b3ad574ec6e7fcf9fe7e4a0f10e27fc Mon Sep 17 00:00:00 2001 From: Anthony Wong Date: Fri, 21 Aug 2026 09:56:45 -0400 Subject: [PATCH 16/16] fix(cli): treat explicit intercept-mode off as final in listener setup tryUpdateListenerConfig treated an explicit --intercept-mode off the same as an empty flag and fell back to the persisted config value. run() selects the listener strategy before it clears the persisted mode, so the first start after a revert to standard mode selected the intercept strategy from a stale dns/hard value while setDNS kept interception off. Extract the resolution into listenerInterceptMode and make an explicit off final, the same contract as setDNS. Add a regression test that fails without the fix. --- cmd/cli/cli.go | 17 +++++++++---- cmd/cli/listener_intercept_mode_test.go | 34 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 cmd/cli/listener_intercept_mode_test.go diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index cf8c943..c497678 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -1564,6 +1564,17 @@ func isExplicitInterceptListener(ip string, port int) bool { return !(ip == "127.0.0.1" && port == 53) } +// listenerInterceptMode resolves the mode that selects the listener binding +// strategy. An explicit "off" is final here, the same as in setDNS. A fallback +// to the config value would select the intercept strategy from a stale +// persisted mode on the first start after a revert to standard mode. +func listenerInterceptMode(cfg *ctrld.Config) string { + if interceptMode == "" { + return cfg.Service.InterceptMode + } + return interceptMode +} + // tryUpdateListenerConfig tries updating listener config with a working one. // If fatal is true, and there's listen address conflicted, the function do // fatal error. @@ -1573,13 +1584,9 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti // 1. If config has explicit non-default IP:port, use exactly that // 2. Otherwise: try 127.0.0.1:53, then 127.0.0.1:5354, then fatal // This bypasses the full cd-mode listener probing loop entirely. - // Check interceptMode (CLI flag) first, then fall back to config value. // dnsIntercept bool is derived later in prog.run(), but we need to know // the intercept mode here to select the right listener probing strategy. - im := interceptMode - if im == "" || im == "off" { - im = cfg.Service.InterceptMode - } + im := listenerInterceptMode(cfg) if (im == "dns" || im == "hard") && runtime.GOOS == "darwin" { return tryUpdateListenerConfigIntercept(cfg, notifyFunc, fatal) } diff --git a/cmd/cli/listener_intercept_mode_test.go b/cmd/cli/listener_intercept_mode_test.go new file mode 100644 index 0000000..b762432 --- /dev/null +++ b/cmd/cli/listener_intercept_mode_test.go @@ -0,0 +1,34 @@ +package cli + +import ( + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func TestListenerInterceptModeExplicitOff(t *testing.T) { + oldIntercept := interceptMode + t.Cleanup(func() { interceptMode = oldIntercept }) + + cfg := &ctrld.Config{} + cfg.Service.InterceptMode = "dns" + + tests := []struct { + name string + flag string + want string + }{ + {name: "explicit off is final", flag: "off", want: "off"}, + {name: "empty flag falls back to config", flag: "", want: "dns"}, + {name: "explicit dns wins over config", flag: "dns", want: "dns"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + interceptMode = tc.flag + if got := listenerInterceptMode(cfg); got != tc.want { + t.Fatalf("listenerInterceptMode() = %q, want %q", got, tc.want) + } + }) + } +}