ctrld/cmd/cli/dns_intercept_darwin_test.go

//go:build darwin

package cli

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"tailscale.com/net/netmon"

	"github.com/Control-D-Inc/ctrld"
)

// =============================================================================
// buildPFAnchorRules tests
// =============================================================================

func TestPFBuildAnchorRules_Basic(t *testing.T) {
	p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}}}
	rules := p.buildPFAnchorRules(nil)

	// rdr (translation) must come before pass (filtering)
	rdrIdx := strings.Index(rules, "rdr on lo0 inet proto udp")
	passRouteIdx := strings.Index(rules, "pass out quick on ! lo0 route-to lo0 inet proto udp")
	passInIdx := strings.Index(rules, "pass in quick on lo0 reply-to lo0")

	if rdrIdx < 0 {
		t.Fatal("missing rdr rule")
	}
	if passRouteIdx < 0 {
		t.Fatal("missing pass out route-to rule")
	}
	if passInIdx < 0 {
		t.Fatal("missing pass in on lo0 rule")
	}
	if rdrIdx >= passRouteIdx {
		t.Error("rdr rules must come before pass out route-to rules")
	}
	if passRouteIdx >= passInIdx {
		t.Error("pass out route-to must come before pass in on lo0")
	}

	// Both UDP and TCP rdr rules
	if !strings.Contains(rules, "proto udp") || !strings.Contains(rules, "proto tcp") {
		t.Error("must have both UDP and TCP rdr rules")
	}
}

func TestPFBuildAnchorRules_WithVPNServers(t *testing.T) {
	p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}}}
	vpnServers := []vpnDNSExemption{
		{Server: "10.8.0.1"},
		{Server: "10.8.0.2"},
	}
	rules := p.buildPFAnchorRules(vpnServers)

	// VPN exemption rules must appear
	for _, s := range vpnServers {
		if !strings.Contains(rules, s.Server) {
			t.Errorf("missing VPN exemption for %s", s.Server)
		}
	}

	// VPN exemptions must come before route-to
	exemptIdx := strings.Index(rules, "10.8.0.1 port 53 group")
	routeIdx := strings.Index(rules, "pass out quick on ! lo0 route-to lo0 inet proto udp")
	if exemptIdx < 0 {
		t.Fatal("missing VPN exemption rule for 10.8.0.1")
	}
	if routeIdx < 0 {
		t.Fatal("missing route-to rule")
	}
	if exemptIdx >= routeIdx {
		t.Error("VPN exemptions must come before route-to rules")
	}
}

func TestPFBuildAnchorRules_IPv4AndIPv6VPN(t *testing.T) {
	p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}}}
	vpnServers := []vpnDNSExemption{
		{Server: "10.8.0.1"},
		{Server: "fd00::1"},
	}
	rules := p.buildPFAnchorRules(vpnServers)

	// IPv4 server should use "inet"
	lines := strings.Split(rules, "\n")
	for _, line := range lines {
		if strings.Contains(line, "10.8.0.1") && strings.HasPrefix(line, "pass") {
			if !strings.Contains(line, "inet ") {
				t.Error("IPv4 VPN server rule should contain 'inet'")
			}
			if strings.Contains(line, "inet6") {
				t.Error("IPv4 VPN server rule should not contain 'inet6'")
			}
		}
		if strings.Contains(line, "fd00::1") && strings.HasPrefix(line, "pass") {
			if !strings.Contains(line, "inet6") {
				t.Error("IPv6 VPN server rule should contain 'inet6'")
			}
		}
	}
}

func TestPFBuildAnchorRules_Ordering(t *testing.T) {
	p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}}}
	vpnServers := []vpnDNSExemption{
		{Server: "10.8.0.1"},
	}
	rules := p.buildPFAnchorRules(vpnServers)

	// Verify ordering: rdr → exemptions → route-to → pass in on lo0
	rdrIdx := strings.Index(rules, "rdr on lo0 inet proto udp")
	exemptIdx := strings.Index(rules, "pass out quick on ! lo0 inet proto { udp, tcp } from any to 10.8.0.1 port 53 group _ctrld")
	routeIdx := strings.Index(rules, "pass out quick on ! lo0 route-to lo0 inet proto udp")
	passInIdx := strings.Index(rules, "pass in quick on lo0 reply-to lo0")

	if rdrIdx < 0 || exemptIdx < 0 || routeIdx < 0 || passInIdx < 0 {
		t.Fatalf("missing expected rules: rdr=%d exempt=%d route=%d passIn=%d", rdrIdx, exemptIdx, routeIdx, passInIdx)
	}

	if !(rdrIdx < exemptIdx && exemptIdx < routeIdx && routeIdx < passInIdx) {
		t.Errorf("incorrect rule ordering: rdr(%d) < exempt(%d) < route(%d) < passIn(%d)", rdrIdx, exemptIdx, routeIdx, passInIdx)
	}
}

// TestPFBuildAnchorRules_FallbackPort verifies that when the listener falls back
// to an alternate local port (e.g. 5354 because mDNSResponder owns *:53), the pf
// rdr rules redirect DNS to the ACTUAL bound port, not the configured default 53.
// Regression test for #551: pf redirected to a dead port after listener fallback.
func TestPFBuildAnchorRules_FallbackPort(t *testing.T) {
	// Configured/generated listener is 127.0.0.1:53, but the runtime bound port is 5354.
	p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}}}}
	rules := p.buildPFAnchorRules(nil)

	// rdr must redirect to the actual bound port 5354.
	if !strings.Contains(rules, "rdr on lo0 inet proto udp from any to ! 127.0.0.1 port 53 -> 127.0.0.1 port 5354") {
		t.Errorf("UDP rdr must redirect to bound port 5354, got:\n%s", rules)
	}
	if !strings.Contains(rules, "rdr on lo0 inet proto tcp from any to ! 127.0.0.1 port 53 -> 127.0.0.1 port 5354") {
		t.Errorf("TCP rdr must redirect to bound port 5354, got:\n%s", rules)
	}

	// The rdr redirect target must NOT point at the dead default port 53.
	// Match the exact port at line end so "port 5354" is not a false positive.
	if strings.Contains(rules, "-> 127.0.0.1 port 53\n") {
		t.Errorf("rdr must not redirect to dead port 53 after fallback, got:\n%s", rules)
	}

	// The inbound accept rule must also target the actual bound port.
	if !strings.Contains(rules, "127.0.0.1 port 5354") {
		t.Errorf("pass in rule must reference bound port 5354, got:\n%s", rules)
	}
}

// TestPFAddressFamily tests the pfAddressFamily helper.
func TestPFAddressFamily(t *testing.T) {
	tests := []struct {
		ip   string
		want string
	}{
		{"10.0.0.1", "inet"},
		{"192.168.1.1", "inet"},
		{"127.0.0.1", "inet"},
		{"::1", "inet6"},
		{"fd00::1", "inet6"},
		{"2001:db8::1", "inet6"},
	}
	for _, tt := range tests {
		if got := pfAddressFamily(tt.ip); got != tt.want {
			t.Errorf("pfAddressFamily(%q) = %q, want %q", tt.ip, got, tt.want)
		}
	}
}

func TestIsResourceExhaustion(t *testing.T) {
	tests := []struct {
		name   string
		err    error
		output []byte
		want   bool
	}{
		{
			name: "exec start failure",
			err:  errors.New("fork/exec /sbin/pfctl: resource temporarily unavailable"),
			want: true,
		},
		{
			name:   "fd exhaustion from stderr output",
			err:    errors.New("exit status 1"),
			output: []byte("pfctl: Pipe: Too many open files"),
			want:   true,
		},
		{
			name: "process exhaustion from wrapped restore error",
			err:  errors.New("failed to dump running filter rules: exit status 1 (output: too many processes)"),
			want: true,
		},
		{
			name:   "ordinary pf syntax failure",
			err:    errors.New("exit status 1"),
			output: []byte("pfctl: syntax error"),
			want:   false,
		},
		{
			name: "nil error and empty output",
			want: false,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := isResourceExhaustion(tt.err, tt.output); got != tt.want {
				t.Fatalf("isResourceExhaustion() = %v, want %v", got, tt.want)
			}
		})
	}
}

func stubPFAnchorCheckCommand(t *testing.T, outputs map[string]string) {
	t.Helper()
	original := runPFAnchorCheckCommand
	runPFAnchorCheckCommand = func(args ...string) ([]byte, error) {
		key := strings.Join(args, " ")
		output, ok := outputs[key]
		if !ok {
			t.Fatalf("unexpected pf anchor check command: pfctl %s", key)
		}
		return []byte(output), nil
	}
	t.Cleanup(func() {
		runPFAnchorCheckCommand = original
	})
}

func TestEnsurePFAnchorActiveRecentRestoreWithIntactRulesDoesNotStabilize(t *testing.T) {
	stubPFAnchorCheckCommand(t, map[string]string{
		"-sn":                       `rdr-anchor "com.controld.ctrld"`,
		"-sr":                       `anchor "com.controld.ctrld"`,
		"-a com.controld.ctrld -sr": "pass in quick on lo0",
		"-a com.controld.ctrld -sn": "rdr on lo0",
	})

	p := &prog{
		dnsInterceptState: &pfState{},
		stopCh:            make(chan struct{}),
	}
	restoredAt := time.Now().Add(-time.Second).UnixMilli()
	p.pfLastRestoreTime.Store(restoredAt)

	if result := p.ensurePFAnchorActive(); result != pfAnchorCheckIntact {
		t.Fatalf("intact rules result = %v, want intact", result)
	}
	if p.pfBackoffMultiplier.Load() != 0 {
		t.Fatalf("intact rules incremented backoff to %d", p.pfBackoffMultiplier.Load())
	}
	if p.pfStabilizing.Load() {
		t.Fatal("intact rules must not enter stabilization")
	}
	if got := p.pfLastRestoreTime.Load(); got != restoredAt {
		t.Fatalf("intact check changed restore timestamp: got %d, want %d", got, restoredAt)
	}
}

func TestEnsurePFAnchorActiveCheckFailureIsNotIntact(t *testing.T) {
	original := runPFAnchorCheckCommand
	runPFAnchorCheckCommand = func(...string) ([]byte, error) {
		return nil, errors.New("pfctl unavailable")
	}
	t.Cleanup(func() { runPFAnchorCheckCommand = original })

	p := &prog{dnsInterceptState: &pfState{}}
	if result := p.ensurePFAnchorActive(); result != pfAnchorCheckFailed {
		t.Fatalf("failed PF inspection result = %v, want failed", result)
	}
}

func TestEnsurePFAnchorActiveRecentActualWipeStartsStabilization(t *testing.T) {
	stubPFAnchorCheckCommand(t, map[string]string{
		"-sn": "",
	})

	stopCh := make(chan struct{})
	close(stopCh)
	p := &prog{
		dnsInterceptState: &pfState{},
		stopCh:            stopCh,
	}
	restoredAt := time.Now().Add(-time.Second).UnixMilli()
	p.pfLastRestoreTime.Store(restoredAt)

	if result := p.ensurePFAnchorActive(); result != pfAnchorCheckDeferred {
		t.Fatalf("recent repeated wipe result = %v, want deferred", result)
	}
	if got := p.pfBackoffMultiplier.Load(); got != 1 {
		t.Fatalf("recent repeated wipe backoff = %d, want 1", got)
	}
	if got := p.pfLastRestoreTime.Load(); got != restoredAt {
		t.Fatalf("deferred restore changed restore timestamp: got %d, want %d", got, restoredAt)
	}
	deadline := time.Now().Add(time.Second)
	for p.pfStabilizing.Load() && time.Now().Before(deadline) {
		time.Sleep(time.Millisecond)
	}
	if p.pfStabilizing.Load() {
		t.Fatal("stabilization goroutine did not observe closed stop channel")
	}
}

func TestDNSInterceptIgnoredChangeReconcileDue(t *testing.T) {
	p := &prog{}
	start := time.Unix(1_000_000, 0)

	if !p.dnsInterceptIgnoredChangeReconcileDue(start) {
		t.Fatal("first ignored change must reconcile immediately")
	}
	if p.dnsInterceptIgnoredChangeReconcileDue(start.Add(pfIgnoredChangeReconcileInterval - time.Millisecond)) {
		t.Fatal("ignored changes inside the interval must be coalesced")
	}
	if !p.dnsInterceptIgnoredChangeReconcileDue(start.Add(pfIgnoredChangeReconcileInterval)) {
		t.Fatal("continuous ignored changes must reconcile again at the interval boundary")
	}
}

func TestIgnoredNetworkChangeCallbackBoundsWorkWithoutBurningStabilizedSlot(t *testing.T) {
	outputs := map[string]string{
		"-sn":                       `rdr-anchor "com.controld.ctrld"`,
		"-sr":                       `anchor "com.controld.ctrld"`,
		"-a com.controld.ctrld -sr": "pass in quick on lo0",
		"-a com.controld.ctrld -sn": "rdr on lo0",
	}
	originalCheck := runPFAnchorCheckCommand
	pfChecks := 0
	runPFAnchorCheckCommand = func(args ...string) ([]byte, error) {
		key := strings.Join(args, " ")
		output, ok := outputs[key]
		if !ok {
			t.Fatalf("unexpected pf anchor check command: pfctl %s", key)
		}
		if key == "-sn" {
			pfChecks++
		}
		return []byte(output), nil
	}
	originalDiscover := discoverTunnelInterfacesForReconcile
	discoverTunnelInterfacesForReconcile = func() []string { return nil }
	t.Cleanup(func() {
		runPFAnchorCheckCommand = originalCheck
		discoverTunnelInterfacesForReconcile = originalDiscover
	})

	refreshes := 0
	vpnDNS := newVPNDNSManager(nil)
	vpnDNS.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig {
		refreshes++
		return nil
	}
	p := &prog{dnsInterceptState: &pfState{}, vpnDNS: vpnDNS}
	t.Cleanup(func() {
		p.pfDelayedRecheckMu.Lock()
		defer p.pfDelayedRecheckMu.Unlock()
		for _, timer := range p.pfDelayedRecheckTimers {
			if timer != nil {
				timer.Stop()
			}
		}
	})

	delta := &netmon.ChangeDelta{
		Old: &netmon.State{Interface: map[string]netmon.Interface{}},
		New: &netmon.State{Interface: map[string]netmon.Interface{}},
	}
	start := time.Unix(1_000_000, 0)
	p.handleDNSInterceptIgnoredNetworkChange(delta, start)
	if pfChecks != 1 || refreshes != 1 {
		t.Fatalf("first ignored delta work: pf checks=%d refreshes=%d, want 1 each", pfChecks, refreshes)
	}

	p.pfStabilizing.Store(true)
	p.handleDNSInterceptIgnoredNetworkChange(delta, start.Add(pfIgnoredChangeReconcileInterval))
	if pfChecks != 1 || refreshes != 1 {
		t.Fatalf("stabilized delta ran leading reconciliation: pf checks=%d refreshes=%d", pfChecks, refreshes)
	}

	p.pfStabilizing.Store(false)
	resumeAt := start.Add(pfIgnoredChangeReconcileInterval + time.Millisecond)
	p.handleDNSInterceptIgnoredNetworkChange(delta, resumeAt)
	if pfChecks != 2 || refreshes != 2 {
		t.Fatalf("first post-stabilization delta did not reconcile immediately: pf checks=%d refreshes=%d", pfChecks, refreshes)
	}

	for i := 1; i <= 8; i++ {
		p.handleDNSInterceptIgnoredNetworkChange(delta, resumeAt.Add(time.Duration(i)*100*time.Millisecond))
	}
	if pfChecks != 2 || refreshes != 2 {
		t.Fatalf("ignored delta burst was not coalesced: pf checks=%d refreshes=%d", pfChecks, refreshes)
	}

	p.handleDNSInterceptIgnoredNetworkChange(delta, resumeAt.Add(pfIgnoredChangeReconcileInterval))
	if pfChecks != 3 || refreshes != 3 {
		t.Fatalf("interval boundary did not reconcile: pf checks=%d refreshes=%d, want 3 each", pfChecks, refreshes)
	}
}

func TestRestorePFAnchorFailureIsNotReportedOrTimestamped(t *testing.T) {
	originalReference := ensurePFAnchorReferenceForRestore
	originalRebuild := rebuildPFAnchorRulesForReconcile
	ensurePFAnchorReferenceForRestore = func(*prog) error { return nil }
	rebuildPFAnchorRulesForReconcile = func(*prog, []vpnDNSExemption) ([]string, error) {
		return nil, errors.New("pf load failed")
	}
	t.Cleanup(func() {
		ensurePFAnchorReferenceForRestore = originalReference
		rebuildPFAnchorRulesForReconcile = originalRebuild
	})

	p := &prog{dnsInterceptState: &pfState{}}
	if result := p.restorePFAnchor("test"); result != pfAnchorCheckFailed {
		t.Fatalf("failed restore result = %v, want failed", result)
	}
	if got := p.pfLastRestoreTime.Load(); got != 0 {
		t.Fatalf("failed restore changed timestamp to %d", got)
	}
	if len(p.lastTunnelIfaces) != 0 {
		t.Fatalf("failed restore committed tunnel state: %v", p.lastTunnelIfaces)
	}
}

func TestPFStabilizationTimeoutReturnsOwnershipToDelayedRecovery(t *testing.T) {
	p := &prog{dnsInterceptState: &pfState{}}
	p.pfStabilizing.Store(true)
	p.pfStabilizationLoopWithMaxWait(t.Context(), time.Hour, 25*time.Millisecond)

	if p.pfStabilizing.Load() {
		t.Fatal("stabilization retained ownership after the maximum wait")
	}
	p.pfDelayedRecheckMu.Lock()
	timers := append([]*time.Timer(nil), p.pfDelayedRecheckTimers...)
	p.pfDelayedRecheckTimers = nil
	p.pfDelayedRecheckMu.Unlock()
	if len(timers) != 2 {
		t.Fatalf("expected bounded timeout to schedule delayed recovery, got %d timers", len(timers))
	}
	for _, timer := range timers {
		timer.Stop()
	}
}

func TestStopDNSInterceptWaitsForInFlightPFMutation(t *testing.T) {
	binDir := t.TempDir()
	pfctlPath := filepath.Join(binDir, "pfctl")
	if err := os.WriteFile(pfctlPath, []byte("#!/bin/sh\nexit 0\n"), 0755); err != nil {
		t.Fatal(err)
	}
	t.Setenv("PATH", binDir+":"+os.Getenv("PATH"))

	anchorFile := filepath.Join(t.TempDir(), "anchor")
	if err := os.WriteFile(anchorFile, []byte("rules"), 0600); err != nil {
		t.Fatal(err)
	}
	p := &prog{dnsInterceptState: &pfState{anchorName: pfAnchorName, anchorFile: anchorFile}}
	p.pfEnsureRunning.Store(true)

	revoked := make(chan struct{})
	originalRevokedHook := pfShutdownStateRevokedForTest
	pfShutdownStateRevokedForTest = func() { close(revoked) }
	t.Cleanup(func() { pfShutdownStateRevokedForTest = originalRevokedHook })

	done := make(chan error, 1)
	go func() { done <- p.stopDNSIntercept() }()

	select {
	case <-revoked:
	case <-time.After(time.Second):
		t.Fatal("shutdown did not revoke PF lifecycle state before waiting")
	}
	select {
	case err := <-done:
		t.Fatalf("shutdown completed before in-flight PF owner released: %v", err)
	case <-time.After(25 * time.Millisecond):
	}

	p.pfEnsureRunning.Store(false)
	if err := <-done; err != nil {
		t.Fatalf("stopDNSIntercept() error: %v", err)
	}
	if _, err := os.Stat(anchorFile); !os.IsNotExist(err) {
		t.Fatalf("anchor file remained after serialized shutdown: %v", err)
	}
}

func TestPostStabilizationReconcileRetainsOwnershipAndForcesRebuild(t *testing.T) {
	stubPFAnchorCheckCommand(t, map[string]string{
		"-sn":                       `rdr-anchor "com.controld.ctrld"`,
		"-sr":                       `anchor "com.controld.ctrld"`,
		"-a com.controld.ctrld -sr": "pass in quick on lo0",
		"-a com.controld.ctrld -sn": "rdr on lo0",
	})
	originalRestore := restorePFAnchorForReconcile
	calls := 0
	restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult {
		calls++
		return pfAnchorCheckRestored
	}
	t.Cleanup(func() { restorePFAnchorForReconcile = originalRestore })

	p := &prog{
		dnsInterceptState:      &pfState{},
		pendingTunnelIfaces:    []string{"utun9"},
		hasPendingTunnelIfaces: true,
	}
	p.pfStabilizing.Store(true)
	if result := p.reconcilePFAnchorAfterStabilization(); result != pfAnchorCheckRestored {
		t.Fatalf("post-stabilization result = %v, want restored", result)
	}
	if calls != 1 {
		t.Fatalf("post-stabilization restore calls = %d, want 1", calls)
	}
	if !p.pfStabilizing.Load() {
		t.Fatal("post-stabilization reconcile released loop ownership")
	}
	if p.pfBackoffMultiplier.Load() != 0 {
		t.Fatalf("post-stabilization reconcile changed backoff to %d", p.pfBackoffMultiplier.Load())
	}
}

func TestPostStabilizationIntactWithoutPendingAvoidsRebuild(t *testing.T) {
	stubPFAnchorCheckCommand(t, map[string]string{
		"-sn":                       `rdr-anchor "com.controld.ctrld"`,
		"-sr":                       `anchor "com.controld.ctrld"`,
		"-a com.controld.ctrld -sr": "pass in quick on lo0",
		"-a com.controld.ctrld -sn": "rdr on lo0",
	})
	originalRestore := restorePFAnchorForReconcile
	calls := 0
	restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult {
		calls++
		return pfAnchorCheckRestored
	}
	t.Cleanup(func() { restorePFAnchorForReconcile = originalRestore })

	p := &prog{dnsInterceptState: &pfState{}}
	p.pfStabilizing.Store(true)
	if result := p.reconcilePFAnchorAfterStabilization(); result != pfAnchorCheckIntact {
		t.Fatalf("post-stabilization result = %v, want intact", result)
	}
	if calls != 0 {
		t.Fatalf("intact post-stabilization anchor rebuilt %d times", calls)
	}
	if !p.pfStabilizing.Load() {
		t.Fatal("intact post-stabilization reconcile released loop ownership")
	}
}

func TestTunnelRemovalFailureRetriesBeforeCommittingBaseline(t *testing.T) {
	originalDiscover := discoverTunnelInterfacesForReconcile
	originalRestore := restorePFAnchorForReconcile
	current := []string{}
	discoverTunnelInterfacesForReconcile = func() []string {
		return append([]string(nil), current...)
	}
	calls := 0
	restorePFAnchorForReconcile = func(p *prog, _ string) pfAnchorCheckResult {
		calls++
		if calls == 1 {
			return pfAnchorCheckFailed
		}
		p.commitPFReconcileState(current)
		return pfAnchorCheckRestored
	}
	t.Cleanup(func() {
		discoverTunnelInterfacesForReconcile = originalDiscover
		restorePFAnchorForReconcile = originalRestore
	})

	p := &prog{
		dnsInterceptState: &pfState{},
		lastTunnelIfaces:  []string{"utun7"},
	}
	if !p.checkTunnelInterfaceChanges() {
		t.Fatal("first tunnel removal was not detected")
	}
	if !stringSlicesEqual(p.lastTunnelIfaces, []string{"utun7"}) {
		t.Fatalf("failed removal committed baseline: %v", p.lastTunnelIfaces)
	}
	if !p.hasPendingTunnelReconcile() {
		t.Fatal("failed removal did not retain desired tunnel state for retry")
	}
	if !p.checkTunnelInterfaceChanges() {
		t.Fatal("failed tunnel removal was not retried")
	}
	if len(p.lastTunnelIfaces) != 0 {
		t.Fatalf("successful retry did not commit empty tunnel baseline: %v", p.lastTunnelIfaces)
	}
	if calls != 2 {
		t.Fatalf("restore calls = %d, want 2", calls)
	}
}

func TestPendingTunnelStateRetriesAfterStabilization(t *testing.T) {
	originalDiscover := discoverTunnelInterfacesForReconcile
	originalRestore := restorePFAnchorForReconcile
	current := []string{}
	discoverTunnelInterfacesForReconcile = func() []string { return nil }
	calls := 0
	restorePFAnchorForReconcile = func(p *prog, _ string) pfAnchorCheckResult {
		calls++
		p.commitPFReconcileState(current)
		return pfAnchorCheckRestored
	}
	t.Cleanup(func() {
		discoverTunnelInterfacesForReconcile = originalDiscover
		restorePFAnchorForReconcile = originalRestore
	})

	p := &prog{
		dnsInterceptState:      &pfState{},
		lastTunnelIfaces:       []string{"utun7"},
		pendingTunnelIfaces:    current,
		hasPendingTunnelIfaces: true,
	}
	if !p.checkTunnelInterfaceChanges() {
		t.Fatal("pending tunnel removal was not retried after stabilization")
	}
	if calls != 1 || len(p.lastTunnelIfaces) != 0 || p.hasPendingTunnelReconcile() {
		t.Fatalf("pending retry result: calls=%d baseline=%v pending=%v", calls, p.lastTunnelIfaces, p.hasPendingTunnelReconcile())
	}
}

func TestTunnelReconcileHonorsPFExecBackoff(t *testing.T) {
	originalDiscover := discoverTunnelInterfacesForReconcile
	originalRestore := restorePFAnchorForReconcile
	current := []string{}
	discoverTunnelInterfacesForReconcile = func() []string { return nil }
	calls := 0
	restorePFAnchorForReconcile = func(p *prog, _ string) pfAnchorCheckResult {
		calls++
		p.commitPFReconcileState(current)
		return pfAnchorCheckRestored
	}
	t.Cleanup(func() {
		discoverTunnelInterfacesForReconcile = originalDiscover
		restorePFAnchorForReconcile = originalRestore
	})

	p := &prog{dnsInterceptState: &pfState{}, lastTunnelIfaces: []string{"utun7"}}
	p.pfExecBackoffUntil.Store(time.Now().Add(time.Minute).UnixMilli())
	if !p.checkTunnelInterfaceChanges() {
		t.Fatal("tunnel removal was not detected during PF exec backoff")
	}
	if calls != 0 || !stringSlicesEqual(p.lastTunnelIfaces, []string{"utun7"}) {
		t.Fatalf("PF restore ran during exec backoff: calls=%d baseline=%v", calls, p.lastTunnelIfaces)
	}
	if p.checkTunnelInterfaceChanges() {
		t.Fatal("identical deferred tunnel retry bypassed the ignored-event limiter")
	}
	p.pfExecBackoffUntil.Store(0)
	if !p.checkTunnelInterfaceChanges() || calls != 1 || len(p.lastTunnelIfaces) != 0 {
		t.Fatalf("tunnel removal did not retry after backoff: calls=%d baseline=%v", calls, p.lastTunnelIfaces)
	}
}

func TestTunnelRapidReversalClearsUnappliedPendingState(t *testing.T) {
	originalDiscover := discoverTunnelInterfacesForReconcile
	discoverTunnelInterfacesForReconcile = func() []string { return nil }
	t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover })

	p := &prog{
		dnsInterceptState:      &pfState{},
		pendingTunnelIfaces:    []string{"utun9"},
		hasPendingTunnelIfaces: true,
	}
	p.pfStabilizing.Store(true)
	if !p.checkTunnelInterfaceChanges() {
		t.Fatal("rapid tunnel reversal was not observed")
	}
	if p.hasPendingTunnelReconcile() || len(p.lastTunnelIfaces) != 0 {
		t.Fatalf("rapid reversal left unapplied tunnel state: baseline=%v pending=%v", p.lastTunnelIfaces, p.hasPendingTunnelReconcile())
	}
}

func TestTunnelAdditionIsCoalescedUntilSuccessfulRebuild(t *testing.T) {
	originalDiscover := discoverTunnelInterfacesForReconcile
	current := []string{"utun9"}
	discoverTunnelInterfacesForReconcile = func() []string {
		return append([]string(nil), current...)
	}
	t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover })

	p := &prog{dnsInterceptState: &pfState{}}
	p.pfStabilizing.Store(true)
	if !p.checkTunnelInterfaceChanges() {
		t.Fatal("new tunnel was not detected")
	}
	if p.checkTunnelInterfaceChanges() {
		t.Fatal("identical pending tunnel state was not coalesced")
	}
	if len(p.lastTunnelIfaces) != 0 {
		t.Fatalf("pending tunnel was committed before PF rebuild: %v", p.lastTunnelIfaces)
	}
	if !p.hasPendingTunnelReconcile() {
		t.Fatal("new tunnel was not retained as pending")
	}

	p.commitPFReconcileState(current)
	if !stringSlicesEqual(p.lastTunnelIfaces, current) {
		t.Fatalf("successful rebuild baseline = %v, want %v", p.lastTunnelIfaces, current)
	}
	if p.hasPendingTunnelReconcile() {
		t.Fatal("successful rebuild did not clear pending tunnel state")
	}
}

// TestVPNDNSRefreshDeferredWhileStabilizing covers the ignored network-change path,
// which can trigger a VPN DNS refresh from outside stabilization.
//
// A refresh rebuilds and reloads the pf anchor. Stabilization owns pf while a VPN's
// ruleset is still settling, so refreshing then is the mutual-overwrite collision
// stabilization exists to prevent - and these deltas arrive exactly when a VPN is
// coming up. Deferring is safe: checkTunnelInterfaceChanges keeps the observation
// pending, so the transition is retried afterwards.
//
// The watchdog tick carries the same guard for the same reason; it is not driven here
// because that would mean running its 30s loop.
func TestVPNDNSRefreshDeferredWhileStabilizing(t *testing.T) {
	newProg := func(t *testing.T, refreshes *int, tunnels []string) *prog {
		t.Helper()
		outputs := map[string]string{
			"-sn":                       `rdr-anchor "com.controld.ctrld"`,
			"-sr":                       `anchor "com.controld.ctrld"`,
			"-a com.controld.ctrld -sr": "pass in quick on lo0",
			"-a com.controld.ctrld -sn": "rdr on lo0",
		}
		originalCheck := runPFAnchorCheckCommand
		runPFAnchorCheckCommand = func(args ...string) ([]byte, error) {
			output, ok := outputs[strings.Join(args, " ")]
			if !ok {
				return nil, fmt.Errorf("unexpected pf anchor check command")
			}
			return []byte(output), nil
		}
		// Discovery reports no tunnels. With a seeded baseline that is a removal, which
		// checkTunnelInterfaceChanges reports as a change without touching pf while
		// stabilizing - so this fixture never reaches a real pfctl write.
		originalDiscover := discoverTunnelInterfacesForReconcile
		discoverTunnelInterfacesForReconcile = func() []string { return nil }
		t.Cleanup(func() {
			runPFAnchorCheckCommand = originalCheck
			discoverTunnelInterfacesForReconcile = originalDiscover
		})

		vpnDNS := newVPNDNSManager(nil)
		vpnDNS.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig {
			*refreshes++
			return nil
		}
		p := &prog{dnsInterceptState: &pfState{}, vpnDNS: vpnDNS, lastTunnelIfaces: tunnels}
		t.Cleanup(func() {
			p.pfDelayedRecheckMu.Lock()
			defer p.pfDelayedRecheckMu.Unlock()
			for _, timer := range p.pfDelayedRecheckTimers {
				if timer != nil {
					timer.Stop()
				}
			}
		})
		return p
	}
	delta := func() *netmon.ChangeDelta {
		return &netmon.ChangeDelta{
			Old: &netmon.State{Interface: map[string]netmon.Interface{}},
			New: &netmon.State{Interface: map[string]netmon.Interface{}},
		}
	}

	t.Run("tunnel change during stabilization does not refresh", func(t *testing.T) {
		refreshes := 0
		// Seeded baseline plus empty discovery = a tunnel transition to report, so the
		// refresh is eligible on everything except the stabilization guard.
		p := newProg(t, &refreshes, []string{"utun9"})
		p.pfStabilizing.Store(true)

		p.handleDNSInterceptIgnoredNetworkChange(delta(), time.Unix(1_000_000, 0))

		if refreshes != 0 {
			t.Errorf("refreshed %d time(s) while stabilizing — that rebuilds the anchor under a settling VPN ruleset", refreshes)
		}
	})

	t.Run("refresh still happens outside stabilization", func(t *testing.T) {
		refreshes := 0
		p := newProg(t, &refreshes, nil)

		p.handleDNSInterceptIgnoredNetworkChange(delta(), time.Unix(1_000_000, 0))

		if refreshes == 0 {
			t.Error("no refresh outside stabilization — the guard must defer, not disable")
		}
	})
}

// TestExemptVPNDNSServersDeferredWhileStabilizing checks the mutation point itself,
// not just the call sites: any future caller reaching it during stabilization is
// refused before the anchor is rewritten.
//
// It returns before pfEnsureRunning is taken and before any pfctl work, so this drives
// the real function without touching the host's pf state.
func TestExemptVPNDNSServersDeferredWhileStabilizing(t *testing.T) {
	p := &prog{dnsInterceptState: &pfState{}}
	p.pfStabilizing.Store(true)

	err := p.exemptVPNDNSServers([]vpnDNSExemption{{Server: "192.168.1.1"}})
	if err == nil {
		t.Fatal("exemption applied while stabilizing — that rewrites the anchor under a settling VPN ruleset")
	}
	if !strings.Contains(err.Error(), "stabilization") {
		t.Errorf("error does not name the reason: %v", err)
	}
	// The refusal must happen before the reconcile latch is claimed, or a deferral
	// would lock out the reconcile that runs once stabilization ends.
	if p.pfEnsureRunning.Load() {
		t.Error("pfEnsureRunning was left held by a deferred exemption")
	}
}

// stubStabilizationProbe replaces the post-stabilization verification seams and returns
// counters for probe and forced-reload calls.
func stubStabilizationProbe(t *testing.T, probeResults []bool, reloadOK bool) (probes, reloads *int) {
	t.Helper()
	originalProbe, originalReload := probePFInterceptFn, forceReloadPFInterceptFn
	t.Cleanup(func() {
		probePFInterceptFn, forceReloadPFInterceptFn = originalProbe, originalReload
	})
	probeCalls, reloadCalls := 0, 0
	probePFInterceptFn = func(*prog) bool {
		result := false
		if probeCalls < len(probeResults) {
			result = probeResults[probeCalls]
		}
		probeCalls++
		return result
	}
	forceReloadPFInterceptFn = func(*prog) bool {
		reloadCalls++
		return reloadOK
	}
	return &probeCalls, &reloadCalls
}

// TestPostStabilizationVerifiesInterceptionFunctionally is the post-wake continuity
// boundary: the reconcile above it only proves rule text, and QA saw rules intact,
// references intact and post-load verification passed while every query through the system
// resolver timed out. Nothing else probes until the periodic watchdog, because the probe
// monitor stands down while stabilization owns pf, so recovery waited for that tick.
func TestPostStabilizationVerifiesInterceptionFunctionally(t *testing.T) {
	// Probe fails once, then passes after the reload.
	probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true)

	p := &prog{dnsInterceptState: &pfState{}}
	p.verifyInterceptAfterStabilization()

	if *probes != 2 {
		t.Errorf("probe calls = %d, want 2: one to detect and one to confirm the repair", *probes)
	}
	if *reloads != 1 {
		t.Errorf("forced reloads = %d, want exactly 1 bounded repair", *reloads)
	}
}

// TestPostStabilizationProbePassSkipsReload keeps the healthy path free of a pf reload,
// which would flush states and kill in-flight DoH connections for nothing.
func TestPostStabilizationProbePassSkipsReload(t *testing.T) {
	probes, reloads := stubStabilizationProbe(t, []bool{true}, true)

	p := &prog{dnsInterceptState: &pfState{}}
	p.verifyInterceptAfterStabilization()

	if *probes != 1 || *reloads != 0 {
		t.Errorf("probe calls = %d, forced reloads = %d, want 1/0", *probes, *reloads)
	}
}

// TestPostStabilizationRepairIsBounded pins the "one bounded recovery" contract: a probe
// that never passes must not turn into a reload loop here - the watchdog owns retries.
func TestPostStabilizationRepairIsBounded(t *testing.T) {
	probes, reloads := stubStabilizationProbe(t, []bool{false, false, false}, true)

	p := &prog{dnsInterceptState: &pfState{}}
	p.verifyInterceptAfterStabilization()

	if *reloads != 1 {
		t.Errorf("forced reloads = %d, want 1: the repair must not loop", *reloads)
	}
	if *probes != 2 {
		t.Errorf("probe calls = %d, want 2", *probes)
	}
}

// TestPostStabilizationWaitsForAProberThatStandsDown is the interleaving that made
// "skip when the flag is set" wrong. A probe monitor started by an ignored network change
// claims functional-probe ownership and then aborts, because stabilization still owns pf.
// If the verifier treats the claimed flag as "somebody is probing", neither path probes and
// the outage lasts until the next watchdog tick - the exact window this is meant to close.
func TestPostStabilizationWaitsForAProberThatStandsDown(t *testing.T) {
	probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true)

	p := &prog{dnsInterceptState: &pfState{}}
	// Model the monitor's claim-then-abort: ownership is held, then released.
	p.pfMonitorRunning.Store(true)
	released := make(chan struct{})
	go func() {
		time.Sleep(50 * time.Millisecond)
		p.pfMonitorRunning.Store(false)
		close(released)
	}()

	p.verifyInterceptAfterStabilization()
	<-released

	if *probes != 2 {
		t.Errorf("probe calls = %d, want 2: the verifier must wait out a prober that stands down", *probes)
	}
	if *reloads != 1 {
		t.Errorf("forced reloads = %d, want 1", *reloads)
	}
}

// TestPostStabilizationYieldsToAProberThatKeepsProbing is the other half of the handoff:
// when the holder is genuinely working through its probe sequence, the verifier must step
// aside rather than run a second prober against the same pf state.
func TestPostStabilizationYieldsToAProberThatKeepsProbing(t *testing.T) {
	originalWait := pfFunctionalProbeOwnerWait
	pfFunctionalProbeOwnerWait = 30 * time.Millisecond
	t.Cleanup(func() { pfFunctionalProbeOwnerWait = originalWait })

	probes, reloads := stubStabilizationProbe(t, []bool{false}, true)

	p := &prog{dnsInterceptState: &pfState{}}
	p.pfMonitorRunning.Store(true) // held for the whole wait
	p.verifyInterceptAfterStabilization()

	if *probes != 0 || *reloads != 0 {
		t.Errorf("probe calls = %d, forced reloads = %d, want 0/0 while another prober is working", *probes, *reloads)
	}
}

// TestInterceptMonitorDoesNotClaimOwnershipWhileStabilizing pins the source of that race:
// a monitor which cannot do useful work must not take functional-probe ownership on its way
// out, or it starves the post-stabilization verifier.
func TestInterceptMonitorDoesNotClaimOwnershipWhileStabilizing(t *testing.T) {
	probes, reloads := stubStabilizationProbe(t, []bool{false}, true)

	p := &prog{dnsInterceptState: &pfState{}}
	p.pfStabilizing.Store(true)
	if p.interceptProbeMonitorAllowed() {
		t.Fatal("the probe monitor considers itself eligible while stabilization owns pf")
	}

	p.pfInterceptMonitor()

	if *probes != 0 || *reloads != 0 {
		t.Errorf("probe calls = %d, forced reloads = %d, want 0/0 from a monitor that cannot run", *probes, *reloads)
	}
	if !p.claimFunctionalProbeOwner(0) {
		t.Error("the aborted monitor left functional-probe ownership taken; the verifier would skip")
	}
	p.pfMonitorRunning.Store(false)
}

// TestFinishPFStabilizationRunsFunctionalVerification wires the fix to the production
// completion path: deleting the verification call, or reordering it before the reconcile,
// makes this fail.
func TestFinishPFStabilizationRunsFunctionalVerification(t *testing.T) {
	stubPFAnchorCheckCommand(t, map[string]string{
		"-sn":                       `rdr-anchor "com.controld.ctrld"`,
		"-sr":                       `anchor "com.controld.ctrld"`,
		"-a com.controld.ctrld -sr": "pass in quick on lo0",
		"-a com.controld.ctrld -sn": "rdr on lo0",
	})
	originalResolver := initializeOsResolver
	initializeOsResolver = func(bool) []string { return nil }
	t.Cleanup(func() { initializeOsResolver = originalResolver })

	probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true)

	p := &prog{dnsInterceptState: &pfState{}}
	p.pfStabilizing.Store(true)
	p.finishPFStabilization(time.Millisecond)

	if *probes == 0 {
		t.Fatal("stabilization completed without probing functional interception; recovery would wait for the watchdog")
	}
	if *reloads != 1 {
		t.Errorf("forced reloads = %d, want 1", *reloads)
	}

	p.pfDelayedRecheckMu.Lock()
	timers := append([]*time.Timer(nil), p.pfDelayedRecheckTimers...)
	p.pfDelayedRecheckTimers = nil
	p.pfDelayedRecheckMu.Unlock()
	for _, timer := range timers {
		timer.Stop()
	}
}