cli: add network fallback and explicit-only Internal Domain modes

Internal Domains now support three modes. "os" is unchanged.
"resolvers" becomes explicit resolver with network fallback: the
configured resolvers are tried first, and a timeout, unreachable
resolver, SERVFAIL or NXDOMAIN hands the query to matching VPN DNS
servers, then domain-less VPN DNS servers, then the OS resolver's LAN
nameservers through the new LanOnlyQueryCtx. "resolvers_only" keeps
the previous strict behavior. Down fallback-mode resolvers are skipped
and re-checked in the background at most every 30s. Generated upstreams
are identified by the UpstreamConfig.InternalDomain marker instead of
the internal_ key prefix.

Master adaptations: the fallback state lives on proxyRequest and is
applied in tryUpstreams, shouldContinueWithNextUpstream and
handleAllUpstreamsFailure instead of v1.0's monolithic proxy loop.
Resolver construction uses master's context-aware NewResolver, and the
background re-check marks its context private so transport errors that
name the resolver stay at debug. Tests use master's zap JSON capture
and VPN manager logger.

cmd/cli/cli.go

@@ -1207,6 +1207,7 @@ func logInternalDomainsSummary(logger *ctrld.Logger, summary internalDomainsSumm
 	event.Int("domains", summary.domains).
 		Int("os_resolver", summary.osMode).
 		Int("explicit_resolver", summary.explicit).
+		Int("explicit_resolver_only", summary.only).
 		Int("resolvers", summary.resolvers).
 		Int("skipped", summary.skipped).
 		Int("preempted", summary.preempted).

cmd/cli/dns_proxy.go

@@ -99,6 +99,12 @@ type proxyRequest struct {
 	staleAnswer     *dns.Msg
 	isLanOrPtrQuery bool
 	upstreamConfigs []*ctrld.UpstreamConfig
+	// internalFallback is set when the upstreams are the configured resolvers
+	// of an Internal Domain in explicit-with-network-fallback mode.
+	// internalNegative keeps the best SERVFAIL or NXDOMAIN answer those
+	// resolvers gave, so a name nobody resolves gets a real negative answer.
+	internalFallback bool
+	internalNegative *dns.Msg
 }
 
 // proxyResponse contains data for proxying a DNS response from upstream.
@@ -761,9 +767,16 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse {
 	// DNS intercept recovery bypass: forward all queries to OS/DHCP resolver.
 	// This runs when upstreams are unreachable (e.g., captive portal network)
 	// and allows the network's DNS to handle authentication pages.
-	// Explicit Internal Domain resolvers remain exclusive during recovery:
-	// general DNS failure does not authorize disclosing private names to DHCP DNS.
-	if dnsIntercept && p.recoveryBypass.Load() && !internalDomainExplicitUpstreams(req.ufr.upstreams) {
+	//
+	// An Internal Domain with explicit resolvers does not take part. The bypass
+	// exists because general DNS is broken, which says nothing about the
+	// administrator's own resolvers: they may well be reachable, and they are
+	// to be tried first. Such a query continues to the normal flow, where it
+	// reaches its configured resolvers first. In "explicit resolver only" mode
+	// it fails if none answer; otherwise it falls back to the VPN resolvers and
+	// the network's LAN nameservers, never to the public nameservers the bypass
+	// may use.
+	if dnsIntercept && p.recoveryBypass.Load() && !p.internalDomainExplicitUpstreams(req.ufr.upstreams) {
 		ctrld.Log(ctx, p.Debug(), "Recovery bypass active: forwarding to OS resolver")
 		// The bypass runs before rule handling, so an Active Directory name
 		// needs its LAN mark here too, or it races the domain controller
@@ -796,9 +809,12 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse {
 	}
 
 	// VPN DNS split routing (only in dns-intercept mode). Explicit Internal
-	// Domain resolvers take precedence over auto-detected VPN routes. OS-mode
+	// Domain resolvers take precedence over auto-detected VPN routes: handing
+	// the query to a VPN DNS server first would override an explicit selection
+	// with a discovered one. In fallback mode the VPN DNS servers are asked
+	// later, once the configured resolvers have not resolved the name. OS-mode
 	// Internal Domains still follow the endpoint's VPN resolver.
-	if dnsIntercept && p.vpnDNS != nil && len(req.msg.Question) > 0 && !internalDomainExplicitUpstreams(upstreams) {
+	if dnsIntercept && p.vpnDNS != nil && len(req.msg.Question) > 0 && !p.internalDomainExplicitUpstreams(upstreams) {
 		domain := req.msg.Question[0].Name
 		if vpnServers := p.vpnDNS.UpstreamForDomain(domain); len(vpnServers) > 0 {
 			ctrld.Log(ctx, p.Debug(), "VPN DNS route matched for domain %s, using servers: %v", domain, vpnServers)
@@ -1005,12 +1021,21 @@ func (p *prog) serveStaleResponse(ctx context.Context, staleAnswer *dns.Msg) *pr
 
 // handleAllUpstreamsFailure handles the failure scenario when all upstream resolvers fail to respond or process the request.
 func (p *prog) handleAllUpstreamsFailure(ctx context.Context, req *proxyRequest, upstreams []string) *proxyResponse {
+	if req.internalFallback {
+		if res := p.internalDomainNetworkFallback(ctx, req, upstreams); res != nil {
+			return res
+		}
+	}
 	ctrld.Log(ctx, p.querySampler.event(sampleClassAllEndpointsFailed, ""), "All %v endpoints failed", journalUpstreamNames(upstreams))
 
-	// An unavailable internal resolver must not trigger endpoint-wide recovery
-	// or send the private name through the OS-resolver catch-all.
-	if internalDomainExplicitUpstreams(upstreams) {
-		ctrld.Log(ctx, p.Debug(), "Internal domain resolvers unreachable; not falling back")
+	// An Internal Domain with explicit resolvers never takes the OS-resolver
+	// catch-all below: it can reach public nameservers, ctrld's own fallback
+	// included, which no mode selects for a private name. Fallback mode has
+	// already tried the VPN resolvers and the network's LAN nameservers above.
+	// The recovery flow exists for the loss of general DNS, not for one
+	// unavailable internal server, so it is not started either.
+	if p.internalDomainExplicitUpstreams(upstreams) {
+		ctrld.Log(ctx, p.Debug(), "Internal domain resolvers unreachable; not using the OS resolver catch all")
 	} else if p.leakOnUpstreamFailure() {
 		ctrld.Log(ctx, p.Debug(), "Leak on upstream failure enabled")
 		if p.um.countHealthy(upstreams) == 0 {
@@ -1036,12 +1061,39 @@ func (p *prog) handleAllUpstreamsFailure(ctx context.Context, req *proxyRequest,
 	return &proxyResponse{answer: answer}
 }
 
+// internalDomainNetworkFallback runs the network fallback of an Internal Domain
+// in explicit-with-network-fallback mode, once its configured resolvers have
+// failed, or answered SERVFAIL or NXDOMAIN. It returns nil when nothing on the
+// network resolved the name and no resolver gave a negative answer either.
+func (p *prog) internalDomainNetworkFallback(ctx context.Context, req *proxyRequest, upstreams []string) *proxyResponse {
+	ctrld.Log(ctx, p.Debug(), "Internal domain resolvers did not resolve the query; trying network resolvers")
+	answer, negative := p.resolveInternalDomainOnNetwork(ctx, req.msg)
+	if answer != nil {
+		answer.Compress = true
+		if p.cache != nil && req.msg.Question[0].Qtype != dns.TypePTR {
+			p.updateCache(ctx, req, answer, upstreams[0])
+		}
+		hostname := ""
+		if req.ci != nil {
+			hostname = req.ci.Hostname
+		}
+		ctrld.Log(ctx, p.Info(), "REPLY: internal domain network fallback -> %s (%s): %s", req.ufr.srcAddr, hostname, dns.RcodeToString[answer.Rcode])
+		return &proxyResponse{answer: answer}
+	}
+	if negative = betterInternalDomainNegative(req.internalNegative, negative); negative != nil {
+		ctrld.Log(ctx, p.Debug(), "No resolver resolved the internal domain; returning %s", dns.RcodeToString[negative.Rcode])
+		negative.Compress = true
+		return &proxyResponse{answer: negative}
+	}
+	return nil
+}
+
 // countFailedClientQuery grades one client query that ended without an answer.
 // A query that only Internal Domain resolvers serve stays out of the grade: an
 // endpoint away from the organization network never reaches them, and that is
 // not an outage of the query path.
 func (p *prog) countFailedClientQuery(upstreams []string) {
-	if internalDomainExplicitUpstreams(upstreams) {
+	if p.internalDomainExplicitUpstreams(upstreams) {
 		return
 	}
 	p.health.countFailedQuery()
@@ -1066,6 +1118,16 @@ func (p *prog) shouldContinueWithNextUpstream(ctx context.Context, req *proxyReq
 		return true
 	}
 
+	// An Internal Domain in explicit-with-network-fallback mode moves on from a
+	// configured resolver that answers SERVFAIL or NXDOMAIN, as it does from
+	// one that does not answer.
+	if req.internalFallback && internalDomainFallbackRcode(answer.Rcode) {
+		ctrld.Log(ctx, p.Debug(), "Internal domain resolver %s answered %s, trying next resolver",
+			upstream, dns.RcodeToString[answer.Rcode])
+		req.internalNegative = betterInternalDomainNegative(req.internalNegative, answer)
+		return true
+	}
+
 	ctrld.Log(ctx, p.Debug(), "Not continuing to next upstream")
 	return false
 }
@@ -1133,11 +1195,22 @@ func (p *prog) prepareSuccessResponse(ctx context.Context, req *proxyRequest, an
 func (p *prog) tryUpstreams(ctx context.Context, req *proxyRequest, upstreams []string, upstreamConfigs []*ctrld.UpstreamConfig) *proxyResponse {
 	serveStaleCache := p.cache != nil && p.cfg.Service.CacheServeStale && !req.ufr.zpaDNSEcho
 	req.upstreamConfigs = upstreamConfigs
+	req.internalFallback = p.internalDomainFallbackUpstreams(upstreams)
 
 	ctrld.Log(ctx, p.Debug(), "Trying %d upstreams", len(upstreamConfigs))
 
 	for n, upstreamConfig := range upstreamConfigs {
 		last := n == len(upstreamConfigs)-1
+		// A fallback-mode Internal Domain resolver that the monitor reports as
+		// down is not waited on: off the organization network every query
+		// would otherwise pay its timeout before the network fallback runs.
+		// Nothing else marks a generated resolver up again, so skipping it
+		// starts a background re-check that does.
+		if req.internalFallback && upstreamConfig != nil && p.um.isDown(upstreams[n]) {
+			ctrld.Log(ctx, p.Debug(), "Internal domain resolver %s is down, skipping it", upstreams[n])
+			p.internalDomainProbes.probe(p.um, upstreams[n], upstreamConfig, req.msg)
+			continue
+		}
 		ctrld.Log(ctx, p.Debug(), "Processing upstream %d/%d: %s", n+1, len(upstreamConfigs), upstreams[n])
 
 		if res := p.processUpstream(ctx, req, upstreams[n], upstreamConfig, serveStaleCache, last); res != nil {
@@ -1200,7 +1273,7 @@ func (p *prog) processUpstream(ctx context.Context, req *proxyRequest, upstream
 
 // queryUpstream sends a DNS query to a specified upstream using its configuration and handles errors and retries.
 func (p *prog) queryUpstream(ctx context.Context, req *proxyRequest, upstream string, upstreamConfig *ctrld.UpstreamConfig) *dns.Msg {
-	if isInternalDomainUpstream(upstream) {
+	if isGeneratedInternalDomainUpstream(upstreamConfig) {
 		ctx = ctrld.PrivateResolverCtx(ctx)
 	}
 	if upstreamConfig.UpstreamSendClientInfo() && req.ci != nil {
@@ -1232,7 +1305,7 @@ func (p *prog) queryUpstream(ctx context.Context, req *proxyRequest, upstream st
 	// so the sampler sets the level of these lines.
 	// Transport errors contain the organization's private resolver address.
 	// Keep that detail at debug while retaining a visible failure classification.
-	if isInternalDomainUpstream(upstream) {
+	if isGeneratedInternalDomainUpstream(upstreamConfig) {
 		ctrld.Log(ctx, p.Debug().Err(err), "Failed to resolve query")
 		ctrld.Log(ctx, p.querySampler.event(sampleClassInternalDomain, upstream).Str("failure", internalDomainFailureReason(err)),
 			"Failed to resolve query using an Internal Domain resolver")
@@ -2949,7 +3022,7 @@ func (p *prog) buildRecoveryUpstreams(reason RecoveryReason) map[string]*ctrld.U
 	case RecoveryReasonOSFailure:
 		for k, uc := range p.cfg.Upstream {
 			name := upstreamPrefix + k
-			if uc != nil && uc.Type != ctrld.ResolverTypeOS && !isInternalDomainUpstream(name) {
+			if uc != nil && uc.Type != ctrld.ResolverTypeOS && !isGeneratedInternalDomainUpstream(uc) {
 				upstreams[name] = uc
 			}
 		}

cmd/cli/internal_domains.go

@@ -9,7 +9,11 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
+	"time"
+
+	"github.com/miekg/dns"
 
 	"github.com/Control-D-Inc/ctrld"
 	"github.com/Control-D-Inc/ctrld/internal/controld"
@@ -17,15 +21,25 @@ import (
 )
 
 // internalDomainUpstreamPrefix names the upstreams generated for organization
-// Internal Domains that select explicit resolvers. The prefix is what tells
-// proxy() that a rule is served only by configured resolvers, so it
-// must not collide with the numeric keys used for the Control D upstream.
+// Internal Domains that select explicit resolvers. It must not collide with
+// the numeric key used for the Control D upstream. The key only names the
+// upstream: what identifies it as generated, and in which mode, is
+// UpstreamConfig.InternalDomain, which no configuration file can set.
 const internalDomainUpstreamPrefix = "internal_"
 
+// Values of UpstreamConfig.InternalDomain on a generated upstream.
+const (
+	// internalDomainUpstreamFallback serves an Internal Domain in explicit
+	// resolver with network fallback mode.
+	internalDomainUpstreamFallback = "fallback"
+	// internalDomainUpstreamOnly serves an Internal Domain in explicit
+	// resolver only mode: its answer or failure is final.
+	internalDomainUpstreamOnly = "only"
+)
+
 // internalDomainUpstreamName is the Name given to every generated Internal
-// Domain upstream. Together with the key prefix and the legacy type it is what
-// distinguishes an upstream ctrld generated from one an endpoint custom
-// configuration happens to name the same way.
+// Domain upstream. It is only a display label: what identifies an upstream as
+// generated is UpstreamConfig.InternalDomain, which no configuration can set.
 const internalDomainUpstreamName = "Internal Domain resolver"
 
 // internalDomainsSummary counts the outcome of applying Internal Domains. It
@@ -35,6 +49,7 @@ type internalDomainsSummary struct {
 	domains   int // domains that produced rules
 	osMode    int // domains routed to the OS/default resolver
 	explicit  int // domains routed to configured resolvers
+	only      int // explicit domains with no network fallback
 	resolvers int // explicit resolver upstreams generated
 	skipped   int // entries dropped: unusable domain, no usable resolver, or duplicate
 	preempted int // domains already covered by a higher precedence rule
@@ -127,43 +142,83 @@ func internalDomainEndpoint(resolver string) (string, bool) {
 	return net.JoinHostPort(addr.String(), port), true
 }
 
-// internalDomainMode reports whether entry selects explicit resolvers, and
-// whether ctrld understands its mode at all.
+// internalDomainResolution is how an Internal Domain is resolved.
+type internalDomainResolution int
+
+const (
+	// internalDomainViaOS is the "network default" mode: the endpoint's
+	// OS/default resolver.
+	internalDomainViaOS internalDomainResolution = iota
+	// internalDomainViaResolvers is the configured resolvers first, then the
+	// active network and VPN resolvers. The default explicit selection.
+	internalDomainViaResolvers
+	// internalDomainViaResolversOnly is the configured resolvers and nothing
+	// else.
+	internalDomainViaResolversOnly
+)
+
+// internalDomainMode reports how entry is resolved, and whether ctrld
+// understands its mode at all.
 //
 // Mode is authoritative when the API sends it. "os" ignores any resolver
 // addresses the entry still carries, so switching a domain back to the OS
 // resolver cannot be undone by addresses a previous selection left behind;
-// "resolvers" stays explicit even when the list it points at turns out to be
-// unusable, rather than silently becoming OS resolution. An empty mode is a
-// deployment that predates the field, where the list is the only signal.
+// the explicit modes stay explicit even when the list they point at turns out
+// to be unusable, rather than silently becoming OS resolution. An empty mode
+// is a deployment that predates the field, where the list is the only signal,
+// and an explicit list gets the default explicit mode.
 //
 // An unrecognized mode is not guessed at: routing a private domain by guess is
-// the disclosure this feature exists to prevent.
-func internalDomainMode(entry controld.SplitDNS) (explicit, known bool) {
+// the disclosure this feature exists to prevent. It fails closed instead. With
+// resolvers it is resolved as explicit resolver only, through the resolvers the
+// administrator chose and nothing else, so a wire value this build does not
+// know, such as a renamed strict mode, cannot send the domain to the Control D
+// upstream or the network. Without resolvers there is nothing to fail closed
+// to, so the entry is reported as unknown and dropped.
+func internalDomainMode(entry controld.SplitDNS) (resolution internalDomainResolution, known bool) {
 	switch strings.ToLower(strings.TrimSpace(entry.Mode)) {
 	case controld.SplitDNSModeOS:
-		return false, true
+		return internalDomainViaOS, true
 	case controld.SplitDNSModeResolvers:
-		return true, true
+		return internalDomainViaResolvers, true
+	case controld.SplitDNSModeResolversOnly:
+		return internalDomainViaResolversOnly, true
 	case "":
-		return len(entry.Resolvers) > 0, true
+		if len(entry.Resolvers) > 0 {
+			return internalDomainViaResolvers, true
+		}
+		return internalDomainViaOS, true
 	default:
-		return false, false
+		if len(entry.Resolvers) > 0 {
+			return internalDomainViaResolversOnly, true
+		}
+		return internalDomainViaOS, false
 	}
 }
 
-// internalDomainRouting returns the endpoints an entry routes to, or a reason
-// it cannot be used. No endpoints and no reason means OS resolution.
+// internalDomainModeRecognized reports whether mode is one this build knows,
+// as opposed to one internalDomainMode resolves by failing closed.
+func internalDomainModeRecognized(mode string) bool {
+	switch strings.ToLower(strings.TrimSpace(mode)) {
+	case "", controld.SplitDNSModeOS, controld.SplitDNSModeResolvers, controld.SplitDNSModeResolversOnly:
+		return true
+	}
+	return false
+}
+
+// internalDomainRouting returns how an entry is resolved and the endpoints it
+// routes to, or a reason it cannot be used. No endpoints and no reason means
+// OS resolution.
 //
 // Generation and refresh comparison both go through here, so the two can never
 // disagree about what an entry means.
-func internalDomainRouting(entry controld.SplitDNS) (endpoints []string, reason string) {
-	explicit, known := internalDomainMode(entry)
+func internalDomainRouting(entry controld.SplitDNS) (resolution internalDomainResolution, endpoints []string, reason string) {
+	resolution, known := internalDomainMode(entry)
 	if !known {
-		return nil, "unknown_mode"
+		return resolution, nil, "unknown_mode"
 	}
-	if !explicit {
-		return nil, ""
+	if resolution == internalDomainViaOS {
+		return resolution, nil, ""
 	}
 	endpoints = make([]string, 0, len(entry.Resolvers))
 	for _, resolver := range entry.Resolvers {
@@ -176,9 +231,9 @@ func internalDomainRouting(entry controld.SplitDNS) (endpoints []string, reason
 	// different one, so the entry is dropped and the domain keeps its normal
 	// path instead.
 	if len(endpoints) == 0 {
-		return nil, "no_usable_resolver"
+		return resolution, nil, "no_usable_resolver"
 	}
-	return endpoints, ""
+	return resolution, endpoints, ""
 }
 
 // internalDomainRuleExists reports whether the listener policy already routes
@@ -239,7 +294,8 @@ func orderInternalDomainsBySpecificity(entries []controld.SplitDNS) []controld.S
 // resolvers routes to the OS/default resolver by way of an empty upstream list,
 // the same representation Magic Folder excludes use. An entry with resolvers
 // gets one generated upstream per address, listed in configured order so the
-// later ones act as failover for the first.
+// later ones act as failover for the first. Those upstreams carry the mode in
+// UpstreamConfig.InternalDomain.
 //
 // cfg is rebuilt from the API response on every fetch, so this function is the
 // only writer of internal_* upstreams: a removed domain leaves nothing behind.
@@ -269,12 +325,15 @@ func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) intern
 		// written: an entry that cannot be used, or a domain already covered by
 		// a higher precedence rule, must not leave a generated upstream behind
 		// with no rule referring to it.
-		endpoints, reason := internalDomainRouting(entry)
+		resolution, endpoints, reason := internalDomainRouting(entry)
 		if reason != "" {
 			mainLog.Load().Debug().Msgf("internal domains: skipping %q: %s", domain, reason)
 			summary.skipped++
 			continue
 		}
+		if !internalDomainModeRecognized(entry.Mode) {
+			mainLog.Load().Debug().Msgf("internal domains: %q has unrecognized mode %q; using its resolvers only", domain, entry.Mode)
+		}
 		seen[domain] = struct{}{}
 
 		sources := []string{domain, "*." + domain}
@@ -295,14 +354,19 @@ func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) intern
 			continue
 		}
 
+		marker := internalDomainUpstreamFallback
+		if resolution == internalDomainViaResolversOnly {
+			marker = internalDomainUpstreamOnly
+		}
 		targets := make([]string, 0, len(endpoints))
 		for _, endpoint := range endpoints {
 			key := internalDomainUpstreamPrefix + strconv.Itoa(summary.resolvers)
 			cfg.Upstream[key] = &ctrld.UpstreamConfig{
-				Name:     internalDomainUpstreamName,
-				Type:     ctrld.ResolverTypeLegacy,
-				Endpoint: endpoint,
-				Timeout:  5000,
+				Name:           internalDomainUpstreamName,
+				Type:           ctrld.ResolverTypeLegacy,
+				Endpoint:       endpoint,
+				Timeout:        5000,
+				InternalDomain: marker,
 			}
 			targets = append(targets, upstreamPrefix+key)
 			summary.resolvers++
@@ -321,63 +385,118 @@ func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) intern
 			mainLog.Load().Debug().Msgf("internal domains: %q routed to the OS resolver", domain)
 		} else {
 			summary.explicit++
+			if resolution == internalDomainViaResolversOnly {
+				summary.only++
+			}
 			mainLog.Load().Debug().Msgf("internal domains: %q routed to %v", domain, targets)
 		}
 	}
 	return summary
 }
 
-// isGeneratedInternalDomainUpstream reports whether a cfg.Upstream entry is one
+// isGeneratedInternalDomainUpstream reports whether uc is an upstream
 // applyInternalDomains created.
 //
 // Generated Internal Domain resolvers are part of the managed Control D
 // configuration, not upstreams the endpoint operator chose, so eligibility
 // checks that ask "is this a plain single-upstream Control D install" must not
-// count them. The key prefix alone is not proof of that, because an endpoint
-// custom configuration replaces the generated config outright and could name an
-// upstream anything; the shape this function checks is only produced here.
-func isGeneratedInternalDomainUpstream(name string, uc *ctrld.UpstreamConfig) bool {
-	return uc != nil &&
-		strings.HasPrefix(name, internalDomainUpstreamPrefix) &&
-		uc.Type == ctrld.ResolverTypeLegacy &&
-		uc.Name == internalDomainUpstreamName
+// count them. Neither the key nor any other configurable field is proof of
+// that: a local config or an endpoint custom configuration can name an upstream
+// anything and give it any type and name. Only the InternalDomain marker is,
+// because ctrld sets it and no configuration file can.
+func isGeneratedInternalDomainUpstream(uc *ctrld.UpstreamConfig) bool {
+	return uc != nil && uc.InternalDomain != ""
 }
 
-// isInternalDomainUpstream reports whether an upstream reference was generated
-// for an Internal Domain with explicit resolvers.
-func isInternalDomainUpstream(upstream string) bool {
-	return strings.HasPrefix(upstream, upstreamPrefix+internalDomainUpstreamPrefix)
+// internalDomainUpstream returns the generated Internal Domain upstream an
+// "upstream.<key>" reference names, or nil if it names any other upstream.
+func (p *prog) internalDomainUpstream(upstream string) *ctrld.UpstreamConfig {
+	if p.cfg == nil {
+		return nil
+	}
+	uc := p.cfg.Upstream[strings.TrimPrefix(upstream, upstreamPrefix)]
+	if !isGeneratedInternalDomainUpstream(uc) {
+		return nil
+	}
+	return uc
+}
+
+// isInternalDomainUpstream reports whether an upstream reference names a
+// generated Internal Domain resolver, in either explicit mode.
+func (p *prog) isInternalDomainUpstream(upstream string) bool {
+	return p.internalDomainUpstream(upstream) != nil
 }
 
 // internalDomainExplicitUpstreams reports whether every upstream serving a
-// query is an configured Internal Domain resolver.
+// query is a configured Internal Domain resolver, in either explicit mode.
 //
-// proxy() uses this to keep such a query off the OS-resolver catch-all when the
-// configured resolvers are unreachable: the administrator selected those
-// resolvers and no other, so a private name must fail rather than be sent
-// somewhere it was not meant to go. It also keeps an unreachable internal
-// resolver from triggering the endpoint-wide recovery flow, which exists for
-// the loss of general DNS, not for one unavailable internal server.
-func internalDomainExplicitUpstreams(upstreams []string) bool {
+// proxy() uses this to try the configured resolvers before anything else, and
+// to keep such a query off the OS-resolver catch-all when they fail: that
+// catch-all may reach public nameservers, and a private name must not be sent
+// somewhere it was not meant to go. In fallback mode proxy() tries the
+// network resolvers instead (see internalDomainFallbackUpstreams). It also
+// keeps an unreachable internal resolver from triggering the endpoint-wide
+// recovery flow, which exists for the loss of general DNS, not for one
+// unavailable internal server.
+func (p *prog) internalDomainExplicitUpstreams(upstreams []string) bool {
 	if len(upstreams) == 0 {
 		return false
 	}
 	for _, upstream := range upstreams {
-		if !isInternalDomainUpstream(upstream) {
+		if !p.isInternalDomainUpstream(upstream) {
 			return false
 		}
 	}
 	return true
 }
 
+// internalDomainFallbackUpstreams reports whether upstreams are the configured
+// resolvers of an Internal Domain in explicit-with-network-fallback mode, whose
+// failure, SERVFAIL or NXDOMAIN hands the query to the active network and VPN
+// resolvers.
+func (p *prog) internalDomainFallbackUpstreams(upstreams []string) bool {
+	if len(upstreams) == 0 {
+		return false
+	}
+	for _, upstream := range upstreams {
+		uc := p.internalDomainUpstream(upstream)
+		if uc == nil || uc.InternalDomain != internalDomainUpstreamFallback {
+			return false
+		}
+	}
+	return true
+}
+
+// internalDomainFallbackRcode reports whether a configured resolver's answer
+// sends a fallback-mode query on to the next resolver. SERVFAIL means the
+// resolver could not answer, and NXDOMAIN may only mean it does not know a
+// name the local network's resolver does. REFUSED and NOTIMP mean it will not
+// serve this endpoint, as an organization resolver with an access list answers
+// an endpoint off the organization network, which is closer to unreachable
+// than to an answer. Any other answer is final, including an empty NOERROR:
+// the name exists, it just has no record of that type.
+func internalDomainFallbackRcode(rcode int) bool {
+	switch rcode {
+	case dns.RcodeServerFailure, dns.RcodeNameError, dns.RcodeRefused, dns.RcodeNotImplemented:
+		return true
+	}
+	return false
+}
+
 // internalDomainsEqual reports whether two Internal Domains lists would produce
 // the same routing. Domains are compared in canonical form and independently of
 // the order the API returned them, so an unchanged list does not trigger a
 // reload on every refresh. Resolver order is significant: it is the order the
-// administrator chose, and it decides which resolver is tried first.
+// administrator chose, and it decides which resolver is tried first. The mode
+// is significant too, including a switch between the two explicit modes.
 func internalDomainsEqual(a, b []controld.SplitDNS) bool {
-	canonical := func(entries []controld.SplitDNS) []controld.SplitDNS {
-		out := make([]controld.SplitDNS, 0, len(entries))
+	type route struct {
+		domain     string
+		resolution internalDomainResolution
+		endpoints  []string
+	}
+	canonical := func(entries []controld.SplitDNS) []route {
+		out := make([]route, 0, len(entries))
 		for _, entry := range entries {
 			domain := normalizeInternalDomain(entry.Domain)
 			if domain == "" {
@@ -385,13 +504,13 @@ func internalDomainsEqual(a, b []controld.SplitDNS) bool {
 			}
 			// Entries that cannot be used route nothing, so they cannot make
 			// an otherwise unchanged list look changed.
-			endpoints, reason := internalDomainRouting(entry)
+			resolution, endpoints, reason := internalDomainRouting(entry)
 			if reason != "" {
 				continue
 			}
-			out = append(out, controld.SplitDNS{Domain: domain, Resolvers: endpoints})
+			out = append(out, route{domain: domain, resolution: resolution, endpoints: endpoints})
 		}
-		sort.Slice(out, func(i, j int) bool { return out[i].Domain < out[j].Domain })
+		sort.Slice(out, func(i, j int) bool { return out[i].domain < out[j].domain })
 		return out
 	}
 	x, y := canonical(a), canonical(b)
@@ -399,7 +518,7 @@ func internalDomainsEqual(a, b []controld.SplitDNS) bool {
 		return false
 	}
 	for i := range x {
-		if x[i].Domain != y[i].Domain || !slices.Equal(x[i].Resolvers, y[i].Resolvers) {
+		if x[i].domain != y[i].domain || x[i].resolution != y[i].resolution || !slices.Equal(x[i].endpoints, y[i].endpoints) {
 			return false
 		}
 	}
@@ -427,12 +546,6 @@ func internalDomainFailureReason(err error) string {
 	return "error"
 }
 
-// isGeneratedInternalDomainUpstreamRef is isGeneratedInternalDomainUpstream for
-// call sites that hold an "upstream.<key>" reference rather than the bare key.
-func isGeneratedInternalDomainUpstreamRef(upstream string, uc *ctrld.UpstreamConfig) bool {
-	return isGeneratedInternalDomainUpstream(strings.TrimPrefix(upstream, upstreamPrefix), uc)
-}
-
 // logUpstreamProbeFailure reports a background probe failure against the
 // upstream that the reference upstream names.
 //
@@ -448,10 +561,202 @@ func (p *prog) logUpstreamProbeFailure(upstream string, uc *ctrld.UpstreamConfig
 	detail := append([]any{}, args...)
 	p.Debug().Err(err).Msgf(format+" for upstream: %q, endpoint: %q", append(detail, uc.Name, uc.Endpoint)...)
 	event := level().Str("upstream", journalUpstreamName(upstream))
-	if !isGeneratedInternalDomainUpstreamRef(upstream, uc) {
+	if !isGeneratedInternalDomainUpstream(uc) {
 		event.Err(err).Msgf(format, detail...)
 		return
 	}
 	event.Str("failure", internalDomainFailureReason(err)).
 		Msgf(format+" for an Internal Domain resolver", detail...)
 }
+
+// betterInternalDomainNegative returns whichever of two negative answers a
+// fallback-mode query should end with: NXDOMAIN over SERVFAIL, since a
+// resolver that says the name does not exist has answered, and otherwise the
+// first one seen.
+func betterInternalDomainNegative(current, candidate *dns.Msg) *dns.Msg {
+	switch {
+	case candidate == nil:
+		return current
+	case current == nil:
+		return candidate
+	case candidate.Rcode == dns.RcodeNameError && current.Rcode != dns.RcodeNameError:
+		return candidate
+	}
+	return current
+}
+
+// internalDomainVPNUpstream returns the upstream the network fallback uses for
+// a VPN DNS server. A seam: VPN DNS servers are dialed on port 53, which a test
+// fixture cannot listen on.
+var internalDomainVPNUpstream = func(m *vpnDNSManager, server string) *ctrld.UpstreamConfig {
+	return m.upstreamConfigFor(server)
+}
+
+// internalDomainOSResolve resolves msg through the OS resolver for the network
+// fallback. A seam, so a test sees the context the fallback passes: that
+// context is what keeps the private name off public nameservers.
+var internalDomainOSResolve = func(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) {
+	resolver, err := ctrld.NewResolver(ctx, osUpstreamConfig)
+	if err != nil {
+		return nil, err
+	}
+	resolveCtx, cancel := osUpstreamConfig.Context(ctx)
+	defer cancel()
+	return resolver.Resolve(resolveCtx, msg)
+}
+
+// resolveInternalDomainOnNetwork is the network fallback of an Internal Domain
+// in explicit-with-network-fallback mode. proxy() runs it once the configured
+// resolvers have failed, or answered SERVFAIL or NXDOMAIN.
+//
+// It asks only resolvers that the endpoint's active network or VPN provides,
+// in this order: VPN DNS servers whose domains match the query, then VPN DNS
+// servers with no domains (both only in dns-intercept mode, where ctrld tracks
+// them), then the OS resolver's LAN nameservers. The OS resolver is asked as a
+// LAN-only query, so the private name never reaches a public nameserver, not
+// even one DHCP supplied, nor ctrld's own public fallback. With no LAN
+// nameserver the OS step sends nothing.
+//
+// The first NOERROR answer is returned as answer, including an empty one,
+// which is final. Otherwise negative is the best SERVFAIL or NXDOMAIN answer
+// seen. While Windows is serving retained VPN DNS state, a VPN transport
+// failure stops the fallback before the OS resolver, as it does for VPN split
+// routing.
+func (p *prog) resolveInternalDomainOnNetwork(ctx context.Context, msg *dns.Msg) (answer, negative *dns.Msg) {
+	domain := msg.Question[0].Name
+	resolve := func(ctx context.Context, uc *ctrld.UpstreamConfig) (*dns.Msg, error) {
+		resolver, err := ctrld.NewResolver(ctx, uc)
+		if err != nil {
+			return nil, err
+		}
+		resolveCtx, cancel := uc.Context(ctx)
+		defer cancel()
+		return resolver.Resolve(resolveCtx, msg)
+	}
+	// accept reports whether a network answer is final, and keeps a negative
+	// one for the end.
+	accept := func(source string, m *dns.Msg) bool {
+		if !sameQuestion(msg, m) {
+			ctrld.Log(ctx, p.Debug(), "internal domain fallback: discarding answer from %s: question mismatch", source)
+			return false
+		}
+		if m.Rcode == dns.RcodeSuccess {
+			ctrld.Log(ctx, p.Debug(), "internal domain fallback: %s answered %s", source, domain)
+			return true
+		}
+		ctrld.Log(ctx, p.Debug(), "internal domain fallback: %s answered %s for %s",
+			source, dns.RcodeToString[m.Rcode], domain)
+		if internalDomainFallbackRcode(m.Rcode) {
+			negative = betterInternalDomainNegative(negative, m)
+		}
+		return false
+	}
+
+	if dnsIntercept && p.vpnDNS != nil {
+		var servers []string
+		for _, server := range slices.Concat(p.vpnDNS.UpstreamForDomain(domain), p.vpnDNS.DomainlessServers()) {
+			if !slices.Contains(servers, server) {
+				servers = append(servers, server)
+			}
+		}
+		var gotAnswer, gotTransportFailure bool
+		for _, server := range servers {
+			m, err := resolve(ctx, internalDomainVPNUpstream(p.vpnDNS, server))
+			if m == nil {
+				gotTransportFailure = true
+				ctrld.Log(ctx, p.Debug().Err(err), "internal domain fallback: VPN DNS server %s failed", server)
+				continue
+			}
+			gotAnswer = true
+			p.vpnDNS.VPNDNSReachable()
+			if accept("VPN DNS server "+server, m) {
+				return m, nil
+			}
+		}
+		if !gotAnswer && gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, servers) {
+			return nil, negative
+		}
+	}
+
+	m, err := internalDomainOSResolve(ctrld.LanOnlyQueryCtx(ctx), msg)
+	if m == nil {
+		ctrld.Log(ctx, p.Debug().Err(err), "internal domain fallback: OS resolver LAN nameservers failed")
+		return nil, negative
+	}
+	if accept("OS resolver", m) {
+		return m, nil
+	}
+	return nil, negative
+}
+
+// internalDomainProbeInterval is how often, at most, a down fallback-mode
+// Internal Domain resolver is re-checked in the background.
+const internalDomainProbeInterval = 30 * time.Second
+
+// internalDomainProber re-checks down fallback-mode Internal Domain resolvers
+// in the background, and marks one up again when it answers.
+//
+// proxy() skips such a resolver while it is down, so that an endpoint away
+// from the organization network goes to the network fallback without waiting
+// on it. A generated resolver comes back up only by answering a query: it is
+// excluded from recovery, and the loop checker leaves down upstreams alone. So
+// without this re-check, a resolver that went down would stay skipped until
+// the next reload, even once the endpoint is back on the organization network.
+//
+// The zero value is ready to use.
+type internalDomainProber struct {
+	mu   sync.Mutex
+	last map[string]time.Time
+
+	// now and start are seams, so a test controls the clock and runs the
+	// re-check itself. nil means time.Now and a new goroutine.
+	now   func() time.Time
+	start func(func())
+}
+
+// probe re-checks upstream in the background, at most once per
+// internalDomainProbeInterval, by sending it a copy of msg. The query goes to
+// the organization's own resolver, which it was meant for anyway. Any DNS
+// answer proves the resolver reachable, the same as on the query path, and
+// marks it up in um; a failure leaves it down until the next re-check.
+func (pr *internalDomainProber) probe(um *upstreamMonitor, upstream string, uc *ctrld.UpstreamConfig, msg *dns.Msg) {
+	now := time.Now
+	if pr.now != nil {
+		now = pr.now
+	}
+	pr.mu.Lock()
+	if last, ok := pr.last[upstream]; ok && now().Sub(last) < internalDomainProbeInterval {
+		pr.mu.Unlock()
+		return
+	}
+	if pr.last == nil {
+		pr.last = make(map[string]time.Time)
+	}
+	pr.last[upstream] = now()
+	pr.mu.Unlock()
+
+	query := msg.Copy()
+	run := func() {
+		var answer *dns.Msg
+		// The resolver address is private: keep transport errors that name it
+		// at debug, as the query path does.
+		ctx := ctrld.PrivateResolverCtx(ctrld.LoggerCtx(context.Background(), mainLog.Load()))
+		resolver, err := ctrld.NewResolver(ctx, uc)
+		if err == nil {
+			resolveCtx, cancel := uc.Context(ctx)
+			answer, err = resolver.Resolve(resolveCtx, query)
+			cancel()
+		}
+		if answer == nil {
+			mainLog.Load().Debug().Err(err).Msgf("internal domains: %s is still down", upstream)
+			return
+		}
+		mainLog.Load().Debug().Msgf("internal domains: %s answered the re-check", upstream)
+		um.noteSuccess(upstream)
+	}
+	if pr.start != nil {
+		pr.start(run)
+		return
+	}
+	go run()
+}

cmd/cli/internal_domains_fallback_test.go

@@ -0,0 +1,587 @@
+package cli
+
+import (
+	"context"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/miekg/dns"
+
+	"github.com/Control-D-Inc/ctrld"
+	"github.com/Control-D-Inc/ctrld/internal/controld"
+)
+
+// startRcodeDNSFixture is startDNSFixture answering every question with rcode.
+// A NOERROR answer carries an A record unless empty is set, which gives the
+// name-exists-but-no-record-of-that-type answer.
+func startRcodeDNSFixture(t *testing.T, rcode int, empty bool) *dnsFixture {
+	t.Helper()
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	f := &dnsFixture{addr: pc.LocalAddr().String()}
+	started := make(chan struct{})
+	server := &dns.Server{
+		PacketConn:        pc,
+		NotifyStartedFunc: func() { close(started) },
+		Handler: dns.HandlerFunc(func(w dns.ResponseWriter, msg *dns.Msg) {
+			f.mu.Lock()
+			if len(msg.Question) > 0 {
+				f.qs = append(f.qs, msg.Question[0].Name)
+			}
+			f.mu.Unlock()
+			reply := new(dns.Msg)
+			reply.SetRcode(msg, rcode)
+			if rcode == dns.RcodeSuccess && !empty && len(msg.Question) > 0 {
+				rr, _ := dns.NewRR(msg.Question[0].Name + " 60 IN A 10.9.9.9")
+				reply.Answer = append(reply.Answer, rr)
+			}
+			_ = w.WriteMsg(reply)
+		}),
+	}
+	go func() { _ = server.ActivateAndServe() }()
+	select {
+	case <-started:
+	case <-time.After(5 * time.Second):
+		t.Fatal("DNS fixture did not start")
+	}
+	t.Cleanup(func() { _ = server.Shutdown() })
+	return f
+}
+
+func deadLoopbackAddr(t *testing.T) string {
+	t.Helper()
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	addr := pc.LocalAddr().String()
+	_ = pc.Close()
+	return addr
+}
+
+// Explicit resolver with network fallback is the default explicit mode: the
+// configured resolver is asked first, and a timeout, an unreachable resolver,
+// SERVFAIL or NXDOMAIN hands the query to the network's resolver. A valid
+// empty answer is final, and "explicit resolver only" never falls back.
+func TestInternalDomainsNetworkFallback(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		mode string
+		// internal is the configured resolver's rcode; -1 is unreachable.
+		internal      int
+		internalEmpty bool
+		network       int
+		wantNetwork   bool
+		wantRcode     int
+		wantAnswer    bool
+	}{
+		{name: "nxdomain falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "servfail falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeServerFailure, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "unreachable falls back", mode: controld.SplitDNSModeResolvers, internal: -1, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "absent mode falls back", mode: "", internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "answer is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeSuccess, network: dns.RcodeSuccess, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "empty answer is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeSuccess, internalEmpty: true, network: dns.RcodeSuccess, wantRcode: dns.RcodeSuccess},
+		{name: "refused falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeRefused, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "notimp falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNotImplemented, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "refused and network nxdomain", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeRefused, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError},
+		{name: "formerr is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeFormatError, network: dns.RcodeSuccess, wantRcode: dns.RcodeFormatError},
+		{name: "only mode keeps refused", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeRefused, network: dns.RcodeSuccess, wantRcode: dns.RcodeRefused},
+		{name: "unknown mode fails closed", mode: "explicit_only", internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantRcode: dns.RcodeNameError},
+		{name: "nobody knows the name", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError},
+		{name: "configured nxdomain beats network servfail", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeServerFailure, wantNetwork: true, wantRcode: dns.RcodeNameError},
+		{name: "unreachable and network nxdomain", mode: controld.SplitDNSModeResolvers, internal: -1, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError},
+		{name: "only mode keeps nxdomain", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantRcode: dns.RcodeNameError},
+		{name: "only mode keeps servfail", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeServerFailure, network: dns.RcodeSuccess, wantRcode: dns.RcodeServerFailure},
+		{name: "only mode fails when unreachable", mode: controld.SplitDNSModeResolversOnly, internal: -1, network: dns.RcodeSuccess, wantRcode: dns.RcodeServerFailure},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			network := startRcodeDNSFixture(t, tc.network, false)
+			useOSResolverFixture(t, network)
+
+			var internal *dnsFixture
+			resolver := deadLoopbackAddr(t)
+			if tc.internal >= 0 {
+				internal = startRcodeDNSFixture(t, tc.internal, tc.internalEmpty)
+				resolver = internal.addr
+			}
+
+			cfg := internalDomainsTestConfig()
+			cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true)
+			applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: tc.mode, Resolvers: []string{resolver}}})
+			p := newInternalDomainsProg(t, cfg)
+
+			addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+			if err != nil {
+				t.Fatal(err)
+			}
+			ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+			ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+			res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+
+			if internal != nil {
+				if got := internal.questions(); len(got) != 1 {
+					t.Errorf("configured resolver questions = %v, want it asked first", got)
+				}
+			}
+			if got := network.questions(); (len(got) > 0) != tc.wantNetwork {
+				t.Errorf("network resolver questions = %v, want asked = %v", got, tc.wantNetwork)
+			}
+			if res == nil || res.answer == nil {
+				t.Fatalf("no answer")
+			}
+			if res.answer.Rcode != tc.wantRcode {
+				t.Errorf("rcode = %s, want %s", dns.RcodeToString[res.answer.Rcode], dns.RcodeToString[tc.wantRcode])
+			}
+			if got := len(res.answer.Answer) > 0; got != tc.wantAnswer {
+				t.Errorf("answer records = %v, want records = %v", res.answer.Answer, tc.wantAnswer)
+			}
+		})
+	}
+}
+
+// The configured resolvers are all tried, in order, before the network: an
+// NXDOMAIN from the first does not skip the second.
+func TestInternalDomainsNetworkFallbackTriesEveryConfiguredResolverFirst(t *testing.T) {
+	network := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+	useOSResolverFixture(t, network)
+	first := startRcodeDNSFixture(t, dns.RcodeNameError, false)
+	second := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+
+	cfg := internalDomainsTestConfig()
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{first.addr, second.addr}}})
+	p := newInternalDomainsProg(t, cfg)
+
+	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+	ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+	res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+
+	if len(first.questions()) != 1 || len(second.questions()) != 1 {
+		t.Errorf("configured resolvers asked %v, %v; want both once", first.questions(), second.questions())
+	}
+	if got := network.questions(); len(got) != 0 {
+		t.Errorf("network resolver asked %v although a configured resolver answered", got)
+	}
+	if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+		t.Errorf("answer = %v, want the second resolver's NOERROR", res)
+	}
+}
+
+// The fallback is no reason to log the private resolver address above debug,
+// or the domain: the summary-only REPLY line carries neither.
+func TestInternalDomainsNetworkFallbackLogsNoPrivateValues(t *testing.T) {
+	// The shared log output is console-encoded on master; capture JSON so the
+	// level of each line can be checked.
+	buf := captureInternalDomainsLogs(t)
+	network := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+	useOSResolverFixture(t, network)
+	internal := startRcodeDNSFixture(t, dns.RcodeNameError, false)
+
+	cfg := internalDomainsTestConfig()
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}})
+	p := newInternalDomainsProg(t, cfg)
+
+	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.WithValue(ctrld.LoggerCtx(context.Background(), p.logger.Load()), ctrld.ReqIdCtxKey{}, requestID())
+	ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+	before := len(buf.String())
+	p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+	out := buf.String()[before:]
+
+	if !strings.Contains(out, "internal domain network fallback") {
+		t.Fatalf("expected the fallback REPLY line, got: %s", out)
+	}
+	for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
+		if line == "" || strings.Contains(line, `"level":"debug"`) {
+			continue
+		}
+		if strings.Contains(line, internal.addr) || strings.Contains(line, "corp.example") {
+			t.Errorf("private value disclosed above debug level: %s", line)
+		}
+	}
+}
+
+// internalDomainUpstreamsProg returns a prog whose config has one legacy
+// upstream per key, each carrying the given InternalDomain marker.
+func internalDomainUpstreamsProg(markers map[string]string) *prog {
+	cfg := &ctrld.Config{Upstream: make(map[string]*ctrld.UpstreamConfig, len(markers))}
+	for key, marker := range markers {
+		cfg.Upstream[key] = &ctrld.UpstreamConfig{
+			Name:           internalDomainUpstreamName,
+			Type:           ctrld.ResolverTypeLegacy,
+			Endpoint:       "10.0.0.53:53",
+			InternalDomain: marker,
+		}
+	}
+	return &prog{cfg: cfg}
+}
+
+func TestInternalDomainFallbackUpstreams(t *testing.T) {
+	p := internalDomainUpstreamsProg(map[string]string{
+		"internal_0": internalDomainUpstreamFallback,
+		"internal_1": internalDomainUpstreamFallback,
+		"internal_2": internalDomainUpstreamOnly,
+		"0":          "",
+	})
+	for _, tc := range []struct {
+		in   []string
+		want bool
+	}{
+		{in: nil, want: false},
+		{in: []string{"upstream.internal_0", "upstream.internal_1"}, want: true},
+		{in: []string{"upstream.internal_2"}, want: false},
+		{in: []string{"upstream.internal_0", "upstream.internal_2"}, want: false},
+		{in: []string{"upstream.internal_0", "upstream.0"}, want: false},
+		{in: []string{upstreamOS}, want: false},
+	} {
+		if got := p.internalDomainFallbackUpstreams(tc.in); got != tc.want {
+			t.Errorf("internalDomainFallbackUpstreams(%v) = %v, want %v", tc.in, got, tc.want)
+		}
+	}
+	if !p.internalDomainExplicitUpstreams([]string{"upstream.internal_2"}) {
+		t.Error("an explicit-only upstream must still count as an Internal Domain resolver")
+	}
+}
+
+// Both explicit modes generate internal_<n> upstreams; the mode is carried by
+// the InternalDomain marker. Switching between the modes is a routing change
+// the next refresh must apply.
+func TestInternalDomainsExplicitModes(t *testing.T) {
+	cfg := internalDomainsTestConfig()
+	summary := applyInternalDomains(cfg, []controld.SplitDNS{
+		{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}},
+		{Domain: "b.example.com", Mode: " Resolvers_Only ", Resolvers: []string{"10.0.0.54"}},
+	})
+	if summary.explicit != 2 || summary.only != 1 || summary.resolvers != 2 {
+		t.Fatalf("summary = %+v", summary)
+	}
+	for _, tc := range []struct{ domain, target, marker string }{
+		{"a.example.com", "upstream.internal_0", internalDomainUpstreamFallback},
+		{"b.example.com", "upstream.internal_1", internalDomainUpstreamOnly},
+	} {
+		if targets, _ := ruleTargets(t, cfg, tc.domain); strings.Join(targets, ",") != tc.target {
+			t.Errorf("%s targets = %v, want %s", tc.domain, targets, tc.target)
+		}
+		uc := cfg.Upstream[strings.TrimPrefix(tc.target, upstreamPrefix)]
+		if uc == nil || uc.InternalDomain != tc.marker {
+			t.Errorf("%s upstream = %+v, want marker %q", tc.domain, uc, tc.marker)
+		}
+	}
+
+	fallback := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}}
+	only := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{"10.0.0.53"}}}
+	legacy := []controld.SplitDNS{{Domain: "a.example.com", Resolvers: []string{"10.0.0.53"}}}
+	if internalDomainsEqual(fallback, only) {
+		t.Error("switching to explicit resolver only must be detected as a change")
+	}
+	if !internalDomainsEqual(fallback, legacy) {
+		t.Error("an absent mode with resolvers is the default explicit mode")
+	}
+}
+
+// An upstream a configuration defines is an ordinary upstream, whatever it is
+// named and however closely it copies a generated one: a local ctrld.toml that
+// names an upstream internal_<n>, gives it the legacy type and the generated
+// name, must still get the OS-resolver catch-all when it fails.
+func TestInternalDomainsUserDefinedLookalikeIsOrdinaryUpstream(t *testing.T) {
+	osFixture := startDNSFixture(t)
+	useOSResolverFixture(t, osFixture)
+
+	cfg := internalDomainsTestConfig()
+	cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true)
+	cfg.Upstream["internal_0"] = &ctrld.UpstreamConfig{
+		Name:     internalDomainUpstreamName,
+		Type:     ctrld.ResolverTypeLegacy,
+		Endpoint: deadLoopbackAddr(t),
+		Timeout:  500,
+	}
+	cfg.Listener["0"].Policy.Rules = append(cfg.Listener["0"].Policy.Rules,
+		ctrld.Rule{"*.corp.example": []string{"upstream.internal_0"}})
+	p := newInternalDomainsProg(t, cfg)
+	if p.isInternalDomainUpstream("upstream.internal_0") || isGeneratedInternalDomainUpstream(cfg.Upstream["internal_0"]) {
+		t.Fatal("a user-defined upstream was recognized as a generated Internal Domain resolver")
+	}
+
+	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+	ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+	res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+
+	if got := osFixture.questions(); len(got) != 1 {
+		t.Errorf("OS resolver questions = %v, want the ordinary catch all", got)
+	}
+	if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+		t.Errorf("answer = %v, want the OS resolver's NOERROR", res)
+	}
+}
+
+// captureInternalDomainProbes makes p's background re-checks run only when the
+// test calls them, and returns the ones scheduled so far.
+func captureInternalDomainProbes(p *prog) *[]func() {
+	var probes []func()
+	p.internalDomainProbes.start = func(run func()) { probes = append(probes, run) }
+	return &probes
+}
+
+// The off-site case that fallback mode exists for: the organization resolver
+// is unreachable and the network answers. Once the monitor reports the
+// resolver down, a query must go to the network without waiting on the
+// resolver's timeout, and a background re-check must bring it back once it
+// answers again.
+func TestInternalDomainsDownResolverIsSkippedAndReChecked(t *testing.T) {
+	network := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+	useOSResolverFixture(t, network)
+	blackhole := blackholeUDPAddr(t)
+
+	cfg := internalDomainsTestConfig()
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{blackhole}}})
+	p := newInternalDomainsProg(t, cfg)
+	probes := captureInternalDomainProbes(p)
+
+	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ask := func() (*proxyResponse, time.Duration) {
+		t.Helper()
+		ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+		ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+		start := time.Now()
+		res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+		return res, time.Since(start)
+	}
+
+	const upstream = upstreamPrefix + internalDomainUpstreamPrefix + "0"
+	p.um.mu.Lock()
+	p.um.markDown(upstream, maxFailureRequest, "immediate")
+	p.um.mu.Unlock()
+
+	for i := 0; i < 2; i++ {
+		res, elapsed := ask()
+		if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+			t.Fatalf("query %d: answer = %v, want the network's NOERROR", i+1, res)
+		}
+		// The blackholed resolver would cost its 2s read timeout.
+		if elapsed > time.Second {
+			t.Fatalf("query %d took %v: it waited on a resolver that is down", i+1, elapsed)
+		}
+	}
+	if got := len(*probes); got != 1 {
+		t.Fatalf("re-checks scheduled = %d, want one per interval", got)
+	}
+
+	// The resolver is still unreachable: the re-check leaves it down.
+	(*probes)[0]()
+	if !p.um.isDown(upstream) {
+		t.Fatal("a failed re-check marked the resolver up")
+	}
+
+	// Back on the organization network: the next re-check, once the interval
+	// has passed, finds it answering and marks it up, so queries use it again.
+	internal := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+	cfg.Upstream["internal_0"].Endpoint = internal.addr
+	later := time.Now().Add(internalDomainProbeInterval)
+	p.internalDomainProbes.now = func() time.Time { return later }
+	ask()
+	if got := len(*probes); got != 2 {
+		t.Fatalf("re-checks scheduled = %d, want a second one after the interval", got)
+	}
+	(*probes)[1]()
+	if p.um.isDown(upstream) {
+		t.Fatal("the resolver answered the re-check but is still down")
+	}
+	before := len(network.questions())
+	if res, _ := ask(); res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+		t.Fatalf("answer = %v, want the resolver's NOERROR", res)
+	}
+	if got := internal.questions(); len(got) != 2 {
+		t.Errorf("resolver questions = %v, want the re-check and the query", got)
+	}
+	if got := len(network.questions()); got != before {
+		t.Errorf("network asked again although the resolver answered")
+	}
+}
+
+// Explicit resolver only has no fallback to skip to, so a down resolver is
+// still asked: skipping it would only turn its answer into a SERVFAIL.
+func TestInternalDomainsOnlyModeDoesNotSkipDownResolver(t *testing.T) {
+	useOSResolverFixture(t, startDNSFixture(t))
+	internal := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+
+	cfg := internalDomainsTestConfig()
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{internal.addr}}})
+	p := newInternalDomainsProg(t, cfg)
+	probes := captureInternalDomainProbes(p)
+
+	p.um.mu.Lock()
+	p.um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"0", maxFailureRequest, "immediate")
+	p.um.mu.Unlock()
+
+	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+	ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+	res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+
+	if got := internal.questions(); len(got) != 1 {
+		t.Errorf("resolver questions = %v, want it asked although it is down", got)
+	}
+	if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+		t.Errorf("answer = %v, want the resolver's NOERROR", res)
+	}
+	if len(*probes) != 0 {
+		t.Errorf("explicit resolver only scheduled %d re-checks", len(*probes))
+	}
+}
+
+// fallbackTestQuery sends one query for host.corp.example through proxy().
+func fallbackTestQuery(t *testing.T, p *prog, cfg *ctrld.Config) *proxyResponse {
+	t.Helper()
+	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+	ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+	return p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+}
+
+// The OS step of the fallback is what keeps the private name off public
+// nameservers, and it does so through the context it passes. A test that
+// swaps the OS resolver for a single-endpoint fixture would pass with any
+// context, so this one sees the context itself and runs the step through a
+// real OS resolver whose pool holds a LAN and a public nameserver.
+func TestInternalDomainsNetworkFallbackOSStepIsLanOnly(t *testing.T) {
+	lan := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+	pool := ctrld.NewResolverWithNameserver([]string{lan.addr, "192.0.2.53:53"})
+	var lanOnly []bool
+	prev := internalDomainOSResolve
+	internalDomainOSResolve = func(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) {
+		isLanOnly, _ := ctx.Value(ctrld.LanOnlyQueryCtxKey{}).(bool)
+		lanOnly = append(lanOnly, isLanOnly)
+		ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
+		defer cancel()
+		return pool.Resolve(ctx, msg)
+	}
+	t.Cleanup(func() { internalDomainOSResolve = prev })
+
+	internal := startRcodeDNSFixture(t, dns.RcodeNameError, false)
+	cfg := internalDomainsTestConfig()
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}})
+	p := newInternalDomainsProg(t, cfg)
+
+	res := fallbackTestQuery(t, p, cfg)
+	if len(lanOnly) != 1 || !lanOnly[0] {
+		t.Fatalf("OS step contexts LAN-only = %v, want one LAN-only query", lanOnly)
+	}
+	if got := lan.questions(); len(got) != 1 {
+		t.Errorf("LAN nameserver questions = %v, want the fallback query", got)
+	}
+	if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+		t.Errorf("answer = %v, want the LAN nameserver's NOERROR", res)
+	}
+}
+
+// useVPNFallbackFixtures routes the fallback's VPN DNS servers to fixtures:
+// real VPN DNS servers are dialed on port 53.
+func useVPNFallbackFixtures(t *testing.T, servers map[string]string) {
+	t.Helper()
+	prev := internalDomainVPNUpstream
+	internalDomainVPNUpstream = func(_ *vpnDNSManager, server string) *ctrld.UpstreamConfig {
+		return &ctrld.UpstreamConfig{Name: "VPN DNS", Type: ctrld.ResolverTypeLegacy, Endpoint: servers[server], Timeout: 1000}
+	}
+	t.Cleanup(func() { internalDomainVPNUpstream = prev })
+}
+
+// The fallback asks VPN DNS servers that match the domain, then domain-less
+// ones, then the OS resolver, and stops at the first NOERROR.
+func TestInternalDomainsNetworkFallbackVPNOrder(t *testing.T) {
+	for _, tc := range []struct {
+		name                     string
+		matched, domainless      int
+		wantMatched, wantDomless int
+		wantOS                   bool
+	}{
+		{name: "matching VPN server answers", matched: dns.RcodeSuccess, domainless: dns.RcodeSuccess, wantMatched: 1},
+		{name: "then the domain-less one", matched: dns.RcodeNameError, domainless: dns.RcodeSuccess, wantMatched: 1, wantDomless: 1},
+		{name: "then the OS resolver", matched: dns.RcodeNameError, domainless: dns.RcodeRefused, wantMatched: 1, wantDomless: 1, wantOS: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			useDNSIntercept(t)
+			osFixture := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+			useOSResolverFixture(t, osFixture)
+			matched := startRcodeDNSFixture(t, tc.matched, false)
+			domainless := startRcodeDNSFixture(t, tc.domainless, false)
+			useVPNFallbackFixtures(t, map[string]string{"10.8.0.1": matched.addr, "10.8.0.2": domainless.addr})
+
+			internal := startRcodeDNSFixture(t, dns.RcodeNameError, false)
+			cfg := internalDomainsTestConfig()
+			applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}})
+			p := newInternalDomainsProg(t, cfg)
+			p.vpnDNS = &vpnDNSManager{
+				routes:            map[string][]string{"corp.example": {"10.8.0.1"}},
+				domainlessServers: []string{"10.8.0.2"},
+				logger:            &p.logger,
+			}
+
+			res := fallbackTestQuery(t, p, cfg)
+			if got := len(matched.questions()); got != tc.wantMatched {
+				t.Errorf("matching VPN server asked %d times, want %d", got, tc.wantMatched)
+			}
+			if got := len(domainless.questions()); got != tc.wantDomless {
+				t.Errorf("domain-less VPN server asked %d times, want %d", got, tc.wantDomless)
+			}
+			if got := len(osFixture.questions()) > 0; got != tc.wantOS {
+				t.Errorf("OS resolver asked = %v, want %v", got, tc.wantOS)
+			}
+			if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+				t.Errorf("answer = %v, want NOERROR", res)
+			}
+		})
+	}
+}
+
+// While Windows is serving retained VPN DNS state, a VPN transport failure
+// stops the fallback before the OS resolver, as it does for VPN split routing.
+func TestInternalDomainsNetworkFallbackStopsOnRetainedVPNTransportFailure(t *testing.T) {
+	useDNSIntercept(t)
+	prevSettling := vpnDNSSettlingEnabled
+	vpnDNSSettlingEnabled = true
+	t.Cleanup(func() { vpnDNSSettlingEnabled = prevSettling })
+	osFixture := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+	useOSResolverFixture(t, osFixture)
+	useVPNFallbackFixtures(t, map[string]string{"10.8.0.1": deadLoopbackAddr(t)})
+
+	internal := startRcodeDNSFixture(t, dns.RcodeNameError, false)
+	cfg := internalDomainsTestConfig()
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}})
+	p := newInternalDomainsProg(t, cfg)
+	p.vpnDNS = &vpnDNSManager{
+		routes:                      map[string][]string{"corp.example": {"10.8.0.1"}},
+		retainedAfterEmptyDiscovery: true,
+		logger:                      &p.logger,
+	}
+
+	res := fallbackTestQuery(t, p, cfg)
+	if got := osFixture.questions(); len(got) != 0 {
+		t.Errorf("OS resolver asked %v while retained VPN DNS state is active", got)
+	}
+	if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeNameError {
+		t.Errorf("answer = %v, want the configured resolver's NXDOMAIN", res)
+	}
+}

cmd/cli/internal_domains_review_test.go

@@ -140,9 +140,10 @@ func TestInternalDomainsExplicitResolverSurvivesRecoveryBypass(t *testing.T) {
 	}
 }
 
-// When the configured resolvers are unreachable the query must fail rather than
-// reach the OS resolver, and the failure must be observable on the wire as an
-// attempt against the configured address.
+// In "explicit resolver only" mode, when the configured resolvers are
+// unreachable the query must fail rather than reach the OS resolver, and the
+// failure must be observable on the wire as an attempt against the configured
+// address.
 func TestInternalDomainsUnreachableResolverDoesNotReachOSResolver(t *testing.T) {
 	osFixture := startDNSFixture(t)
 	useOSResolverFixture(t, osFixture)
@@ -158,7 +159,7 @@ func TestInternalDomainsUnreachableResolverDoesNotReachOSResolver(t *testing.T)
 
 	cfg := internalDomainsTestConfig()
 	cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true)
-	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{dead}}})
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{dead}}})
 	p := newInternalDomainsProg(t, cfg)
 
 	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
@@ -304,6 +305,10 @@ func TestInternalDomainsSetupKeepsResolverAddressesOutOfInfoLogs(t *testing.T) {
 // at error level.
 func TestInternalDomainsResolverFailureLogsNoAddress(t *testing.T) {
 	buf := captureInternalDomainsLogs(t)
+	// The default mode falls back to the network after the failure; keep that
+	// on loopback.
+	useOSResolverFixture(t, startDNSFixture(t))
+
 	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
 	if err != nil {
 		t.Fatal(err)
@@ -344,9 +349,13 @@ func TestInternalDomainsResolverFailureLogsNoAddress(t *testing.T) {
 
 	// Master's transport-level diagnostic must stay at error for ordinary
 	// legacy upstreams; only generated Internal Domain queries opt into privacy.
+	// Generated upstreams are identified by their InternalDomain marker, not
+	// by name, so the control is an ordinary upstream to the same endpoint.
+	ordinary := &ctrld.UpstreamConfig{Name: "ordinary", Type: ctrld.ResolverTypeLegacy, Endpoint: dead, Timeout: 5000}
+	ordinary.Init(ctx)
 	before = len(buf.String())
 	p.queryUpstream(ctx, &proxyRequest{msg: newDnsMsgWithHostname("public.example.", dns.TypeA)},
-		"upstream.0", cfg.Upstream["internal_0"])
+		"upstream.0", ordinary)
 	ordinaryError := false
 	for _, line := range strings.Split(buf.String()[before:], "\n") {
 		if strings.Contains(line, `"level":"error"`) && strings.Contains(line, "Legacy request failed") && strings.Contains(line, dead) {

cmd/cli/internal_domains_test.go

@@ -422,6 +422,11 @@ func TestInternalDomainsEqual(t *testing.T) {
 }
 
 func TestInternalDomainExplicitUpstreams(t *testing.T) {
+	p := internalDomainUpstreamsProg(map[string]string{
+		"internal_0": internalDomainUpstreamFallback,
+		"internal_1": internalDomainUpstreamOnly,
+		"0":          "",
+	})
 	for _, tc := range []struct {
 		in   []string
 		want bool
@@ -434,7 +439,7 @@ func TestInternalDomainExplicitUpstreams(t *testing.T) {
 		{nil, false},
 		{[]string{}, false},
 	} {
-		if got := internalDomainExplicitUpstreams(tc.in); got != tc.want {
+		if got := p.internalDomainExplicitUpstreams(tc.in); got != tc.want {
 			t.Errorf("internalDomainExplicitUpstreams(%v) = %v, want %v", tc.in, got, tc.want)
 		}
 	}
@@ -467,14 +472,14 @@ func TestInternalDomainsSummaryLogsCountsOnly(t *testing.T) {
 
 // An explicit-resolver Internal Domain must not be handed to VPN DNS split
 // routing: the administrator named those resolvers, and VPN suffixes are
-// auto-detected. When the named resolvers are unreachable the query fails
-// instead of reaching the OS resolver.
+// auto-detected. In "explicit resolver only" mode, when the named resolvers
+// are unreachable the query fails instead of reaching the OS resolver.
 func TestInternalDomainsBeatVPNSplitRoutingAndDoNotLeak(t *testing.T) {
 	buf := captureInternalDomainsLogs(t)
 	cfg := internalDomainsTestConfig()
 	cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true)
 	applyInternalDomains(cfg, []controld.SplitDNS{
-		{Domain: "aws.example.com", Resolvers: []string{"127.0.0.1:1"}},
+		{Domain: "aws.example.com", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{"127.0.0.1:1"}},
 	})
 
 	p := newInternalDomainsProg(t, cfg)
@@ -490,7 +495,7 @@ func TestInternalDomainsBeatVPNSplitRoutingAndDoNotLeak(t *testing.T) {
 	}
 	ctx := context.WithValue(ctrld.LoggerCtx(context.Background(), p.logger.Load()), ctrld.ReqIdCtxKey{}, requestID())
 	ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.aws.example.com")
-	if !internalDomainExplicitUpstreams(ufr.upstreams) {
+	if !p.internalDomainExplicitUpstreams(ufr.upstreams) {
 		t.Fatalf("upstreams = %v, want only internal resolvers", ufr.upstreams)
 	}
 
@@ -587,9 +592,18 @@ func TestApplyInternalDomainsModeIsAuthoritative(t *testing.T) {
 			wantUpstreams: 1,
 		},
 		{
-			// A mode ctrld does not know is not guessed at.
-			name:          "unknown mode is dropped",
+			// A wire value this build does not know fails closed to the
+			// resolvers the administrator chose, never to the Control D upstream.
+			name:          "unknown mode with resolvers fails closed",
 			entry:         controld.SplitDNS{Domain: "a.example.com", Mode: "forward-someday", Resolvers: []string{"10.0.0.53"}},
+			wantTargets:   []string{"upstream.internal_0"},
+			wantRule:      true,
+			wantUpstreams: 2,
+		},
+		{
+			// With no resolvers there is nothing to fail closed to.
+			name:          "unknown mode without resolvers is dropped",
+			entry:         controld.SplitDNS{Domain: "a.example.com", Mode: "forward-someday"},
 			wantRule:      false,
 			wantUpstreams: 1,
 		},

cmd/cli/prog.go

@@ -148,6 +148,7 @@ type prog struct {
 	internalLogWriter         *logWriter
 	internalJournalWriter     *logWriter
 	querySampler              errorSampler
+	internalDomainProbes      internalDomainProber
 	lastNetworkState          atomic.Pointer[netmon.State]
 	logUpload                 logUploadGate
 	logSendMu                 sync.Mutex
@@ -982,8 +983,8 @@ func (p *prog) setupUpstream(cfg *ctrld.Config) {
 	// not independent user upstreams. Recompute eligibility on every setup/reload;
 	// uninstall still requires API confirmation that the device is gone.
 	managedUpstreams := 0
-	for n, uc := range cfg.Upstream {
-		if !isGeneratedInternalDomainUpstream(n, uc) {
+	for _, uc := range cfg.Upstream {
+		if !isGeneratedInternalDomainUpstream(uc) {
 			managedUpstreams++
 		}
 	}
@@ -1259,7 +1260,7 @@ func (p *prog) startNetworkJournal() func() {
 		close(stop)
 	}()
 	healthDone := p.health.startLoop(stop, func() (int, bool) {
-		return p.upstreamMonitorNow().countDownExcept(isInternalDomainUpstream), p.recoveryBypass.Load()
+		return p.upstreamMonitorNow().countDownExcept(p.isInternalDomainUpstream), p.recoveryBypass.Load()
 	})
 	var configDone <-chan struct{}
 	if p.dnsConfig != nil {

cmd/cli/query_health_prog_test.go

@@ -108,10 +108,11 @@ func Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade(t *testing.T) {
 	cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false)
 	cfg.Upstream = map[string]*ctrld.UpstreamConfig{
 		internalDomainUpstreamPrefix + "1": {
-			Name:     "dead internal resolver",
-			Type:     ctrld.ResolverTypeLegacy,
-			Endpoint: deadUpstreamEndpoint,
-			Timeout:  500,
+			Name:           "dead internal resolver",
+			Type:           ctrld.ResolverTypeLegacy,
+			Endpoint:       deadUpstreamEndpoint,
+			Timeout:        500,
+			InternalDomain: internalDomainUpstreamOnly,
 		},
 	}
 	p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load()), health: newQueryHealth()}
@@ -142,7 +143,7 @@ func Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade(t *testing.T) {
 	p.um.markDown(internalUpstream, 3, "immediate")
 	p.um.mu.Unlock()
 
-	event, emit := p.health.evaluate(time.Now(), p.um.countDownExcept(isInternalDomainUpstream), false)
+	event, emit := p.health.evaluate(time.Now(), p.um.countDownExcept(p.isInternalDomainUpstream), false)
 	if !emit {
 		t.Fatal("the first evaluate did not emit")
 	}

cmd/cli/recovery_scope_test.go

@@ -323,7 +323,9 @@ func TestScopeHarnessRestoresRecoveryWait(t *testing.T) {
 
 func TestOSRecoveryCandidates(t *testing.T) {
 	remote := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeDOH}
-	internal := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy}
+	internal := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy, InternalDomain: internalDomainUpstreamFallback}
+	// A user-defined upstream that copies a generated key carries no marker.
+	lookalike := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy}
 	osConfig := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeOS}
 	for _, tc := range []struct {
 		name       string
@@ -331,9 +333,10 @@ func TestOSRecoveryCandidates(t *testing.T) {
 		want       map[string]*ctrld.UpstreamConfig
 	}{
 		{"configured", map[string]*ctrld.UpstreamConfig{"0": remote, "internal_0": internal, "1": osConfig, "2": nil}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote}},
-		{"custom_policy_or_unused", map[string]*ctrld.UpstreamConfig{"0": remote, "unused": internal}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote, "upstream.unused": internal}},
+		{"custom_policy_or_unused", map[string]*ctrld.UpstreamConfig{"0": remote, "unused": lookalike}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote, "upstream.unused": lookalike}},
 		{"only_os", map[string]*ctrld.UpstreamConfig{"0": osConfig}, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}},
 		{"only_internal", map[string]*ctrld.UpstreamConfig{"internal_0": internal}, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}},
+		{"internal_lookalike", map[string]*ctrld.UpstreamConfig{"internal_0": lookalike}, map[string]*ctrld.UpstreamConfig{"upstream.internal_0": lookalike}},
 		{"empty", nil, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}},
 	} {
 		t.Run(tc.name, func(t *testing.T) {

cmd/cli/recovery_skip_test.go

@@ -139,7 +139,11 @@ func TestOSRecoverySkipSurvivesDebugTruncation(t *testing.T) {
 	for range 40 {
 		mainLog.Load().Debug().Msg(strings.Repeat("debug filler", 20))
 	}
-	debugPart, journalPart := splitUpload(t, readLogReader(t, p, false), time.Now())
+	// Taken before the read renders the upload header: as an argument after
+	// readLogReader it would be evaluated later, and fail whenever a second
+	// boundary falls between the two.
+	start := time.Now()
+	debugPart, journalPart := splitUpload(t, readLogReader(t, p, false), start)
 	if strings.Contains(string(debugPart), recoverySkipMessage) {
 		t.Fatal("fixture did not truncate the skip from debug")
 	}

cmd/cli/upstream_monitor_test.go

@@ -277,7 +277,12 @@ func Test_upstreamMonitorCountDownExcept(t *testing.T) {
 	if got := um.countDown(); got != 3 {
 		t.Fatalf("countDown = %d, want 3", got)
 	}
-	if got := um.countDownExcept(isInternalDomainUpstream); got != 1 {
+	p := internalDomainUpstreamsProg(map[string]string{
+		"0":                                "",
+		internalDomainUpstreamPrefix + "1": internalDomainUpstreamFallback,
+		internalDomainUpstreamPrefix + "2": internalDomainUpstreamOnly,
+	})
+	if got := um.countDownExcept(p.isInternalDomainUpstream); got != 1 {
 		t.Fatalf("countDownExcept(isInternalDomainUpstream) = %d, want 1", got)
 	}
 	if got := um.countDownExcept(nil); got != 3 {