cli: add network fallback and explicit-only Internal Domain modes

Internal Domains now support three modes. "os" (network default) is
unchanged. "resolvers" becomes explicit resolver with network fallback,
the default explicit selection: the configured resolvers are tried
first, and a timeout, unreachable resolver, SERVFAIL, NXDOMAIN, REFUSED
or NOTIMP hands the query to the network resolvers - matching VPN DNS
servers, then domain-less VPN DNS servers, then the OS resolver's LAN
nameservers. A valid empty answer stays final. "resolvers_only" keeps
the previous strict behavior.

The OS step uses a new LanOnlyQueryCtx: the OS resolver then asks only
its LAN-classified nameservers (private, loopback, link-local, CGNAT)
and sends nothing when it has none, so the private name never reaches a
public nameserver, whether DHCP supplied it or it is ctrld's own public
fallback. LanQueryCtx only dropped the latter. LAN-only queries get
their own singleflight and hot-cache key, so they are never answered
with an ordinary query's public answer.

In fallback mode a configured resolver that the upstream monitor has
marked down is skipped instead of waited on, so an endpoint off the
organization network reaches the fallback without paying the
resolver's 2s timeout on every query. Nothing else marks a generated
resolver up again, so skipping one re-checks it in the background, at
most once every 30s; an answer marks it up. Strict mode never skips.

An absent mode with resolvers gets the fallback mode. An unrecognized
mode with resolvers fails closed to explicit resolver only, so a wire
value this build does not know never sends the domain to the Control D
upstream; without resolvers it is dropped. Switching between the
explicit modes is detected on refresh.

Generated upstreams are now identified by an UpstreamConfig.InternalDomain
marker that records their mode and that no configuration file can set,
instead of by the internal_ key prefix. An upstream a local ctrld.toml or
custom config defines as internal_foo, or as a copy of a generated one,
is an ordinary upstream again: it keeps the OS-resolver catch-all,
recovery and query-health grading.

The "resolvers_only" wire value is provisional until the API agrees on
it; the dashboard does not offer strict mode yet.

cmd/cli/cli.go

@@ -1058,6 +1058,7 @@ func logInternalDomainsSummary(logger zerolog.Logger, summary internalDomainsSum
 	event.Int("domains", summary.domains).
 		Int("os_resolver", summary.osMode).
 		Int("explicit_resolver", summary.explicit).
+		Int("explicit_resolver_only", summary.only).
 		Int("resolvers", summary.resolvers).
 		Int("skipped", summary.skipped).
 		Int("preempted", summary.preempted).

cmd/cli/dns_proxy.go

@@ -577,11 +577,13 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse {
 	//
 	// An Internal Domain with explicit resolvers does not take part. The bypass
 	// exists because general DNS is broken, which says nothing about the
-	// administrator's own resolvers: they may well be reachable, and sending a
-	// private name to the network's DNS is the disclosure the explicit
-	// selection exists to prevent. Such a query continues to the normal flow,
-	// where it reaches only its configured resolvers and fails if none answer.
-	if dnsIntercept && p.recoveryBypass.Load() && !internalDomainExplicitUpstreams(req.ufr.upstreams) {
+	// administrator's own resolvers: they may well be reachable, and they are
+	// to be tried first. Such a query continues to the normal flow, where it
+	// reaches its configured resolvers first. In "explicit resolver only" mode
+	// it fails if none answer; otherwise it falls back to the VPN resolvers and
+	// the network's LAN nameservers, never to the public nameservers the bypass
+	// may use.
+	if dnsIntercept && p.recoveryBypass.Load() && !p.internalDomainExplicitUpstreams(req.ufr.upstreams) {
 		ctrld.Log(ctx, mainLog.Load().Debug(), "Recovery bypass active: forwarding to OS resolver")
 		resolver, err := ctrld.NewResolver(osUpstreamConfig)
 		if err == nil {
@@ -713,13 +715,15 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse {
 	//
 	// An Internal Domain with explicit resolvers is skipped here: the
 	// administrator named the resolvers for that suffix, and VPN suffixes
-	// are auto-detected, so handing the query to a VPN DNS server would
-	// override an explicit selection with a discovered one.
+	// are auto-detected, so handing the query to a VPN DNS server first would
+	// override an explicit selection with a discovered one. In fallback mode
+	// the VPN DNS servers are asked later, once the configured resolvers have
+	// not resolved the name.
 	//
 	// Internal Domains in OS-resolver mode are not skipped, they ask for the
 	// endpoint's default resolution, which under intercept mode includes
 	// the VPN's own resolver.
-	if dnsIntercept && p.vpnDNS != nil && len(req.msg.Question) > 0 && !internalDomainExplicitUpstreams(upstreams) {
+	if dnsIntercept && p.vpnDNS != nil && len(req.msg.Question) > 0 && !p.internalDomainExplicitUpstreams(upstreams) {
 		domain := req.msg.Question[0].Name
 		if vpnServers := p.vpnDNS.UpstreamForDomain(domain); len(vpnServers) > 0 {
 			ctrld.Log(ctx, mainLog.Load().Debug(), "VPN DNS route matched for domain %s, using servers: %v", domain, vpnServers)
@@ -871,7 +875,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse {
 		// generated Internal Domain upstream is the organization's private
 		// resolver address. Report the classification through the sampler and
 		// keep the address-bearing error at debug.
-		if isInternalDomainUpstream(upstream) {
+		if isGeneratedInternalDomainUpstream(upstreamConfig) {
 			ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "failed to resolve query")
 			ctrld.Log(ctx, p.querySampler.event(sampleClassInternalDomain, upstream).Str("failure", internalDomainFailureReason(err)),
 				"failed to resolve query using an Internal Domain resolver")
@@ -897,6 +901,12 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse {
 
 		return nil
 	}
+	// An Internal Domain in explicit-with-network-fallback mode moves on from a
+	// configured resolver that answers SERVFAIL or NXDOMAIN, as it does from
+	// one that does not answer. internalNegative keeps the best of those
+	// answers, so a name nobody resolves gets a real negative answer.
+	internalFallback := p.internalDomainFallbackUpstreams(upstreams)
+	var internalNegative *dns.Msg
 	for n, upstreamConfig := range upstreamConfigs {
 		if upstreamConfig == nil {
 			continue
@@ -911,6 +921,16 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse {
 			ctrld.Log(ctx, logger, "DNS loop detected")
 			continue
 		}
+		// A fallback-mode Internal Domain resolver that the monitor reports as
+		// down is not waited on: off the organization network every query
+		// would otherwise pay its timeout before the network fallback runs.
+		// Nothing else marks a generated resolver up again, so skipping it
+		// starts a background re-check that does.
+		if internalFallback && p.um.isDown(upstreams[n]) {
+			ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolver %s is down, skipping it", upstreams[n])
+			p.internalDomainProbes.probe(p.um, upstreams[n], upstreamConfig, req.msg)
+			continue
+		}
 		answer := resolve(upstreams[n], upstreamConfig, req.msg)
 		if answer == nil {
 			if serveStaleCache && staleAnswer != nil {
@@ -945,6 +965,12 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse {
 			ctrld.Log(ctx, mainLog.Load().Debug(), "failover rcode matched, process to next upstream")
 			continue
 		}
+		if internalFallback && internalDomainFallbackRcode(answer.Rcode) {
+			ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolver %s answered %s, trying next resolver",
+				upstreams[n], dns.RcodeToString[answer.Rcode])
+			internalNegative = betterInternalDomainNegative(internalNegative, answer)
+			continue
+		}
 
 		// set compression, as it is not set by default when unpacking
 		answer.Compress = true
@@ -989,15 +1015,40 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse {
 		res.upstream = upstreamConfig.Endpoint
 		return res
 	}
+	if internalFallback {
+		ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolvers did not resolve the query; trying network resolvers")
+		answer, negative := p.resolveInternalDomainOnNetwork(ctx, req.msg)
+		if answer != nil {
+			answer.Compress = true
+			if cacheable {
+				p.addCachedResponse(dnscache.NewKey(req.msg, upstreams[0]), answer)
+				ctrld.Log(ctx, mainLog.Load().Debug(), "add cached response")
+			}
+			hostname := ""
+			if req.ci != nil {
+				hostname = req.ci.Hostname
+			}
+			ctrld.Log(ctx, mainLog.Load().Info(), "REPLY: internal domain network fallback -> %s (%s): %s", req.ufr.srcAddr, hostname, dns.RcodeToString[answer.Rcode])
+			res.answer = answer
+			return res
+		}
+		if negative = betterInternalDomainNegative(internalNegative, negative); negative != nil {
+			ctrld.Log(ctx, mainLog.Load().Debug(), "no resolver resolved the internal domain; returning %s", dns.RcodeToString[negative.Rcode])
+			negative.Compress = true
+			res.answer = negative
+			return res
+		}
+	}
 	ctrld.Log(ctx, p.querySampler.event(sampleClassAllEndpointsFailed, ""), "all %v endpoints failed", journalUpstreamNames(upstreams))
 
-	// An Internal Domain with explicit resolvers is served only by the
-	// configured resolvers. When all of them are unreachable the query fails
-	// here: the OS-resolver catch-all below would send a private name to a resolver
-	// that was not selected for it, and the recovery flow exists for the loss of
-	// general DNS, not for one unavailable internal server.
-	if internalDomainExplicitUpstreams(upstreams) {
-		ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolvers unreachable; not falling back")
+	// An Internal Domain with explicit resolvers never takes the OS-resolver
+	// catch-all below: it can reach public nameservers, ctrld's own fallback
+	// included, which no mode selects for a private name. Fallback mode has
+	// already tried the VPN resolvers and the network's LAN nameservers above.
+	// The recovery flow exists for the loss of general DNS, not for one
+	// unavailable internal server, so it is not started either.
+	if p.internalDomainExplicitUpstreams(upstreams) {
+		ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolvers unreachable; not using the OS resolver catch all")
 	} else if p.leakOnUpstreamFailure() {
 		if p.um.countHealthy(upstreams) == 0 {
 			p.recoveryCancelMu.Lock()
@@ -1046,7 +1097,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse {
 // endpoint away from the organization network never reaches them, and that is
 // not an outage of the query path.
 func (p *prog) countFailedClientQuery(upstreams []string) {
-	if internalDomainExplicitUpstreams(upstreams) {
+	if p.internalDomainExplicitUpstreams(upstreams) {
 		return
 	}
 	p.health.countFailedQuery()
@@ -2705,7 +2756,7 @@ func (p *prog) buildRecoveryUpstreams(reason RecoveryReason) map[string]*ctrld.U
 	case RecoveryReasonOSFailure:
 		for k, uc := range p.cfg.Upstream {
 			name := upstreamPrefix + k
-			if uc != nil && uc.Type != ctrld.ResolverTypeOS && !isInternalDomainUpstream(name) {
+			if uc != nil && uc.Type != ctrld.ResolverTypeOS && !isGeneratedInternalDomainUpstream(uc) {
 				upstreams[name] = uc
 			}
 		}

cmd/cli/internal_domains.go

@@ -9,8 +9,11 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
+	"time"
 
+	"github.com/miekg/dns"
 	"github.com/rs/zerolog"
 
 	"github.com/Control-D-Inc/ctrld"
@@ -19,15 +22,25 @@ import (
 )
 
 // internalDomainUpstreamPrefix names the upstreams generated for organization
-// Internal Domains that select explicit resolvers. The prefix is what tells
-// proxy() that a rule is served only by configured resolvers, so it
-// must not collide with the numeric keys used for the Control D upstream.
+// Internal Domains that select explicit resolvers. It must not collide with
+// the numeric key used for the Control D upstream. The key only names the
+// upstream: what identifies it as generated, and in which mode, is
+// UpstreamConfig.InternalDomain, which no configuration file can set.
 const internalDomainUpstreamPrefix = "internal_"
 
+// Values of UpstreamConfig.InternalDomain on a generated upstream.
+const (
+	// internalDomainUpstreamFallback serves an Internal Domain in explicit
+	// resolver with network fallback mode.
+	internalDomainUpstreamFallback = "fallback"
+	// internalDomainUpstreamOnly serves an Internal Domain in explicit
+	// resolver only mode: its answer or failure is final.
+	internalDomainUpstreamOnly = "only"
+)
+
 // internalDomainUpstreamName is the Name given to every generated Internal
-// Domain upstream. Together with the key prefix and the legacy type it is what
-// distinguishes an upstream ctrld generated from one an endpoint custom
-// configuration happens to name the same way.
+// Domain upstream. It is only a display label: what identifies an upstream as
+// generated is UpstreamConfig.InternalDomain, which no configuration can set.
 const internalDomainUpstreamName = "Internal Domain resolver"
 
 // internalDomainsSummary counts the outcome of applying Internal Domains. It
@@ -37,6 +50,7 @@ type internalDomainsSummary struct {
 	domains   int // domains that produced rules
 	osMode    int // domains routed to the OS/default resolver
 	explicit  int // domains routed to configured resolvers
+	only      int // explicit domains with no network fallback
 	resolvers int // explicit resolver upstreams generated
 	skipped   int // entries dropped: unusable domain, no usable resolver, or duplicate
 	preempted int // domains already covered by a higher precedence rule
@@ -129,43 +143,83 @@ func internalDomainEndpoint(resolver string) (string, bool) {
 	return net.JoinHostPort(addr.String(), port), true
 }
 
-// internalDomainMode reports whether entry selects explicit resolvers, and
-// whether ctrld understands its mode at all.
+// internalDomainResolution is how an Internal Domain is resolved.
+type internalDomainResolution int
+
+const (
+	// internalDomainViaOS is the "network default" mode: the endpoint's
+	// OS/default resolver.
+	internalDomainViaOS internalDomainResolution = iota
+	// internalDomainViaResolvers is the configured resolvers first, then the
+	// active network and VPN resolvers. The default explicit selection.
+	internalDomainViaResolvers
+	// internalDomainViaResolversOnly is the configured resolvers and nothing
+	// else.
+	internalDomainViaResolversOnly
+)
+
+// internalDomainMode reports how entry is resolved, and whether ctrld
+// understands its mode at all.
 //
 // Mode is authoritative when the API sends it. "os" ignores any resolver
 // addresses the entry still carries, so switching a domain back to the OS
 // resolver cannot be undone by addresses a previous selection left behind;
-// "resolvers" stays explicit even when the list it points at turns out to be
-// unusable, rather than silently becoming OS resolution. An empty mode is a
-// deployment that predates the field, where the list is the only signal.
+// the explicit modes stay explicit even when the list they point at turns out
+// to be unusable, rather than silently becoming OS resolution. An empty mode
+// is a deployment that predates the field, where the list is the only signal,
+// and an explicit list gets the default explicit mode.
 //
 // An unrecognized mode is not guessed at: routing a private domain by guess is
-// the disclosure this feature exists to prevent.
-func internalDomainMode(entry controld.SplitDNS) (explicit, known bool) {
+// the disclosure this feature exists to prevent. It fails closed instead. With
+// resolvers it is resolved as explicit resolver only, through the resolvers the
+// administrator chose and nothing else, so a wire value this build does not
+// know, such as a renamed strict mode, cannot send the domain to the Control D
+// upstream or the network. Without resolvers there is nothing to fail closed
+// to, so the entry is reported as unknown and dropped.
+func internalDomainMode(entry controld.SplitDNS) (resolution internalDomainResolution, known bool) {
 	switch strings.ToLower(strings.TrimSpace(entry.Mode)) {
 	case controld.SplitDNSModeOS:
-		return false, true
+		return internalDomainViaOS, true
 	case controld.SplitDNSModeResolvers:
-		return true, true
+		return internalDomainViaResolvers, true
+	case controld.SplitDNSModeResolversOnly:
+		return internalDomainViaResolversOnly, true
 	case "":
-		return len(entry.Resolvers) > 0, true
+		if len(entry.Resolvers) > 0 {
+			return internalDomainViaResolvers, true
+		}
+		return internalDomainViaOS, true
 	default:
-		return false, false
+		if len(entry.Resolvers) > 0 {
+			return internalDomainViaResolversOnly, true
+		}
+		return internalDomainViaOS, false
 	}
 }
 
-// internalDomainRouting returns the endpoints an entry routes to, or a reason
-// it cannot be used. No endpoints and no reason means OS resolution.
+// internalDomainModeRecognized reports whether mode is one this build knows,
+// as opposed to one internalDomainMode resolves by failing closed.
+func internalDomainModeRecognized(mode string) bool {
+	switch strings.ToLower(strings.TrimSpace(mode)) {
+	case "", controld.SplitDNSModeOS, controld.SplitDNSModeResolvers, controld.SplitDNSModeResolversOnly:
+		return true
+	}
+	return false
+}
+
+// internalDomainRouting returns how an entry is resolved and the endpoints it
+// routes to, or a reason it cannot be used. No endpoints and no reason means
+// OS resolution.
 //
 // Generation and refresh comparison both go through here, so the two can never
 // disagree about what an entry means.
-func internalDomainRouting(entry controld.SplitDNS) (endpoints []string, reason string) {
-	explicit, known := internalDomainMode(entry)
+func internalDomainRouting(entry controld.SplitDNS) (resolution internalDomainResolution, endpoints []string, reason string) {
+	resolution, known := internalDomainMode(entry)
 	if !known {
-		return nil, "unknown_mode"
+		return resolution, nil, "unknown_mode"
 	}
-	if !explicit {
-		return nil, ""
+	if resolution == internalDomainViaOS {
+		return resolution, nil, ""
 	}
 	endpoints = make([]string, 0, len(entry.Resolvers))
 	for _, resolver := range entry.Resolvers {
@@ -178,9 +232,9 @@ func internalDomainRouting(entry controld.SplitDNS) (endpoints []string, reason
 	// different one, so the entry is dropped and the domain keeps its normal
 	// path instead.
 	if len(endpoints) == 0 {
-		return nil, "no_usable_resolver"
+		return resolution, nil, "no_usable_resolver"
 	}
-	return endpoints, ""
+	return resolution, endpoints, ""
 }
 
 // internalDomainRuleExists reports whether the listener policy already routes
@@ -241,7 +295,8 @@ func orderInternalDomainsBySpecificity(entries []controld.SplitDNS) []controld.S
 // resolvers routes to the OS/default resolver by way of an empty upstream list,
 // the same representation Magic Folder excludes use. An entry with resolvers
 // gets one generated upstream per address, listed in configured order so the
-// later ones act as failover for the first.
+// later ones act as failover for the first. Those upstreams carry the mode in
+// UpstreamConfig.InternalDomain.
 //
 // cfg is rebuilt from the API response on every fetch, so this function is the
 // only writer of internal_* upstreams: a removed domain leaves nothing behind.
@@ -271,12 +326,15 @@ func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) intern
 		// written: an entry that cannot be used, or a domain already covered by
 		// a higher precedence rule, must not leave a generated upstream behind
 		// with no rule referring to it.
-		endpoints, reason := internalDomainRouting(entry)
+		resolution, endpoints, reason := internalDomainRouting(entry)
 		if reason != "" {
 			mainLog.Load().Debug().Msgf("internal domains: skipping %q: %s", domain, reason)
 			summary.skipped++
 			continue
 		}
+		if !internalDomainModeRecognized(entry.Mode) {
+			mainLog.Load().Debug().Msgf("internal domains: %q has unrecognized mode %q; using its resolvers only", domain, entry.Mode)
+		}
 		seen[domain] = struct{}{}
 
 		sources := []string{domain, "*." + domain}
@@ -297,14 +355,19 @@ func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) intern
 			continue
 		}
 
+		marker := internalDomainUpstreamFallback
+		if resolution == internalDomainViaResolversOnly {
+			marker = internalDomainUpstreamOnly
+		}
 		targets := make([]string, 0, len(endpoints))
 		for _, endpoint := range endpoints {
 			key := internalDomainUpstreamPrefix + strconv.Itoa(summary.resolvers)
 			cfg.Upstream[key] = &ctrld.UpstreamConfig{
-				Name:     internalDomainUpstreamName,
-				Type:     ctrld.ResolverTypeLegacy,
-				Endpoint: endpoint,
-				Timeout:  5000,
+				Name:           internalDomainUpstreamName,
+				Type:           ctrld.ResolverTypeLegacy,
+				Endpoint:       endpoint,
+				Timeout:        5000,
+				InternalDomain: marker,
 			}
 			targets = append(targets, upstreamPrefix+key)
 			summary.resolvers++
@@ -323,63 +386,118 @@ func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) intern
 			mainLog.Load().Debug().Msgf("internal domains: %q routed to the OS resolver", domain)
 		} else {
 			summary.explicit++
+			if resolution == internalDomainViaResolversOnly {
+				summary.only++
+			}
 			mainLog.Load().Debug().Msgf("internal domains: %q routed to %v", domain, targets)
 		}
 	}
 	return summary
 }
 
-// isGeneratedInternalDomainUpstream reports whether a cfg.Upstream entry is one
+// isGeneratedInternalDomainUpstream reports whether uc is an upstream
 // applyInternalDomains created.
 //
 // Generated Internal Domain resolvers are part of the managed Control D
 // configuration, not upstreams the endpoint operator chose, so eligibility
 // checks that ask "is this a plain single-upstream Control D install" must not
-// count them. The key prefix alone is not proof of that, because an endpoint
-// custom configuration replaces the generated config outright and could name an
-// upstream anything; the shape this function checks is only produced here.
-func isGeneratedInternalDomainUpstream(name string, uc *ctrld.UpstreamConfig) bool {
-	return uc != nil &&
-		strings.HasPrefix(name, internalDomainUpstreamPrefix) &&
-		uc.Type == ctrld.ResolverTypeLegacy &&
-		uc.Name == internalDomainUpstreamName
+// count them. Neither the key nor any other configurable field is proof of
+// that: a local config or an endpoint custom configuration can name an upstream
+// anything and give it any type and name. Only the InternalDomain marker is,
+// because ctrld sets it and no configuration file can.
+func isGeneratedInternalDomainUpstream(uc *ctrld.UpstreamConfig) bool {
+	return uc != nil && uc.InternalDomain != ""
 }
 
-// isInternalDomainUpstream reports whether an upstream reference was generated
-// for an Internal Domain with explicit resolvers.
-func isInternalDomainUpstream(upstream string) bool {
-	return strings.HasPrefix(upstream, upstreamPrefix+internalDomainUpstreamPrefix)
+// internalDomainUpstream returns the generated Internal Domain upstream an
+// "upstream.<key>" reference names, or nil if it names any other upstream.
+func (p *prog) internalDomainUpstream(upstream string) *ctrld.UpstreamConfig {
+	if p.cfg == nil {
+		return nil
+	}
+	uc := p.cfg.Upstream[strings.TrimPrefix(upstream, upstreamPrefix)]
+	if !isGeneratedInternalDomainUpstream(uc) {
+		return nil
+	}
+	return uc
+}
+
+// isInternalDomainUpstream reports whether an upstream reference names a
+// generated Internal Domain resolver, in either explicit mode.
+func (p *prog) isInternalDomainUpstream(upstream string) bool {
+	return p.internalDomainUpstream(upstream) != nil
 }
 
 // internalDomainExplicitUpstreams reports whether every upstream serving a
-// query is an configured Internal Domain resolver.
+// query is a configured Internal Domain resolver, in either explicit mode.
 //
-// proxy() uses this to keep such a query off the OS-resolver catch-all when the
-// configured resolvers are unreachable: the administrator selected those
-// resolvers and no other, so a private name must fail rather than be sent
-// somewhere it was not meant to go. It also keeps an unreachable internal
-// resolver from triggering the endpoint-wide recovery flow, which exists for
-// the loss of general DNS, not for one unavailable internal server.
-func internalDomainExplicitUpstreams(upstreams []string) bool {
+// proxy() uses this to try the configured resolvers before anything else, and
+// to keep such a query off the OS-resolver catch-all when they fail: that
+// catch-all may reach public nameservers, and a private name must not be sent
+// somewhere it was not meant to go. In fallback mode proxy() tries the
+// network resolvers instead (see internalDomainFallbackUpstreams). It also
+// keeps an unreachable internal resolver from triggering the endpoint-wide
+// recovery flow, which exists for the loss of general DNS, not for one
+// unavailable internal server.
+func (p *prog) internalDomainExplicitUpstreams(upstreams []string) bool {
 	if len(upstreams) == 0 {
 		return false
 	}
 	for _, upstream := range upstreams {
-		if !isInternalDomainUpstream(upstream) {
+		if !p.isInternalDomainUpstream(upstream) {
 			return false
 		}
 	}
 	return true
 }
 
+// internalDomainFallbackUpstreams reports whether upstreams are the configured
+// resolvers of an Internal Domain in explicit-with-network-fallback mode, whose
+// failure, SERVFAIL or NXDOMAIN hands the query to the active network and VPN
+// resolvers.
+func (p *prog) internalDomainFallbackUpstreams(upstreams []string) bool {
+	if len(upstreams) == 0 {
+		return false
+	}
+	for _, upstream := range upstreams {
+		uc := p.internalDomainUpstream(upstream)
+		if uc == nil || uc.InternalDomain != internalDomainUpstreamFallback {
+			return false
+		}
+	}
+	return true
+}
+
+// internalDomainFallbackRcode reports whether a configured resolver's answer
+// sends a fallback-mode query on to the next resolver. SERVFAIL means the
+// resolver could not answer, and NXDOMAIN may only mean it does not know a
+// name the local network's resolver does. REFUSED and NOTIMP mean it will not
+// serve this endpoint, as an organization resolver with an access list answers
+// an endpoint off the organization network, which is closer to unreachable
+// than to an answer. Any other answer is final, including an empty NOERROR:
+// the name exists, it just has no record of that type.
+func internalDomainFallbackRcode(rcode int) bool {
+	switch rcode {
+	case dns.RcodeServerFailure, dns.RcodeNameError, dns.RcodeRefused, dns.RcodeNotImplemented:
+		return true
+	}
+	return false
+}
+
 // internalDomainsEqual reports whether two Internal Domains lists would produce
 // the same routing. Domains are compared in canonical form and independently of
 // the order the API returned them, so an unchanged list does not trigger a
 // reload on every refresh. Resolver order is significant: it is the order the
-// administrator chose, and it decides which resolver is tried first.
+// administrator chose, and it decides which resolver is tried first. The mode
+// is significant too, including a switch between the two explicit modes.
 func internalDomainsEqual(a, b []controld.SplitDNS) bool {
-	canonical := func(entries []controld.SplitDNS) []controld.SplitDNS {
-		out := make([]controld.SplitDNS, 0, len(entries))
+	type route struct {
+		domain     string
+		resolution internalDomainResolution
+		endpoints  []string
+	}
+	canonical := func(entries []controld.SplitDNS) []route {
+		out := make([]route, 0, len(entries))
 		for _, entry := range entries {
 			domain := normalizeInternalDomain(entry.Domain)
 			if domain == "" {
@@ -387,13 +505,13 @@ func internalDomainsEqual(a, b []controld.SplitDNS) bool {
 			}
 			// Entries that cannot be used route nothing, so they cannot make
 			// an otherwise unchanged list look changed.
-			endpoints, reason := internalDomainRouting(entry)
+			resolution, endpoints, reason := internalDomainRouting(entry)
 			if reason != "" {
 				continue
 			}
-			out = append(out, controld.SplitDNS{Domain: domain, Resolvers: endpoints})
+			out = append(out, route{domain: domain, resolution: resolution, endpoints: endpoints})
 		}
-		sort.Slice(out, func(i, j int) bool { return out[i].Domain < out[j].Domain })
+		sort.Slice(out, func(i, j int) bool { return out[i].domain < out[j].domain })
 		return out
 	}
 	x, y := canonical(a), canonical(b)
@@ -401,7 +519,7 @@ func internalDomainsEqual(a, b []controld.SplitDNS) bool {
 		return false
 	}
 	for i := range x {
-		if x[i].Domain != y[i].Domain || !slices.Equal(x[i].Resolvers, y[i].Resolvers) {
+		if x[i].domain != y[i].domain || x[i].resolution != y[i].resolution || !slices.Equal(x[i].endpoints, y[i].endpoints) {
 			return false
 		}
 	}
@@ -429,12 +547,6 @@ func internalDomainFailureReason(err error) string {
 	return "error"
 }
 
-// isGeneratedInternalDomainUpstreamRef is isGeneratedInternalDomainUpstream for
-// call sites that hold an "upstream.<key>" reference rather than the bare key.
-func isGeneratedInternalDomainUpstreamRef(upstream string, uc *ctrld.UpstreamConfig) bool {
-	return isGeneratedInternalDomainUpstream(strings.TrimPrefix(upstream, upstreamPrefix), uc)
-}
-
 // logUpstreamProbeFailure reports a background probe failure against the
 // upstream that the reference upstream names.
 //
@@ -450,10 +562,199 @@ func logUpstreamProbeFailure(upstream string, uc *ctrld.UpstreamConfig, err erro
 	detail := append([]any{}, args...)
 	mainLog.Load().Debug().Err(err).Msgf(format+" for upstream: %q, endpoint: %q", append(detail, uc.Name, uc.Endpoint)...)
 	event := level().Str("upstream", journalUpstreamName(upstream))
-	if !isGeneratedInternalDomainUpstreamRef(upstream, uc) {
+	if !isGeneratedInternalDomainUpstream(uc) {
 		event.Err(err).Msgf(format, detail...)
 		return
 	}
 	event.Str("failure", internalDomainFailureReason(err)).
 		Msgf(format+" for an Internal Domain resolver", detail...)
 }
+
+// betterInternalDomainNegative returns whichever of two negative answers a
+// fallback-mode query should end with: NXDOMAIN over SERVFAIL, since a
+// resolver that says the name does not exist has answered, and otherwise the
+// first one seen.
+func betterInternalDomainNegative(current, candidate *dns.Msg) *dns.Msg {
+	switch {
+	case candidate == nil:
+		return current
+	case current == nil:
+		return candidate
+	case candidate.Rcode == dns.RcodeNameError && current.Rcode != dns.RcodeNameError:
+		return candidate
+	}
+	return current
+}
+
+// internalDomainVPNUpstream returns the upstream the network fallback uses for
+// a VPN DNS server. A seam: VPN DNS servers are dialed on port 53, which a test
+// fixture cannot listen on.
+var internalDomainVPNUpstream = func(m *vpnDNSManager, server string) *ctrld.UpstreamConfig {
+	return m.upstreamConfigFor(server)
+}
+
+// internalDomainOSResolve resolves msg through the OS resolver for the network
+// fallback. A seam, so a test sees the context the fallback passes: that
+// context is what keeps the private name off public nameservers.
+var internalDomainOSResolve = func(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) {
+	resolver, err := ctrld.NewResolver(osUpstreamConfig)
+	if err != nil {
+		return nil, err
+	}
+	resolveCtx, cancel := osUpstreamConfig.Context(ctx)
+	defer cancel()
+	return resolver.Resolve(resolveCtx, msg)
+}
+
+// resolveInternalDomainOnNetwork is the network fallback of an Internal Domain
+// in explicit-with-network-fallback mode. proxy() runs it once the configured
+// resolvers have failed, or answered SERVFAIL or NXDOMAIN.
+//
+// It asks only resolvers that the endpoint's active network or VPN provides,
+// in this order: VPN DNS servers whose domains match the query, then VPN DNS
+// servers with no domains (both only in dns-intercept mode, where ctrld tracks
+// them), then the OS resolver's LAN nameservers. The OS resolver is asked as a
+// LAN-only query, so the private name never reaches a public nameserver, not
+// even one DHCP supplied, nor ctrld's own public fallback. With no LAN
+// nameserver the OS step sends nothing.
+//
+// The first NOERROR answer is returned as answer, including an empty one,
+// which is final. Otherwise negative is the best SERVFAIL or NXDOMAIN answer
+// seen. While Windows is serving retained VPN DNS state, a VPN transport
+// failure stops the fallback before the OS resolver, as it does for VPN split
+// routing.
+func (p *prog) resolveInternalDomainOnNetwork(ctx context.Context, msg *dns.Msg) (answer, negative *dns.Msg) {
+	domain := msg.Question[0].Name
+	resolve := func(ctx context.Context, uc *ctrld.UpstreamConfig) (*dns.Msg, error) {
+		resolver, err := ctrld.NewResolver(uc)
+		if err != nil {
+			return nil, err
+		}
+		resolveCtx, cancel := uc.Context(ctx)
+		defer cancel()
+		return resolver.Resolve(resolveCtx, msg)
+	}
+	// accept reports whether a network answer is final, and keeps a negative
+	// one for the end.
+	accept := func(source string, m *dns.Msg) bool {
+		if !sameQuestion(msg, m) {
+			ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain fallback: discarding answer from %s: question mismatch", source)
+			return false
+		}
+		if m.Rcode == dns.RcodeSuccess {
+			ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain fallback: %s answered %s", source, domain)
+			return true
+		}
+		ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain fallback: %s answered %s for %s",
+			source, dns.RcodeToString[m.Rcode], domain)
+		if internalDomainFallbackRcode(m.Rcode) {
+			negative = betterInternalDomainNegative(negative, m)
+		}
+		return false
+	}
+
+	if dnsIntercept && p.vpnDNS != nil {
+		var servers []string
+		for _, server := range slices.Concat(p.vpnDNS.UpstreamForDomain(domain), p.vpnDNS.DomainlessServers()) {
+			if !slices.Contains(servers, server) {
+				servers = append(servers, server)
+			}
+		}
+		var gotAnswer, gotTransportFailure bool
+		for _, server := range servers {
+			m, err := resolve(ctx, internalDomainVPNUpstream(p.vpnDNS, server))
+			if m == nil {
+				gotTransportFailure = true
+				ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "internal domain fallback: VPN DNS server %s failed", server)
+				continue
+			}
+			gotAnswer = true
+			p.vpnDNS.VPNDNSReachable()
+			if accept("VPN DNS server "+server, m) {
+				return m, nil
+			}
+		}
+		if !gotAnswer && gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, servers) {
+			return nil, negative
+		}
+	}
+
+	m, err := internalDomainOSResolve(ctrld.LanOnlyQueryCtx(ctx), msg)
+	if m == nil {
+		ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "internal domain fallback: OS resolver LAN nameservers failed")
+		return nil, negative
+	}
+	if accept("OS resolver", m) {
+		return m, nil
+	}
+	return nil, negative
+}
+
+// internalDomainProbeInterval is how often, at most, a down fallback-mode
+// Internal Domain resolver is re-checked in the background.
+const internalDomainProbeInterval = 30 * time.Second
+
+// internalDomainProber re-checks down fallback-mode Internal Domain resolvers
+// in the background, and marks one up again when it answers.
+//
+// proxy() skips such a resolver while it is down, so that an endpoint away
+// from the organization network goes to the network fallback without waiting
+// on it. A generated resolver comes back up only by answering a query: it is
+// excluded from recovery, and the loop checker leaves down upstreams alone. So
+// without this re-check, a resolver that went down would stay skipped until
+// the next reload, even once the endpoint is back on the organization network.
+//
+// The zero value is ready to use.
+type internalDomainProber struct {
+	mu   sync.Mutex
+	last map[string]time.Time
+
+	// now and start are seams, so a test controls the clock and runs the
+	// re-check itself. nil means time.Now and a new goroutine.
+	now   func() time.Time
+	start func(func())
+}
+
+// probe re-checks upstream in the background, at most once per
+// internalDomainProbeInterval, by sending it a copy of msg. The query goes to
+// the organization's own resolver, which it was meant for anyway. Any DNS
+// answer proves the resolver reachable, the same as on the query path, and
+// marks it up in um; a failure leaves it down until the next re-check.
+func (pr *internalDomainProber) probe(um *upstreamMonitor, upstream string, uc *ctrld.UpstreamConfig, msg *dns.Msg) {
+	now := time.Now
+	if pr.now != nil {
+		now = pr.now
+	}
+	pr.mu.Lock()
+	if last, ok := pr.last[upstream]; ok && now().Sub(last) < internalDomainProbeInterval {
+		pr.mu.Unlock()
+		return
+	}
+	if pr.last == nil {
+		pr.last = make(map[string]time.Time)
+	}
+	pr.last[upstream] = now()
+	pr.mu.Unlock()
+
+	query := msg.Copy()
+	run := func() {
+		var answer *dns.Msg
+		resolver, err := ctrld.NewResolver(uc)
+		if err == nil {
+			ctx, cancel := uc.Context(context.Background())
+			answer, err = resolver.Resolve(ctx, query)
+			cancel()
+		}
+		if answer == nil {
+			mainLog.Load().Debug().Err(err).Msgf("internal domains: %s is still down", upstream)
+			return
+		}
+		mainLog.Load().Debug().Msgf("internal domains: %s answered the re-check", upstream)
+		um.noteSuccess(upstream)
+	}
+	if pr.start != nil {
+		pr.start(run)
+		return
+	}
+	go run()
+}

cmd/cli/internal_domains_fallback_test.go

@@ -0,0 +1,582 @@
+package cli
+
+import (
+	"context"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/miekg/dns"
+
+	"github.com/Control-D-Inc/ctrld"
+	"github.com/Control-D-Inc/ctrld/internal/controld"
+)
+
+// startRcodeDNSFixture is startDNSFixture answering every question with rcode.
+// A NOERROR answer carries an A record unless empty is set, which gives the
+// name-exists-but-no-record-of-that-type answer.
+func startRcodeDNSFixture(t *testing.T, rcode int, empty bool) *dnsFixture {
+	t.Helper()
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	f := &dnsFixture{addr: pc.LocalAddr().String()}
+	started := make(chan struct{})
+	server := &dns.Server{
+		PacketConn:        pc,
+		NotifyStartedFunc: func() { close(started) },
+		Handler: dns.HandlerFunc(func(w dns.ResponseWriter, msg *dns.Msg) {
+			f.mu.Lock()
+			if len(msg.Question) > 0 {
+				f.qs = append(f.qs, msg.Question[0].Name)
+			}
+			f.mu.Unlock()
+			reply := new(dns.Msg)
+			reply.SetRcode(msg, rcode)
+			if rcode == dns.RcodeSuccess && !empty && len(msg.Question) > 0 {
+				rr, _ := dns.NewRR(msg.Question[0].Name + " 60 IN A 10.9.9.9")
+				reply.Answer = append(reply.Answer, rr)
+			}
+			_ = w.WriteMsg(reply)
+		}),
+	}
+	go func() { _ = server.ActivateAndServe() }()
+	select {
+	case <-started:
+	case <-time.After(5 * time.Second):
+		t.Fatal("DNS fixture did not start")
+	}
+	t.Cleanup(func() { _ = server.Shutdown() })
+	return f
+}
+
+func deadLoopbackAddr(t *testing.T) string {
+	t.Helper()
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	addr := pc.LocalAddr().String()
+	_ = pc.Close()
+	return addr
+}
+
+// Explicit resolver with network fallback is the default explicit mode: the
+// configured resolver is asked first, and a timeout, an unreachable resolver,
+// SERVFAIL or NXDOMAIN hands the query to the network's resolver. A valid
+// empty answer is final, and "explicit resolver only" never falls back.
+func TestInternalDomainsNetworkFallback(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		mode string
+		// internal is the configured resolver's rcode; -1 is unreachable.
+		internal      int
+		internalEmpty bool
+		network       int
+		wantNetwork   bool
+		wantRcode     int
+		wantAnswer    bool
+	}{
+		{name: "nxdomain falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "servfail falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeServerFailure, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "unreachable falls back", mode: controld.SplitDNSModeResolvers, internal: -1, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "absent mode falls back", mode: "", internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "answer is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeSuccess, network: dns.RcodeSuccess, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "empty answer is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeSuccess, internalEmpty: true, network: dns.RcodeSuccess, wantRcode: dns.RcodeSuccess},
+		{name: "refused falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeRefused, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "notimp falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNotImplemented, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true},
+		{name: "refused and network nxdomain", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeRefused, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError},
+		{name: "formerr is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeFormatError, network: dns.RcodeSuccess, wantRcode: dns.RcodeFormatError},
+		{name: "only mode keeps refused", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeRefused, network: dns.RcodeSuccess, wantRcode: dns.RcodeRefused},
+		{name: "unknown mode fails closed", mode: "explicit_only", internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantRcode: dns.RcodeNameError},
+		{name: "nobody knows the name", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError},
+		{name: "configured nxdomain beats network servfail", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeServerFailure, wantNetwork: true, wantRcode: dns.RcodeNameError},
+		{name: "unreachable and network nxdomain", mode: controld.SplitDNSModeResolvers, internal: -1, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError},
+		{name: "only mode keeps nxdomain", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantRcode: dns.RcodeNameError},
+		{name: "only mode keeps servfail", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeServerFailure, network: dns.RcodeSuccess, wantRcode: dns.RcodeServerFailure},
+		{name: "only mode fails when unreachable", mode: controld.SplitDNSModeResolversOnly, internal: -1, network: dns.RcodeSuccess, wantRcode: dns.RcodeServerFailure},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			network := startRcodeDNSFixture(t, tc.network, false)
+			useOSResolverFixture(t, network)
+
+			var internal *dnsFixture
+			resolver := deadLoopbackAddr(t)
+			if tc.internal >= 0 {
+				internal = startRcodeDNSFixture(t, tc.internal, tc.internalEmpty)
+				resolver = internal.addr
+			}
+
+			cfg := internalDomainsTestConfig()
+			cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true)
+			applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: tc.mode, Resolvers: []string{resolver}}})
+			p := newInternalDomainsProg(t, cfg)
+
+			addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+			if err != nil {
+				t.Fatal(err)
+			}
+			ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+			ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+			res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+
+			if internal != nil {
+				if got := internal.questions(); len(got) != 1 {
+					t.Errorf("configured resolver questions = %v, want it asked first", got)
+				}
+			}
+			if got := network.questions(); (len(got) > 0) != tc.wantNetwork {
+				t.Errorf("network resolver questions = %v, want asked = %v", got, tc.wantNetwork)
+			}
+			if res == nil || res.answer == nil {
+				t.Fatalf("no answer")
+			}
+			if res.answer.Rcode != tc.wantRcode {
+				t.Errorf("rcode = %s, want %s", dns.RcodeToString[res.answer.Rcode], dns.RcodeToString[tc.wantRcode])
+			}
+			if got := len(res.answer.Answer) > 0; got != tc.wantAnswer {
+				t.Errorf("answer records = %v, want records = %v", res.answer.Answer, tc.wantAnswer)
+			}
+		})
+	}
+}
+
+// The configured resolvers are all tried, in order, before the network: an
+// NXDOMAIN from the first does not skip the second.
+func TestInternalDomainsNetworkFallbackTriesEveryConfiguredResolverFirst(t *testing.T) {
+	network := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+	useOSResolverFixture(t, network)
+	first := startRcodeDNSFixture(t, dns.RcodeNameError, false)
+	second := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+
+	cfg := internalDomainsTestConfig()
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{first.addr, second.addr}}})
+	p := newInternalDomainsProg(t, cfg)
+
+	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+	ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+	res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+
+	if len(first.questions()) != 1 || len(second.questions()) != 1 {
+		t.Errorf("configured resolvers asked %v, %v; want both once", first.questions(), second.questions())
+	}
+	if got := network.questions(); len(got) != 0 {
+		t.Errorf("network resolver asked %v although a configured resolver answered", got)
+	}
+	if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+		t.Errorf("answer = %v, want the second resolver's NOERROR", res)
+	}
+}
+
+// The fallback is no reason to log the private resolver address above debug,
+// or the domain: the summary-only REPLY line carries neither.
+func TestInternalDomainsNetworkFallbackLogsNoPrivateValues(t *testing.T) {
+	network := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+	useOSResolverFixture(t, network)
+	internal := startRcodeDNSFixture(t, dns.RcodeNameError, false)
+
+	cfg := internalDomainsTestConfig()
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}})
+	p := newInternalDomainsProg(t, cfg)
+
+	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+	ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+	before := len(logOutput.String())
+	p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+	out := logOutput.String()[before:]
+
+	if !strings.Contains(out, "internal domain network fallback") {
+		t.Fatalf("expected the fallback REPLY line, got: %s", out)
+	}
+	for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
+		if line == "" || strings.Contains(line, `"level":"debug"`) {
+			continue
+		}
+		if strings.Contains(line, internal.addr) || strings.Contains(line, "corp.example") {
+			t.Errorf("private value disclosed above debug level: %s", line)
+		}
+	}
+}
+
+// internalDomainUpstreamsProg returns a prog whose config has one legacy
+// upstream per key, each carrying the given InternalDomain marker.
+func internalDomainUpstreamsProg(markers map[string]string) *prog {
+	cfg := &ctrld.Config{Upstream: make(map[string]*ctrld.UpstreamConfig, len(markers))}
+	for key, marker := range markers {
+		cfg.Upstream[key] = &ctrld.UpstreamConfig{
+			Name:           internalDomainUpstreamName,
+			Type:           ctrld.ResolverTypeLegacy,
+			Endpoint:       "10.0.0.53:53",
+			InternalDomain: marker,
+		}
+	}
+	return &prog{cfg: cfg}
+}
+
+func TestInternalDomainFallbackUpstreams(t *testing.T) {
+	p := internalDomainUpstreamsProg(map[string]string{
+		"internal_0": internalDomainUpstreamFallback,
+		"internal_1": internalDomainUpstreamFallback,
+		"internal_2": internalDomainUpstreamOnly,
+		"0":          "",
+	})
+	for _, tc := range []struct {
+		in   []string
+		want bool
+	}{
+		{in: nil, want: false},
+		{in: []string{"upstream.internal_0", "upstream.internal_1"}, want: true},
+		{in: []string{"upstream.internal_2"}, want: false},
+		{in: []string{"upstream.internal_0", "upstream.internal_2"}, want: false},
+		{in: []string{"upstream.internal_0", "upstream.0"}, want: false},
+		{in: []string{upstreamOS}, want: false},
+	} {
+		if got := p.internalDomainFallbackUpstreams(tc.in); got != tc.want {
+			t.Errorf("internalDomainFallbackUpstreams(%v) = %v, want %v", tc.in, got, tc.want)
+		}
+	}
+	if !p.internalDomainExplicitUpstreams([]string{"upstream.internal_2"}) {
+		t.Error("an explicit-only upstream must still count as an Internal Domain resolver")
+	}
+}
+
+// Both explicit modes generate internal_<n> upstreams; the mode is carried by
+// the InternalDomain marker. Switching between the modes is a routing change
+// the next refresh must apply.
+func TestInternalDomainsExplicitModes(t *testing.T) {
+	cfg := internalDomainsTestConfig()
+	summary := applyInternalDomains(cfg, []controld.SplitDNS{
+		{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}},
+		{Domain: "b.example.com", Mode: " Resolvers_Only ", Resolvers: []string{"10.0.0.54"}},
+	})
+	if summary.explicit != 2 || summary.only != 1 || summary.resolvers != 2 {
+		t.Fatalf("summary = %+v", summary)
+	}
+	for _, tc := range []struct{ domain, target, marker string }{
+		{"a.example.com", "upstream.internal_0", internalDomainUpstreamFallback},
+		{"b.example.com", "upstream.internal_1", internalDomainUpstreamOnly},
+	} {
+		if targets, _ := ruleTargets(t, cfg, tc.domain); strings.Join(targets, ",") != tc.target {
+			t.Errorf("%s targets = %v, want %s", tc.domain, targets, tc.target)
+		}
+		uc := cfg.Upstream[strings.TrimPrefix(tc.target, upstreamPrefix)]
+		if uc == nil || uc.InternalDomain != tc.marker {
+			t.Errorf("%s upstream = %+v, want marker %q", tc.domain, uc, tc.marker)
+		}
+	}
+
+	fallback := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}}
+	only := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{"10.0.0.53"}}}
+	legacy := []controld.SplitDNS{{Domain: "a.example.com", Resolvers: []string{"10.0.0.53"}}}
+	if internalDomainsEqual(fallback, only) {
+		t.Error("switching to explicit resolver only must be detected as a change")
+	}
+	if !internalDomainsEqual(fallback, legacy) {
+		t.Error("an absent mode with resolvers is the default explicit mode")
+	}
+}
+
+// An upstream a configuration defines is an ordinary upstream, whatever it is
+// named and however closely it copies a generated one: a local ctrld.toml that
+// names an upstream internal_<n>, gives it the legacy type and the generated
+// name, must still get the OS-resolver catch-all when it fails.
+func TestInternalDomainsUserDefinedLookalikeIsOrdinaryUpstream(t *testing.T) {
+	osFixture := startDNSFixture(t)
+	useOSResolverFixture(t, osFixture)
+
+	cfg := internalDomainsTestConfig()
+	cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true)
+	cfg.Upstream["internal_0"] = &ctrld.UpstreamConfig{
+		Name:     internalDomainUpstreamName,
+		Type:     ctrld.ResolverTypeLegacy,
+		Endpoint: deadLoopbackAddr(t),
+		Timeout:  500,
+	}
+	cfg.Listener["0"].Policy.Rules = append(cfg.Listener["0"].Policy.Rules,
+		ctrld.Rule{"*.corp.example": []string{"upstream.internal_0"}})
+	p := newInternalDomainsProg(t, cfg)
+	if p.isInternalDomainUpstream("upstream.internal_0") || isGeneratedInternalDomainUpstream(cfg.Upstream["internal_0"]) {
+		t.Fatal("a user-defined upstream was recognized as a generated Internal Domain resolver")
+	}
+
+	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+	ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+	res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+
+	if got := osFixture.questions(); len(got) != 1 {
+		t.Errorf("OS resolver questions = %v, want the ordinary catch all", got)
+	}
+	if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+		t.Errorf("answer = %v, want the OS resolver's NOERROR", res)
+	}
+}
+
+// captureInternalDomainProbes makes p's background re-checks run only when the
+// test calls them, and returns the ones scheduled so far.
+func captureInternalDomainProbes(p *prog) *[]func() {
+	var probes []func()
+	p.internalDomainProbes.start = func(run func()) { probes = append(probes, run) }
+	return &probes
+}
+
+// The off-site case that fallback mode exists for: the organization resolver
+// is unreachable and the network answers. Once the monitor reports the
+// resolver down, a query must go to the network without waiting on the
+// resolver's timeout, and a background re-check must bring it back once it
+// answers again.
+func TestInternalDomainsDownResolverIsSkippedAndReChecked(t *testing.T) {
+	network := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+	useOSResolverFixture(t, network)
+	blackhole := blackholeUDPAddr(t)
+
+	cfg := internalDomainsTestConfig()
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{blackhole}}})
+	p := newInternalDomainsProg(t, cfg)
+	probes := captureInternalDomainProbes(p)
+
+	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ask := func() (*proxyResponse, time.Duration) {
+		t.Helper()
+		ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+		ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+		start := time.Now()
+		res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+		return res, time.Since(start)
+	}
+
+	const upstream = upstreamPrefix + internalDomainUpstreamPrefix + "0"
+	p.um.mu.Lock()
+	p.um.markDown(upstream, maxFailureRequest, "immediate")
+	p.um.mu.Unlock()
+
+	for i := 0; i < 2; i++ {
+		res, elapsed := ask()
+		if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+			t.Fatalf("query %d: answer = %v, want the network's NOERROR", i+1, res)
+		}
+		// The blackholed resolver would cost its 2s read timeout.
+		if elapsed > time.Second {
+			t.Fatalf("query %d took %v: it waited on a resolver that is down", i+1, elapsed)
+		}
+	}
+	if got := len(*probes); got != 1 {
+		t.Fatalf("re-checks scheduled = %d, want one per interval", got)
+	}
+
+	// The resolver is still unreachable: the re-check leaves it down.
+	(*probes)[0]()
+	if !p.um.isDown(upstream) {
+		t.Fatal("a failed re-check marked the resolver up")
+	}
+
+	// Back on the organization network: the next re-check, once the interval
+	// has passed, finds it answering and marks it up, so queries use it again.
+	internal := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+	cfg.Upstream["internal_0"].Endpoint = internal.addr
+	later := time.Now().Add(internalDomainProbeInterval)
+	p.internalDomainProbes.now = func() time.Time { return later }
+	ask()
+	if got := len(*probes); got != 2 {
+		t.Fatalf("re-checks scheduled = %d, want a second one after the interval", got)
+	}
+	(*probes)[1]()
+	if p.um.isDown(upstream) {
+		t.Fatal("the resolver answered the re-check but is still down")
+	}
+	before := len(network.questions())
+	if res, _ := ask(); res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+		t.Fatalf("answer = %v, want the resolver's NOERROR", res)
+	}
+	if got := internal.questions(); len(got) != 2 {
+		t.Errorf("resolver questions = %v, want the re-check and the query", got)
+	}
+	if got := len(network.questions()); got != before {
+		t.Errorf("network asked again although the resolver answered")
+	}
+}
+
+// Explicit resolver only has no fallback to skip to, so a down resolver is
+// still asked: skipping it would only turn its answer into a SERVFAIL.
+func TestInternalDomainsOnlyModeDoesNotSkipDownResolver(t *testing.T) {
+	useOSResolverFixture(t, startDNSFixture(t))
+	internal := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+
+	cfg := internalDomainsTestConfig()
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{internal.addr}}})
+	p := newInternalDomainsProg(t, cfg)
+	probes := captureInternalDomainProbes(p)
+
+	p.um.mu.Lock()
+	p.um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"0", maxFailureRequest, "immediate")
+	p.um.mu.Unlock()
+
+	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+	ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+	res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+
+	if got := internal.questions(); len(got) != 1 {
+		t.Errorf("resolver questions = %v, want it asked although it is down", got)
+	}
+	if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+		t.Errorf("answer = %v, want the resolver's NOERROR", res)
+	}
+	if len(*probes) != 0 {
+		t.Errorf("explicit resolver only scheduled %d re-checks", len(*probes))
+	}
+}
+
+// fallbackTestQuery sends one query for host.corp.example through proxy().
+func fallbackTestQuery(t *testing.T, p *prog, cfg *ctrld.Config) *proxyResponse {
+	t.Helper()
+	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
+	ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example")
+	return p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr})
+}
+
+// The OS step of the fallback is what keeps the private name off public
+// nameservers, and it does so through the context it passes. A test that
+// swaps the OS resolver for a single-endpoint fixture would pass with any
+// context, so this one sees the context itself and runs the step through a
+// real OS resolver whose pool holds a LAN and a public nameserver.
+func TestInternalDomainsNetworkFallbackOSStepIsLanOnly(t *testing.T) {
+	lan := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+	pool := ctrld.NewResolverWithNameserver([]string{lan.addr, "192.0.2.53:53"})
+	var lanOnly []bool
+	prev := internalDomainOSResolve
+	internalDomainOSResolve = func(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) {
+		isLanOnly, _ := ctx.Value(ctrld.LanOnlyQueryCtxKey{}).(bool)
+		lanOnly = append(lanOnly, isLanOnly)
+		ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
+		defer cancel()
+		return pool.Resolve(ctx, msg)
+	}
+	t.Cleanup(func() { internalDomainOSResolve = prev })
+
+	internal := startRcodeDNSFixture(t, dns.RcodeNameError, false)
+	cfg := internalDomainsTestConfig()
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}})
+	p := newInternalDomainsProg(t, cfg)
+
+	res := fallbackTestQuery(t, p, cfg)
+	if len(lanOnly) != 1 || !lanOnly[0] {
+		t.Fatalf("OS step contexts LAN-only = %v, want one LAN-only query", lanOnly)
+	}
+	if got := lan.questions(); len(got) != 1 {
+		t.Errorf("LAN nameserver questions = %v, want the fallback query", got)
+	}
+	if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+		t.Errorf("answer = %v, want the LAN nameserver's NOERROR", res)
+	}
+}
+
+// useVPNFallbackFixtures routes the fallback's VPN DNS servers to fixtures:
+// real VPN DNS servers are dialed on port 53.
+func useVPNFallbackFixtures(t *testing.T, servers map[string]string) {
+	t.Helper()
+	prev := internalDomainVPNUpstream
+	internalDomainVPNUpstream = func(_ *vpnDNSManager, server string) *ctrld.UpstreamConfig {
+		return &ctrld.UpstreamConfig{Name: "VPN DNS", Type: ctrld.ResolverTypeLegacy, Endpoint: servers[server], Timeout: 1000}
+	}
+	t.Cleanup(func() { internalDomainVPNUpstream = prev })
+}
+
+// The fallback asks VPN DNS servers that match the domain, then domain-less
+// ones, then the OS resolver, and stops at the first NOERROR.
+func TestInternalDomainsNetworkFallbackVPNOrder(t *testing.T) {
+	for _, tc := range []struct {
+		name                     string
+		matched, domainless      int
+		wantMatched, wantDomless int
+		wantOS                   bool
+	}{
+		{name: "matching VPN server answers", matched: dns.RcodeSuccess, domainless: dns.RcodeSuccess, wantMatched: 1},
+		{name: "then the domain-less one", matched: dns.RcodeNameError, domainless: dns.RcodeSuccess, wantMatched: 1, wantDomless: 1},
+		{name: "then the OS resolver", matched: dns.RcodeNameError, domainless: dns.RcodeRefused, wantMatched: 1, wantDomless: 1, wantOS: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			useDNSIntercept(t)
+			osFixture := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+			useOSResolverFixture(t, osFixture)
+			matched := startRcodeDNSFixture(t, tc.matched, false)
+			domainless := startRcodeDNSFixture(t, tc.domainless, false)
+			useVPNFallbackFixtures(t, map[string]string{"10.8.0.1": matched.addr, "10.8.0.2": domainless.addr})
+
+			internal := startRcodeDNSFixture(t, dns.RcodeNameError, false)
+			cfg := internalDomainsTestConfig()
+			applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}})
+			p := newInternalDomainsProg(t, cfg)
+			p.vpnDNS = &vpnDNSManager{
+				routes:            map[string][]string{"corp.example": {"10.8.0.1"}},
+				domainlessServers: []string{"10.8.0.2"},
+			}
+
+			res := fallbackTestQuery(t, p, cfg)
+			if got := len(matched.questions()); got != tc.wantMatched {
+				t.Errorf("matching VPN server asked %d times, want %d", got, tc.wantMatched)
+			}
+			if got := len(domainless.questions()); got != tc.wantDomless {
+				t.Errorf("domain-less VPN server asked %d times, want %d", got, tc.wantDomless)
+			}
+			if got := len(osFixture.questions()) > 0; got != tc.wantOS {
+				t.Errorf("OS resolver asked = %v, want %v", got, tc.wantOS)
+			}
+			if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess {
+				t.Errorf("answer = %v, want NOERROR", res)
+			}
+		})
+	}
+}
+
+// While Windows is serving retained VPN DNS state, a VPN transport failure
+// stops the fallback before the OS resolver, as it does for VPN split routing.
+func TestInternalDomainsNetworkFallbackStopsOnRetainedVPNTransportFailure(t *testing.T) {
+	useDNSIntercept(t)
+	prevSettling := vpnDNSSettlingEnabled
+	vpnDNSSettlingEnabled = true
+	t.Cleanup(func() { vpnDNSSettlingEnabled = prevSettling })
+	osFixture := startRcodeDNSFixture(t, dns.RcodeSuccess, false)
+	useOSResolverFixture(t, osFixture)
+	useVPNFallbackFixtures(t, map[string]string{"10.8.0.1": deadLoopbackAddr(t)})
+
+	internal := startRcodeDNSFixture(t, dns.RcodeNameError, false)
+	cfg := internalDomainsTestConfig()
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}})
+	p := newInternalDomainsProg(t, cfg)
+	p.vpnDNS = &vpnDNSManager{
+		routes:                      map[string][]string{"corp.example": {"10.8.0.1"}},
+		retainedAfterEmptyDiscovery: true,
+	}
+
+	res := fallbackTestQuery(t, p, cfg)
+	if got := osFixture.questions(); len(got) != 0 {
+		t.Errorf("OS resolver asked %v while retained VPN DNS state is active", got)
+	}
+	if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeNameError {
+		t.Errorf("answer = %v, want the configured resolver's NXDOMAIN", res)
+	}
+}

cmd/cli/internal_domains_review_test.go

@@ -139,9 +139,10 @@ func TestInternalDomainsExplicitResolverSurvivesRecoveryBypass(t *testing.T) {
 	}
 }
 
-// When the configured resolvers are unreachable the query must fail rather than
-// reach the OS resolver, and the failure must be observable on the wire as an
-// attempt against the configured address.
+// In "explicit resolver only" mode, when the configured resolvers are
+// unreachable the query must fail rather than reach the OS resolver, and the
+// failure must be observable on the wire as an attempt against the configured
+// address.
 func TestInternalDomainsUnreachableResolverDoesNotReachOSResolver(t *testing.T) {
 	osFixture := startDNSFixture(t)
 	useOSResolverFixture(t, osFixture)
@@ -157,7 +158,7 @@ func TestInternalDomainsUnreachableResolverDoesNotReachOSResolver(t *testing.T)
 
 	cfg := internalDomainsTestConfig()
 	cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true)
-	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{dead}}})
+	applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{dead}}})
 	p := newInternalDomainsProg(t, cfg)
 
 	addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0")
@@ -301,6 +302,10 @@ func TestInternalDomainsSetupKeepsResolverAddressesOutOfInfoLogs(t *testing.T) {
 // Domain upstream that endpoint is private, so only a classification may appear
 // at error level.
 func TestInternalDomainsResolverFailureLogsNoAddress(t *testing.T) {
+	// The default mode falls back to the network after the failure; keep that
+	// on loopback.
+	useOSResolverFixture(t, startDNSFixture(t))
+
 	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
 	if err != nil {
 		t.Fatal(err)

cmd/cli/internal_domains_test.go

@@ -403,6 +403,11 @@ func TestInternalDomainsEqual(t *testing.T) {
 }
 
 func TestInternalDomainExplicitUpstreams(t *testing.T) {
+	p := internalDomainUpstreamsProg(map[string]string{
+		"internal_0": internalDomainUpstreamFallback,
+		"internal_1": internalDomainUpstreamOnly,
+		"0":          "",
+	})
 	for _, tc := range []struct {
 		in   []string
 		want bool
@@ -415,7 +420,7 @@ func TestInternalDomainExplicitUpstreams(t *testing.T) {
 		{nil, false},
 		{[]string{}, false},
 	} {
-		if got := internalDomainExplicitUpstreams(tc.in); got != tc.want {
+		if got := p.internalDomainExplicitUpstreams(tc.in); got != tc.want {
 			t.Errorf("internalDomainExplicitUpstreams(%v) = %v, want %v", tc.in, got, tc.want)
 		}
 	}
@@ -448,13 +453,13 @@ func TestInternalDomainsSummaryLogsCountsOnly(t *testing.T) {
 
 // An explicit-resolver Internal Domain must not be handed to VPN DNS split
 // routing: the administrator named those resolvers, and VPN suffixes are
-// auto-detected. When the named resolvers are unreachable the query fails
-// instead of reaching the OS resolver.
+// auto-detected. In "explicit resolver only" mode, when the named resolvers
+// are unreachable the query fails instead of reaching the OS resolver.
 func TestInternalDomainsBeatVPNSplitRoutingAndDoNotLeak(t *testing.T) {
 	cfg := internalDomainsTestConfig()
 	cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true)
 	applyInternalDomains(cfg, []controld.SplitDNS{
-		{Domain: "aws.example.com", Resolvers: []string{"127.0.0.1:1"}},
+		{Domain: "aws.example.com", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{"127.0.0.1:1"}},
 	})
 
 	p := &prog{cfg: cfg}
@@ -473,7 +478,7 @@ func TestInternalDomainsBeatVPNSplitRoutingAndDoNotLeak(t *testing.T) {
 	}
 	ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID())
 	ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.aws.example.com")
-	if !internalDomainExplicitUpstreams(ufr.upstreams) {
+	if !p.internalDomainExplicitUpstreams(ufr.upstreams) {
 		t.Fatalf("upstreams = %v, want only internal resolvers", ufr.upstreams)
 	}
 
@@ -571,9 +576,18 @@ func TestApplyInternalDomainsModeIsAuthoritative(t *testing.T) {
 			wantUpstreams: 1,
 		},
 		{
-			// A mode ctrld does not know is not guessed at.
-			name:          "unknown mode is dropped",
+			// A wire value this build does not know fails closed to the
+			// resolvers the administrator chose, never to the Control D upstream.
+			name:          "unknown mode with resolvers fails closed",
 			entry:         controld.SplitDNS{Domain: "a.example.com", Mode: "forward-someday", Resolvers: []string{"10.0.0.53"}},
+			wantTargets:   []string{"upstream.internal_0"},
+			wantRule:      true,
+			wantUpstreams: 2,
+		},
+		{
+			// With no resolvers there is nothing to fail closed to.
+			name:          "unknown mode without resolvers is dropped",
+			entry:         controld.SplitDNS{Domain: "a.example.com", Mode: "forward-someday"},
 			wantRule:      false,
 			wantUpstreams: 1,
 		},

cmd/cli/prog.go

@@ -140,6 +140,7 @@ type prog struct {
 	internalLogWriter         *logWriter
 	internalJournalWriter     *logWriter
 	querySampler              errorSampler
+	internalDomainProbes      internalDomainProber
 	lastNetworkState          atomic.Pointer[netmon.State]
 	internalLogSent           time.Time
 	runningIface              string
@@ -859,7 +860,7 @@ func (p *prog) setupUpstream(cfg *ctrld.Config) {
 	// gated on the API confirming the device is gone.
 	managedUpstreams := 0
 	for n := range cfg.Upstream {
-		if isGeneratedInternalDomainUpstream(n, cfg.Upstream[n]) {
+		if isGeneratedInternalDomainUpstream(cfg.Upstream[n]) {
 			continue
 		}
 		managedUpstreams++
@@ -1161,7 +1162,7 @@ func (p *prog) startNetworkJournal() func() {
 		close(stop)
 	}()
 	healthDone := p.health.startLoop(stop, func() (int, bool) {
-		return p.upstreamMonitorNow().countDownExcept(isInternalDomainUpstream), p.recoveryBypass.Load()
+		return p.upstreamMonitorNow().countDownExcept(p.isInternalDomainUpstream), p.recoveryBypass.Load()
 	})
 	var configDone <-chan struct{}
 	if p.dnsConfig != nil {

cmd/cli/query_health_prog_test.go

@@ -107,10 +107,11 @@ func Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade(t *testing.T) {
 	cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false)
 	cfg.Upstream = map[string]*ctrld.UpstreamConfig{
 		internalDomainUpstreamPrefix + "1": {
-			Name:     "dead internal resolver",
-			Type:     ctrld.ResolverTypeLegacy,
-			Endpoint: deadUpstreamEndpoint,
-			Timeout:  500,
+			Name:           "dead internal resolver",
+			Type:           ctrld.ResolverTypeLegacy,
+			Endpoint:       deadUpstreamEndpoint,
+			Timeout:        500,
+			InternalDomain: internalDomainUpstreamOnly,
 		},
 	}
 	p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg), health: newQueryHealth()}
@@ -140,7 +141,7 @@ func Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade(t *testing.T) {
 	p.um.markDown(internalUpstream, 3, "immediate")
 	p.um.mu.Unlock()
 
-	event, emit := p.health.evaluate(time.Now(), p.um.countDownExcept(isInternalDomainUpstream), false)
+	event, emit := p.health.evaluate(time.Now(), p.um.countDownExcept(p.isInternalDomainUpstream), false)
 	if !emit {
 		t.Fatal("the first evaluate did not emit")
 	}

cmd/cli/recovery_scope_test.go

@@ -303,7 +303,9 @@ func TestScopeHarnessRestoresRecoveryWait(t *testing.T) {
 
 func TestOSRecoveryCandidates(t *testing.T) {
 	remote := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeDOH}
-	internal := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy}
+	internal := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy, InternalDomain: internalDomainUpstreamFallback}
+	// A user-defined upstream that copies a generated key carries no marker.
+	lookalike := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy}
 	osConfig := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeOS}
 	for _, tc := range []struct {
 		name       string
@@ -311,9 +313,10 @@ func TestOSRecoveryCandidates(t *testing.T) {
 		want       map[string]*ctrld.UpstreamConfig
 	}{
 		{"configured", map[string]*ctrld.UpstreamConfig{"0": remote, "internal_0": internal, "1": osConfig, "2": nil}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote}},
-		{"custom_policy_or_unused", map[string]*ctrld.UpstreamConfig{"0": remote, "unused": internal}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote, "upstream.unused": internal}},
+		{"custom_policy_or_unused", map[string]*ctrld.UpstreamConfig{"0": remote, "unused": lookalike}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote, "upstream.unused": lookalike}},
 		{"only_os", map[string]*ctrld.UpstreamConfig{"0": osConfig}, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}},
 		{"only_internal", map[string]*ctrld.UpstreamConfig{"internal_0": internal}, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}},
+		{"internal_lookalike", map[string]*ctrld.UpstreamConfig{"internal_0": lookalike}, map[string]*ctrld.UpstreamConfig{"upstream.internal_0": lookalike}},
 		{"empty", nil, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}},
 	} {
 		t.Run(tc.name, func(t *testing.T) {

cmd/cli/upstream_monitor_test.go

@@ -293,7 +293,12 @@ func Test_upstreamMonitorCountDownExcept(t *testing.T) {
 	if got := um.countDown(); got != 3 {
 		t.Fatalf("countDown = %d, want 3", got)
 	}
-	if got := um.countDownExcept(isInternalDomainUpstream); got != 1 {
+	p := internalDomainUpstreamsProg(map[string]string{
+		"0":                                "",
+		internalDomainUpstreamPrefix + "1": internalDomainUpstreamFallback,
+		internalDomainUpstreamPrefix + "2": internalDomainUpstreamOnly,
+	})
+	if got := um.countDownExcept(p.isInternalDomainUpstream); got != 1 {
 		t.Fatalf("countDownExcept(isInternalDomainUpstream) = %d, want 1", got)
 	}
 	if got := um.countDownExcept(nil); got != 3 {