diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 1f5374f..4183518 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -1058,6 +1058,7 @@ func logInternalDomainsSummary(logger zerolog.Logger, summary internalDomainsSum event.Int("domains", summary.domains). Int("os_resolver", summary.osMode). Int("explicit_resolver", summary.explicit). + Int("explicit_resolver_only", summary.only). Int("resolvers", summary.resolvers). Int("skipped", summary.skipped). Int("preempted", summary.preempted). diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index 87e0b1b..50a1dd6 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -577,11 +577,13 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { // // An Internal Domain with explicit resolvers does not take part. The bypass // exists because general DNS is broken, which says nothing about the - // administrator's own resolvers: they may well be reachable, and sending a - // private name to the network's DNS is the disclosure the explicit - // selection exists to prevent. Such a query continues to the normal flow, - // where it reaches only its configured resolvers and fails if none answer. - if dnsIntercept && p.recoveryBypass.Load() && !internalDomainExplicitUpstreams(req.ufr.upstreams) { + // administrator's own resolvers: they may well be reachable, and they are + // to be tried first. Such a query continues to the normal flow, where it + // reaches its configured resolvers first. In "explicit resolver only" mode + // it fails if none answer; otherwise it falls back to the VPN resolvers and + // the network's LAN nameservers, never to the public nameservers the bypass + // may use. + if dnsIntercept && p.recoveryBypass.Load() && !p.internalDomainExplicitUpstreams(req.ufr.upstreams) { ctrld.Log(ctx, mainLog.Load().Debug(), "Recovery bypass active: forwarding to OS resolver") resolver, err := ctrld.NewResolver(osUpstreamConfig) if err == nil { @@ -713,13 +715,15 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { // // An Internal Domain with explicit resolvers is skipped here: the // administrator named the resolvers for that suffix, and VPN suffixes - // are auto-detected, so handing the query to a VPN DNS server would - // override an explicit selection with a discovered one. + // are auto-detected, so handing the query to a VPN DNS server first would + // override an explicit selection with a discovered one. In fallback mode + // the VPN DNS servers are asked later, once the configured resolvers have + // not resolved the name. // // Internal Domains in OS-resolver mode are not skipped, they ask for the // endpoint's default resolution, which under intercept mode includes // the VPN's own resolver. - if dnsIntercept && p.vpnDNS != nil && len(req.msg.Question) > 0 && !internalDomainExplicitUpstreams(upstreams) { + if dnsIntercept && p.vpnDNS != nil && len(req.msg.Question) > 0 && !p.internalDomainExplicitUpstreams(upstreams) { domain := req.msg.Question[0].Name if vpnServers := p.vpnDNS.UpstreamForDomain(domain); len(vpnServers) > 0 { ctrld.Log(ctx, mainLog.Load().Debug(), "VPN DNS route matched for domain %s, using servers: %v", domain, vpnServers) @@ -871,7 +875,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { // generated Internal Domain upstream is the organization's private // resolver address. Report the classification through the sampler and // keep the address-bearing error at debug. - if isInternalDomainUpstream(upstream) { + if isGeneratedInternalDomainUpstream(upstreamConfig) { ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "failed to resolve query") ctrld.Log(ctx, p.querySampler.event(sampleClassInternalDomain, upstream).Str("failure", internalDomainFailureReason(err)), "failed to resolve query using an Internal Domain resolver") @@ -897,6 +901,12 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { return nil } + // An Internal Domain in explicit-with-network-fallback mode moves on from a + // configured resolver that answers SERVFAIL or NXDOMAIN, as it does from + // one that does not answer. internalNegative keeps the best of those + // answers, so a name nobody resolves gets a real negative answer. + internalFallback := p.internalDomainFallbackUpstreams(upstreams) + var internalNegative *dns.Msg for n, upstreamConfig := range upstreamConfigs { if upstreamConfig == nil { continue @@ -911,6 +921,16 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { ctrld.Log(ctx, logger, "DNS loop detected") continue } + // A fallback-mode Internal Domain resolver that the monitor reports as + // down is not waited on: off the organization network every query + // would otherwise pay its timeout before the network fallback runs. + // Nothing else marks a generated resolver up again, so skipping it + // starts a background re-check that does. + if internalFallback && p.um.isDown(upstreams[n]) { + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolver %s is down, skipping it", upstreams[n]) + p.internalDomainProbes.probe(p.um, upstreams[n], upstreamConfig, req.msg) + continue + } answer := resolve(upstreams[n], upstreamConfig, req.msg) if answer == nil { if serveStaleCache && staleAnswer != nil { @@ -945,6 +965,12 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { ctrld.Log(ctx, mainLog.Load().Debug(), "failover rcode matched, process to next upstream") continue } + if internalFallback && internalDomainFallbackRcode(answer.Rcode) { + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolver %s answered %s, trying next resolver", + upstreams[n], dns.RcodeToString[answer.Rcode]) + internalNegative = betterInternalDomainNegative(internalNegative, answer) + continue + } // set compression, as it is not set by default when unpacking answer.Compress = true @@ -989,15 +1015,40 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { res.upstream = upstreamConfig.Endpoint return res } + if internalFallback { + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolvers did not resolve the query; trying network resolvers") + answer, negative := p.resolveInternalDomainOnNetwork(ctx, req.msg) + if answer != nil { + answer.Compress = true + if cacheable { + p.addCachedResponse(dnscache.NewKey(req.msg, upstreams[0]), answer) + ctrld.Log(ctx, mainLog.Load().Debug(), "add cached response") + } + hostname := "" + if req.ci != nil { + hostname = req.ci.Hostname + } + ctrld.Log(ctx, mainLog.Load().Info(), "REPLY: internal domain network fallback -> %s (%s): %s", req.ufr.srcAddr, hostname, dns.RcodeToString[answer.Rcode]) + res.answer = answer + return res + } + if negative = betterInternalDomainNegative(internalNegative, negative); negative != nil { + ctrld.Log(ctx, mainLog.Load().Debug(), "no resolver resolved the internal domain; returning %s", dns.RcodeToString[negative.Rcode]) + negative.Compress = true + res.answer = negative + return res + } + } ctrld.Log(ctx, p.querySampler.event(sampleClassAllEndpointsFailed, ""), "all %v endpoints failed", journalUpstreamNames(upstreams)) - // An Internal Domain with explicit resolvers is served only by the - // configured resolvers. When all of them are unreachable the query fails - // here: the OS-resolver catch-all below would send a private name to a resolver - // that was not selected for it, and the recovery flow exists for the loss of - // general DNS, not for one unavailable internal server. - if internalDomainExplicitUpstreams(upstreams) { - ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolvers unreachable; not falling back") + // An Internal Domain with explicit resolvers never takes the OS-resolver + // catch-all below: it can reach public nameservers, ctrld's own fallback + // included, which no mode selects for a private name. Fallback mode has + // already tried the VPN resolvers and the network's LAN nameservers above. + // The recovery flow exists for the loss of general DNS, not for one + // unavailable internal server, so it is not started either. + if p.internalDomainExplicitUpstreams(upstreams) { + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolvers unreachable; not using the OS resolver catch all") } else if p.leakOnUpstreamFailure() { if p.um.countHealthy(upstreams) == 0 { p.recoveryCancelMu.Lock() @@ -1046,7 +1097,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { // endpoint away from the organization network never reaches them, and that is // not an outage of the query path. func (p *prog) countFailedClientQuery(upstreams []string) { - if internalDomainExplicitUpstreams(upstreams) { + if p.internalDomainExplicitUpstreams(upstreams) { return } p.health.countFailedQuery() @@ -2705,7 +2756,7 @@ func (p *prog) buildRecoveryUpstreams(reason RecoveryReason) map[string]*ctrld.U case RecoveryReasonOSFailure: for k, uc := range p.cfg.Upstream { name := upstreamPrefix + k - if uc != nil && uc.Type != ctrld.ResolverTypeOS && !isInternalDomainUpstream(name) { + if uc != nil && uc.Type != ctrld.ResolverTypeOS && !isGeneratedInternalDomainUpstream(uc) { upstreams[name] = uc } } diff --git a/cmd/cli/internal_domains.go b/cmd/cli/internal_domains.go index 836a7dc..f53f982 100644 --- a/cmd/cli/internal_domains.go +++ b/cmd/cli/internal_domains.go @@ -9,8 +9,11 @@ import ( "sort" "strconv" "strings" + "sync" "syscall" + "time" + "github.com/miekg/dns" "github.com/rs/zerolog" "github.com/Control-D-Inc/ctrld" @@ -19,15 +22,25 @@ import ( ) // internalDomainUpstreamPrefix names the upstreams generated for organization -// Internal Domains that select explicit resolvers. The prefix is what tells -// proxy() that a rule is served only by configured resolvers, so it -// must not collide with the numeric keys used for the Control D upstream. +// Internal Domains that select explicit resolvers. It must not collide with +// the numeric key used for the Control D upstream. The key only names the +// upstream: what identifies it as generated, and in which mode, is +// UpstreamConfig.InternalDomain, which no configuration file can set. const internalDomainUpstreamPrefix = "internal_" +// Values of UpstreamConfig.InternalDomain on a generated upstream. +const ( + // internalDomainUpstreamFallback serves an Internal Domain in explicit + // resolver with network fallback mode. + internalDomainUpstreamFallback = "fallback" + // internalDomainUpstreamOnly serves an Internal Domain in explicit + // resolver only mode: its answer or failure is final. + internalDomainUpstreamOnly = "only" +) + // internalDomainUpstreamName is the Name given to every generated Internal -// Domain upstream. Together with the key prefix and the legacy type it is what -// distinguishes an upstream ctrld generated from one an endpoint custom -// configuration happens to name the same way. +// Domain upstream. It is only a display label: what identifies an upstream as +// generated is UpstreamConfig.InternalDomain, which no configuration can set. const internalDomainUpstreamName = "Internal Domain resolver" // internalDomainsSummary counts the outcome of applying Internal Domains. It @@ -37,6 +50,7 @@ type internalDomainsSummary struct { domains int // domains that produced rules osMode int // domains routed to the OS/default resolver explicit int // domains routed to configured resolvers + only int // explicit domains with no network fallback resolvers int // explicit resolver upstreams generated skipped int // entries dropped: unusable domain, no usable resolver, or duplicate preempted int // domains already covered by a higher precedence rule @@ -129,43 +143,83 @@ func internalDomainEndpoint(resolver string) (string, bool) { return net.JoinHostPort(addr.String(), port), true } -// internalDomainMode reports whether entry selects explicit resolvers, and -// whether ctrld understands its mode at all. +// internalDomainResolution is how an Internal Domain is resolved. +type internalDomainResolution int + +const ( + // internalDomainViaOS is the "network default" mode: the endpoint's + // OS/default resolver. + internalDomainViaOS internalDomainResolution = iota + // internalDomainViaResolvers is the configured resolvers first, then the + // active network and VPN resolvers. The default explicit selection. + internalDomainViaResolvers + // internalDomainViaResolversOnly is the configured resolvers and nothing + // else. + internalDomainViaResolversOnly +) + +// internalDomainMode reports how entry is resolved, and whether ctrld +// understands its mode at all. // // Mode is authoritative when the API sends it. "os" ignores any resolver // addresses the entry still carries, so switching a domain back to the OS // resolver cannot be undone by addresses a previous selection left behind; -// "resolvers" stays explicit even when the list it points at turns out to be -// unusable, rather than silently becoming OS resolution. An empty mode is a -// deployment that predates the field, where the list is the only signal. +// the explicit modes stay explicit even when the list they point at turns out +// to be unusable, rather than silently becoming OS resolution. An empty mode +// is a deployment that predates the field, where the list is the only signal, +// and an explicit list gets the default explicit mode. // // An unrecognized mode is not guessed at: routing a private domain by guess is -// the disclosure this feature exists to prevent. -func internalDomainMode(entry controld.SplitDNS) (explicit, known bool) { +// the disclosure this feature exists to prevent. It fails closed instead. With +// resolvers it is resolved as explicit resolver only, through the resolvers the +// administrator chose and nothing else, so a wire value this build does not +// know, such as a renamed strict mode, cannot send the domain to the Control D +// upstream or the network. Without resolvers there is nothing to fail closed +// to, so the entry is reported as unknown and dropped. +func internalDomainMode(entry controld.SplitDNS) (resolution internalDomainResolution, known bool) { switch strings.ToLower(strings.TrimSpace(entry.Mode)) { case controld.SplitDNSModeOS: - return false, true + return internalDomainViaOS, true case controld.SplitDNSModeResolvers: - return true, true + return internalDomainViaResolvers, true + case controld.SplitDNSModeResolversOnly: + return internalDomainViaResolversOnly, true case "": - return len(entry.Resolvers) > 0, true + if len(entry.Resolvers) > 0 { + return internalDomainViaResolvers, true + } + return internalDomainViaOS, true default: - return false, false + if len(entry.Resolvers) > 0 { + return internalDomainViaResolversOnly, true + } + return internalDomainViaOS, false } } -// internalDomainRouting returns the endpoints an entry routes to, or a reason -// it cannot be used. No endpoints and no reason means OS resolution. +// internalDomainModeRecognized reports whether mode is one this build knows, +// as opposed to one internalDomainMode resolves by failing closed. +func internalDomainModeRecognized(mode string) bool { + switch strings.ToLower(strings.TrimSpace(mode)) { + case "", controld.SplitDNSModeOS, controld.SplitDNSModeResolvers, controld.SplitDNSModeResolversOnly: + return true + } + return false +} + +// internalDomainRouting returns how an entry is resolved and the endpoints it +// routes to, or a reason it cannot be used. No endpoints and no reason means +// OS resolution. // // Generation and refresh comparison both go through here, so the two can never // disagree about what an entry means. -func internalDomainRouting(entry controld.SplitDNS) (endpoints []string, reason string) { - explicit, known := internalDomainMode(entry) +func internalDomainRouting(entry controld.SplitDNS) (resolution internalDomainResolution, endpoints []string, reason string) { + resolution, known := internalDomainMode(entry) if !known { - return nil, "unknown_mode" + return resolution, nil, "unknown_mode" } - if !explicit { - return nil, "" + if resolution == internalDomainViaOS { + return resolution, nil, "" } endpoints = make([]string, 0, len(entry.Resolvers)) for _, resolver := range entry.Resolvers { @@ -178,9 +232,9 @@ func internalDomainRouting(entry controld.SplitDNS) (endpoints []string, reason // different one, so the entry is dropped and the domain keeps its normal // path instead. if len(endpoints) == 0 { - return nil, "no_usable_resolver" + return resolution, nil, "no_usable_resolver" } - return endpoints, "" + return resolution, endpoints, "" } // internalDomainRuleExists reports whether the listener policy already routes @@ -241,7 +295,8 @@ func orderInternalDomainsBySpecificity(entries []controld.SplitDNS) []controld.S // resolvers routes to the OS/default resolver by way of an empty upstream list, // the same representation Magic Folder excludes use. An entry with resolvers // gets one generated upstream per address, listed in configured order so the -// later ones act as failover for the first. +// later ones act as failover for the first. Those upstreams carry the mode in +// UpstreamConfig.InternalDomain. // // cfg is rebuilt from the API response on every fetch, so this function is the // only writer of internal_* upstreams: a removed domain leaves nothing behind. @@ -271,12 +326,15 @@ func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) intern // written: an entry that cannot be used, or a domain already covered by // a higher precedence rule, must not leave a generated upstream behind // with no rule referring to it. - endpoints, reason := internalDomainRouting(entry) + resolution, endpoints, reason := internalDomainRouting(entry) if reason != "" { mainLog.Load().Debug().Msgf("internal domains: skipping %q: %s", domain, reason) summary.skipped++ continue } + if !internalDomainModeRecognized(entry.Mode) { + mainLog.Load().Debug().Msgf("internal domains: %q has unrecognized mode %q; using its resolvers only", domain, entry.Mode) + } seen[domain] = struct{}{} sources := []string{domain, "*." + domain} @@ -297,14 +355,19 @@ func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) intern continue } + marker := internalDomainUpstreamFallback + if resolution == internalDomainViaResolversOnly { + marker = internalDomainUpstreamOnly + } targets := make([]string, 0, len(endpoints)) for _, endpoint := range endpoints { key := internalDomainUpstreamPrefix + strconv.Itoa(summary.resolvers) cfg.Upstream[key] = &ctrld.UpstreamConfig{ - Name: internalDomainUpstreamName, - Type: ctrld.ResolverTypeLegacy, - Endpoint: endpoint, - Timeout: 5000, + Name: internalDomainUpstreamName, + Type: ctrld.ResolverTypeLegacy, + Endpoint: endpoint, + Timeout: 5000, + InternalDomain: marker, } targets = append(targets, upstreamPrefix+key) summary.resolvers++ @@ -323,63 +386,118 @@ func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) intern mainLog.Load().Debug().Msgf("internal domains: %q routed to the OS resolver", domain) } else { summary.explicit++ + if resolution == internalDomainViaResolversOnly { + summary.only++ + } mainLog.Load().Debug().Msgf("internal domains: %q routed to %v", domain, targets) } } return summary } -// isGeneratedInternalDomainUpstream reports whether a cfg.Upstream entry is one +// isGeneratedInternalDomainUpstream reports whether uc is an upstream // applyInternalDomains created. // // Generated Internal Domain resolvers are part of the managed Control D // configuration, not upstreams the endpoint operator chose, so eligibility // checks that ask "is this a plain single-upstream Control D install" must not -// count them. The key prefix alone is not proof of that, because an endpoint -// custom configuration replaces the generated config outright and could name an -// upstream anything; the shape this function checks is only produced here. -func isGeneratedInternalDomainUpstream(name string, uc *ctrld.UpstreamConfig) bool { - return uc != nil && - strings.HasPrefix(name, internalDomainUpstreamPrefix) && - uc.Type == ctrld.ResolverTypeLegacy && - uc.Name == internalDomainUpstreamName +// count them. Neither the key nor any other configurable field is proof of +// that: a local config or an endpoint custom configuration can name an upstream +// anything and give it any type and name. Only the InternalDomain marker is, +// because ctrld sets it and no configuration file can. +func isGeneratedInternalDomainUpstream(uc *ctrld.UpstreamConfig) bool { + return uc != nil && uc.InternalDomain != "" } -// isInternalDomainUpstream reports whether an upstream reference was generated -// for an Internal Domain with explicit resolvers. -func isInternalDomainUpstream(upstream string) bool { - return strings.HasPrefix(upstream, upstreamPrefix+internalDomainUpstreamPrefix) +// internalDomainUpstream returns the generated Internal Domain upstream an +// "upstream." reference names, or nil if it names any other upstream. +func (p *prog) internalDomainUpstream(upstream string) *ctrld.UpstreamConfig { + if p.cfg == nil { + return nil + } + uc := p.cfg.Upstream[strings.TrimPrefix(upstream, upstreamPrefix)] + if !isGeneratedInternalDomainUpstream(uc) { + return nil + } + return uc +} + +// isInternalDomainUpstream reports whether an upstream reference names a +// generated Internal Domain resolver, in either explicit mode. +func (p *prog) isInternalDomainUpstream(upstream string) bool { + return p.internalDomainUpstream(upstream) != nil } // internalDomainExplicitUpstreams reports whether every upstream serving a -// query is an configured Internal Domain resolver. +// query is a configured Internal Domain resolver, in either explicit mode. // -// proxy() uses this to keep such a query off the OS-resolver catch-all when the -// configured resolvers are unreachable: the administrator selected those -// resolvers and no other, so a private name must fail rather than be sent -// somewhere it was not meant to go. It also keeps an unreachable internal -// resolver from triggering the endpoint-wide recovery flow, which exists for -// the loss of general DNS, not for one unavailable internal server. -func internalDomainExplicitUpstreams(upstreams []string) bool { +// proxy() uses this to try the configured resolvers before anything else, and +// to keep such a query off the OS-resolver catch-all when they fail: that +// catch-all may reach public nameservers, and a private name must not be sent +// somewhere it was not meant to go. In fallback mode proxy() tries the +// network resolvers instead (see internalDomainFallbackUpstreams). It also +// keeps an unreachable internal resolver from triggering the endpoint-wide +// recovery flow, which exists for the loss of general DNS, not for one +// unavailable internal server. +func (p *prog) internalDomainExplicitUpstreams(upstreams []string) bool { if len(upstreams) == 0 { return false } for _, upstream := range upstreams { - if !isInternalDomainUpstream(upstream) { + if !p.isInternalDomainUpstream(upstream) { return false } } return true } +// internalDomainFallbackUpstreams reports whether upstreams are the configured +// resolvers of an Internal Domain in explicit-with-network-fallback mode, whose +// failure, SERVFAIL or NXDOMAIN hands the query to the active network and VPN +// resolvers. +func (p *prog) internalDomainFallbackUpstreams(upstreams []string) bool { + if len(upstreams) == 0 { + return false + } + for _, upstream := range upstreams { + uc := p.internalDomainUpstream(upstream) + if uc == nil || uc.InternalDomain != internalDomainUpstreamFallback { + return false + } + } + return true +} + +// internalDomainFallbackRcode reports whether a configured resolver's answer +// sends a fallback-mode query on to the next resolver. SERVFAIL means the +// resolver could not answer, and NXDOMAIN may only mean it does not know a +// name the local network's resolver does. REFUSED and NOTIMP mean it will not +// serve this endpoint, as an organization resolver with an access list answers +// an endpoint off the organization network, which is closer to unreachable +// than to an answer. Any other answer is final, including an empty NOERROR: +// the name exists, it just has no record of that type. +func internalDomainFallbackRcode(rcode int) bool { + switch rcode { + case dns.RcodeServerFailure, dns.RcodeNameError, dns.RcodeRefused, dns.RcodeNotImplemented: + return true + } + return false +} + // internalDomainsEqual reports whether two Internal Domains lists would produce // the same routing. Domains are compared in canonical form and independently of // the order the API returned them, so an unchanged list does not trigger a // reload on every refresh. Resolver order is significant: it is the order the -// administrator chose, and it decides which resolver is tried first. +// administrator chose, and it decides which resolver is tried first. The mode +// is significant too, including a switch between the two explicit modes. func internalDomainsEqual(a, b []controld.SplitDNS) bool { - canonical := func(entries []controld.SplitDNS) []controld.SplitDNS { - out := make([]controld.SplitDNS, 0, len(entries)) + type route struct { + domain string + resolution internalDomainResolution + endpoints []string + } + canonical := func(entries []controld.SplitDNS) []route { + out := make([]route, 0, len(entries)) for _, entry := range entries { domain := normalizeInternalDomain(entry.Domain) if domain == "" { @@ -387,13 +505,13 @@ func internalDomainsEqual(a, b []controld.SplitDNS) bool { } // Entries that cannot be used route nothing, so they cannot make // an otherwise unchanged list look changed. - endpoints, reason := internalDomainRouting(entry) + resolution, endpoints, reason := internalDomainRouting(entry) if reason != "" { continue } - out = append(out, controld.SplitDNS{Domain: domain, Resolvers: endpoints}) + out = append(out, route{domain: domain, resolution: resolution, endpoints: endpoints}) } - sort.Slice(out, func(i, j int) bool { return out[i].Domain < out[j].Domain }) + sort.Slice(out, func(i, j int) bool { return out[i].domain < out[j].domain }) return out } x, y := canonical(a), canonical(b) @@ -401,7 +519,7 @@ func internalDomainsEqual(a, b []controld.SplitDNS) bool { return false } for i := range x { - if x[i].Domain != y[i].Domain || !slices.Equal(x[i].Resolvers, y[i].Resolvers) { + if x[i].domain != y[i].domain || x[i].resolution != y[i].resolution || !slices.Equal(x[i].endpoints, y[i].endpoints) { return false } } @@ -429,12 +547,6 @@ func internalDomainFailureReason(err error) string { return "error" } -// isGeneratedInternalDomainUpstreamRef is isGeneratedInternalDomainUpstream for -// call sites that hold an "upstream." reference rather than the bare key. -func isGeneratedInternalDomainUpstreamRef(upstream string, uc *ctrld.UpstreamConfig) bool { - return isGeneratedInternalDomainUpstream(strings.TrimPrefix(upstream, upstreamPrefix), uc) -} - // logUpstreamProbeFailure reports a background probe failure against the // upstream that the reference upstream names. // @@ -450,10 +562,199 @@ func logUpstreamProbeFailure(upstream string, uc *ctrld.UpstreamConfig, err erro detail := append([]any{}, args...) mainLog.Load().Debug().Err(err).Msgf(format+" for upstream: %q, endpoint: %q", append(detail, uc.Name, uc.Endpoint)...) event := level().Str("upstream", journalUpstreamName(upstream)) - if !isGeneratedInternalDomainUpstreamRef(upstream, uc) { + if !isGeneratedInternalDomainUpstream(uc) { event.Err(err).Msgf(format, detail...) return } event.Str("failure", internalDomainFailureReason(err)). Msgf(format+" for an Internal Domain resolver", detail...) } + +// betterInternalDomainNegative returns whichever of two negative answers a +// fallback-mode query should end with: NXDOMAIN over SERVFAIL, since a +// resolver that says the name does not exist has answered, and otherwise the +// first one seen. +func betterInternalDomainNegative(current, candidate *dns.Msg) *dns.Msg { + switch { + case candidate == nil: + return current + case current == nil: + return candidate + case candidate.Rcode == dns.RcodeNameError && current.Rcode != dns.RcodeNameError: + return candidate + } + return current +} + +// internalDomainVPNUpstream returns the upstream the network fallback uses for +// a VPN DNS server. A seam: VPN DNS servers are dialed on port 53, which a test +// fixture cannot listen on. +var internalDomainVPNUpstream = func(m *vpnDNSManager, server string) *ctrld.UpstreamConfig { + return m.upstreamConfigFor(server) +} + +// internalDomainOSResolve resolves msg through the OS resolver for the network +// fallback. A seam, so a test sees the context the fallback passes: that +// context is what keeps the private name off public nameservers. +var internalDomainOSResolve = func(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + resolver, err := ctrld.NewResolver(osUpstreamConfig) + if err != nil { + return nil, err + } + resolveCtx, cancel := osUpstreamConfig.Context(ctx) + defer cancel() + return resolver.Resolve(resolveCtx, msg) +} + +// resolveInternalDomainOnNetwork is the network fallback of an Internal Domain +// in explicit-with-network-fallback mode. proxy() runs it once the configured +// resolvers have failed, or answered SERVFAIL or NXDOMAIN. +// +// It asks only resolvers that the endpoint's active network or VPN provides, +// in this order: VPN DNS servers whose domains match the query, then VPN DNS +// servers with no domains (both only in dns-intercept mode, where ctrld tracks +// them), then the OS resolver's LAN nameservers. The OS resolver is asked as a +// LAN-only query, so the private name never reaches a public nameserver, not +// even one DHCP supplied, nor ctrld's own public fallback. With no LAN +// nameserver the OS step sends nothing. +// +// The first NOERROR answer is returned as answer, including an empty one, +// which is final. Otherwise negative is the best SERVFAIL or NXDOMAIN answer +// seen. While Windows is serving retained VPN DNS state, a VPN transport +// failure stops the fallback before the OS resolver, as it does for VPN split +// routing. +func (p *prog) resolveInternalDomainOnNetwork(ctx context.Context, msg *dns.Msg) (answer, negative *dns.Msg) { + domain := msg.Question[0].Name + resolve := func(ctx context.Context, uc *ctrld.UpstreamConfig) (*dns.Msg, error) { + resolver, err := ctrld.NewResolver(uc) + if err != nil { + return nil, err + } + resolveCtx, cancel := uc.Context(ctx) + defer cancel() + return resolver.Resolve(resolveCtx, msg) + } + // accept reports whether a network answer is final, and keeps a negative + // one for the end. + accept := func(source string, m *dns.Msg) bool { + if !sameQuestion(msg, m) { + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain fallback: discarding answer from %s: question mismatch", source) + return false + } + if m.Rcode == dns.RcodeSuccess { + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain fallback: %s answered %s", source, domain) + return true + } + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain fallback: %s answered %s for %s", + source, dns.RcodeToString[m.Rcode], domain) + if internalDomainFallbackRcode(m.Rcode) { + negative = betterInternalDomainNegative(negative, m) + } + return false + } + + if dnsIntercept && p.vpnDNS != nil { + var servers []string + for _, server := range slices.Concat(p.vpnDNS.UpstreamForDomain(domain), p.vpnDNS.DomainlessServers()) { + if !slices.Contains(servers, server) { + servers = append(servers, server) + } + } + var gotAnswer, gotTransportFailure bool + for _, server := range servers { + m, err := resolve(ctx, internalDomainVPNUpstream(p.vpnDNS, server)) + if m == nil { + gotTransportFailure = true + ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "internal domain fallback: VPN DNS server %s failed", server) + continue + } + gotAnswer = true + p.vpnDNS.VPNDNSReachable() + if accept("VPN DNS server "+server, m) { + return m, nil + } + } + if !gotAnswer && gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, servers) { + return nil, negative + } + } + + m, err := internalDomainOSResolve(ctrld.LanOnlyQueryCtx(ctx), msg) + if m == nil { + ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "internal domain fallback: OS resolver LAN nameservers failed") + return nil, negative + } + if accept("OS resolver", m) { + return m, nil + } + return nil, negative +} + +// internalDomainProbeInterval is how often, at most, a down fallback-mode +// Internal Domain resolver is re-checked in the background. +const internalDomainProbeInterval = 30 * time.Second + +// internalDomainProber re-checks down fallback-mode Internal Domain resolvers +// in the background, and marks one up again when it answers. +// +// proxy() skips such a resolver while it is down, so that an endpoint away +// from the organization network goes to the network fallback without waiting +// on it. A generated resolver comes back up only by answering a query: it is +// excluded from recovery, and the loop checker leaves down upstreams alone. So +// without this re-check, a resolver that went down would stay skipped until +// the next reload, even once the endpoint is back on the organization network. +// +// The zero value is ready to use. +type internalDomainProber struct { + mu sync.Mutex + last map[string]time.Time + + // now and start are seams, so a test controls the clock and runs the + // re-check itself. nil means time.Now and a new goroutine. + now func() time.Time + start func(func()) +} + +// probe re-checks upstream in the background, at most once per +// internalDomainProbeInterval, by sending it a copy of msg. The query goes to +// the organization's own resolver, which it was meant for anyway. Any DNS +// answer proves the resolver reachable, the same as on the query path, and +// marks it up in um; a failure leaves it down until the next re-check. +func (pr *internalDomainProber) probe(um *upstreamMonitor, upstream string, uc *ctrld.UpstreamConfig, msg *dns.Msg) { + now := time.Now + if pr.now != nil { + now = pr.now + } + pr.mu.Lock() + if last, ok := pr.last[upstream]; ok && now().Sub(last) < internalDomainProbeInterval { + pr.mu.Unlock() + return + } + if pr.last == nil { + pr.last = make(map[string]time.Time) + } + pr.last[upstream] = now() + pr.mu.Unlock() + + query := msg.Copy() + run := func() { + var answer *dns.Msg + resolver, err := ctrld.NewResolver(uc) + if err == nil { + ctx, cancel := uc.Context(context.Background()) + answer, err = resolver.Resolve(ctx, query) + cancel() + } + if answer == nil { + mainLog.Load().Debug().Err(err).Msgf("internal domains: %s is still down", upstream) + return + } + mainLog.Load().Debug().Msgf("internal domains: %s answered the re-check", upstream) + um.noteSuccess(upstream) + } + if pr.start != nil { + pr.start(run) + return + } + go run() +} diff --git a/cmd/cli/internal_domains_fallback_test.go b/cmd/cli/internal_domains_fallback_test.go new file mode 100644 index 0000000..bcf32b9 --- /dev/null +++ b/cmd/cli/internal_domains_fallback_test.go @@ -0,0 +1,582 @@ +package cli + +import ( + "context" + "net" + "strings" + "testing" + "time" + + "github.com/miekg/dns" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// startRcodeDNSFixture is startDNSFixture answering every question with rcode. +// A NOERROR answer carries an A record unless empty is set, which gives the +// name-exists-but-no-record-of-that-type answer. +func startRcodeDNSFixture(t *testing.T, rcode int, empty bool) *dnsFixture { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + f := &dnsFixture{addr: pc.LocalAddr().String()} + started := make(chan struct{}) + server := &dns.Server{ + PacketConn: pc, + NotifyStartedFunc: func() { close(started) }, + Handler: dns.HandlerFunc(func(w dns.ResponseWriter, msg *dns.Msg) { + f.mu.Lock() + if len(msg.Question) > 0 { + f.qs = append(f.qs, msg.Question[0].Name) + } + f.mu.Unlock() + reply := new(dns.Msg) + reply.SetRcode(msg, rcode) + if rcode == dns.RcodeSuccess && !empty && len(msg.Question) > 0 { + rr, _ := dns.NewRR(msg.Question[0].Name + " 60 IN A 10.9.9.9") + reply.Answer = append(reply.Answer, rr) + } + _ = w.WriteMsg(reply) + }), + } + go func() { _ = server.ActivateAndServe() }() + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("DNS fixture did not start") + } + t.Cleanup(func() { _ = server.Shutdown() }) + return f +} + +func deadLoopbackAddr(t *testing.T) string { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := pc.LocalAddr().String() + _ = pc.Close() + return addr +} + +// Explicit resolver with network fallback is the default explicit mode: the +// configured resolver is asked first, and a timeout, an unreachable resolver, +// SERVFAIL or NXDOMAIN hands the query to the network's resolver. A valid +// empty answer is final, and "explicit resolver only" never falls back. +func TestInternalDomainsNetworkFallback(t *testing.T) { + for _, tc := range []struct { + name string + mode string + // internal is the configured resolver's rcode; -1 is unreachable. + internal int + internalEmpty bool + network int + wantNetwork bool + wantRcode int + wantAnswer bool + }{ + {name: "nxdomain falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "servfail falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeServerFailure, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "unreachable falls back", mode: controld.SplitDNSModeResolvers, internal: -1, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "absent mode falls back", mode: "", internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "answer is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeSuccess, network: dns.RcodeSuccess, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "empty answer is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeSuccess, internalEmpty: true, network: dns.RcodeSuccess, wantRcode: dns.RcodeSuccess}, + {name: "refused falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeRefused, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "notimp falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNotImplemented, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "refused and network nxdomain", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeRefused, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError}, + {name: "formerr is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeFormatError, network: dns.RcodeSuccess, wantRcode: dns.RcodeFormatError}, + {name: "only mode keeps refused", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeRefused, network: dns.RcodeSuccess, wantRcode: dns.RcodeRefused}, + {name: "unknown mode fails closed", mode: "explicit_only", internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantRcode: dns.RcodeNameError}, + {name: "nobody knows the name", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError}, + {name: "configured nxdomain beats network servfail", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeServerFailure, wantNetwork: true, wantRcode: dns.RcodeNameError}, + {name: "unreachable and network nxdomain", mode: controld.SplitDNSModeResolvers, internal: -1, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError}, + {name: "only mode keeps nxdomain", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantRcode: dns.RcodeNameError}, + {name: "only mode keeps servfail", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeServerFailure, network: dns.RcodeSuccess, wantRcode: dns.RcodeServerFailure}, + {name: "only mode fails when unreachable", mode: controld.SplitDNSModeResolversOnly, internal: -1, network: dns.RcodeSuccess, wantRcode: dns.RcodeServerFailure}, + } { + t.Run(tc.name, func(t *testing.T) { + network := startRcodeDNSFixture(t, tc.network, false) + useOSResolverFixture(t, network) + + var internal *dnsFixture + resolver := deadLoopbackAddr(t) + if tc.internal >= 0 { + internal = startRcodeDNSFixture(t, tc.internal, tc.internalEmpty) + resolver = internal.addr + } + + cfg := internalDomainsTestConfig() + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: tc.mode, Resolvers: []string{resolver}}}) + p := newInternalDomainsProg(t, cfg) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if internal != nil { + if got := internal.questions(); len(got) != 1 { + t.Errorf("configured resolver questions = %v, want it asked first", got) + } + } + if got := network.questions(); (len(got) > 0) != tc.wantNetwork { + t.Errorf("network resolver questions = %v, want asked = %v", got, tc.wantNetwork) + } + if res == nil || res.answer == nil { + t.Fatalf("no answer") + } + if res.answer.Rcode != tc.wantRcode { + t.Errorf("rcode = %s, want %s", dns.RcodeToString[res.answer.Rcode], dns.RcodeToString[tc.wantRcode]) + } + if got := len(res.answer.Answer) > 0; got != tc.wantAnswer { + t.Errorf("answer records = %v, want records = %v", res.answer.Answer, tc.wantAnswer) + } + }) + } +} + +// The configured resolvers are all tried, in order, before the network: an +// NXDOMAIN from the first does not skip the second. +func TestInternalDomainsNetworkFallbackTriesEveryConfiguredResolverFirst(t *testing.T) { + network := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, network) + first := startRcodeDNSFixture(t, dns.RcodeNameError, false) + second := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{first.addr, second.addr}}}) + p := newInternalDomainsProg(t, cfg) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if len(first.questions()) != 1 || len(second.questions()) != 1 { + t.Errorf("configured resolvers asked %v, %v; want both once", first.questions(), second.questions()) + } + if got := network.questions(); len(got) != 0 { + t.Errorf("network resolver asked %v although a configured resolver answered", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the second resolver's NOERROR", res) + } +} + +// The fallback is no reason to log the private resolver address above debug, +// or the domain: the summary-only REPLY line carries neither. +func TestInternalDomainsNetworkFallbackLogsNoPrivateValues(t *testing.T) { + network := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, network) + internal := startRcodeDNSFixture(t, dns.RcodeNameError, false) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + before := len(logOutput.String()) + p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + out := logOutput.String()[before:] + + if !strings.Contains(out, "internal domain network fallback") { + t.Fatalf("expected the fallback REPLY line, got: %s", out) + } + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + if line == "" || strings.Contains(line, `"level":"debug"`) { + continue + } + if strings.Contains(line, internal.addr) || strings.Contains(line, "corp.example") { + t.Errorf("private value disclosed above debug level: %s", line) + } + } +} + +// internalDomainUpstreamsProg returns a prog whose config has one legacy +// upstream per key, each carrying the given InternalDomain marker. +func internalDomainUpstreamsProg(markers map[string]string) *prog { + cfg := &ctrld.Config{Upstream: make(map[string]*ctrld.UpstreamConfig, len(markers))} + for key, marker := range markers { + cfg.Upstream[key] = &ctrld.UpstreamConfig{ + Name: internalDomainUpstreamName, + Type: ctrld.ResolverTypeLegacy, + Endpoint: "10.0.0.53:53", + InternalDomain: marker, + } + } + return &prog{cfg: cfg} +} + +func TestInternalDomainFallbackUpstreams(t *testing.T) { + p := internalDomainUpstreamsProg(map[string]string{ + "internal_0": internalDomainUpstreamFallback, + "internal_1": internalDomainUpstreamFallback, + "internal_2": internalDomainUpstreamOnly, + "0": "", + }) + for _, tc := range []struct { + in []string + want bool + }{ + {in: nil, want: false}, + {in: []string{"upstream.internal_0", "upstream.internal_1"}, want: true}, + {in: []string{"upstream.internal_2"}, want: false}, + {in: []string{"upstream.internal_0", "upstream.internal_2"}, want: false}, + {in: []string{"upstream.internal_0", "upstream.0"}, want: false}, + {in: []string{upstreamOS}, want: false}, + } { + if got := p.internalDomainFallbackUpstreams(tc.in); got != tc.want { + t.Errorf("internalDomainFallbackUpstreams(%v) = %v, want %v", tc.in, got, tc.want) + } + } + if !p.internalDomainExplicitUpstreams([]string{"upstream.internal_2"}) { + t.Error("an explicit-only upstream must still count as an Internal Domain resolver") + } +} + +// Both explicit modes generate internal_ upstreams; the mode is carried by +// the InternalDomain marker. Switching between the modes is a routing change +// the next refresh must apply. +func TestInternalDomainsExplicitModes(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}, + {Domain: "b.example.com", Mode: " Resolvers_Only ", Resolvers: []string{"10.0.0.54"}}, + }) + if summary.explicit != 2 || summary.only != 1 || summary.resolvers != 2 { + t.Fatalf("summary = %+v", summary) + } + for _, tc := range []struct{ domain, target, marker string }{ + {"a.example.com", "upstream.internal_0", internalDomainUpstreamFallback}, + {"b.example.com", "upstream.internal_1", internalDomainUpstreamOnly}, + } { + if targets, _ := ruleTargets(t, cfg, tc.domain); strings.Join(targets, ",") != tc.target { + t.Errorf("%s targets = %v, want %s", tc.domain, targets, tc.target) + } + uc := cfg.Upstream[strings.TrimPrefix(tc.target, upstreamPrefix)] + if uc == nil || uc.InternalDomain != tc.marker { + t.Errorf("%s upstream = %+v, want marker %q", tc.domain, uc, tc.marker) + } + } + + fallback := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}} + only := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{"10.0.0.53"}}} + legacy := []controld.SplitDNS{{Domain: "a.example.com", Resolvers: []string{"10.0.0.53"}}} + if internalDomainsEqual(fallback, only) { + t.Error("switching to explicit resolver only must be detected as a change") + } + if !internalDomainsEqual(fallback, legacy) { + t.Error("an absent mode with resolvers is the default explicit mode") + } +} + +// An upstream a configuration defines is an ordinary upstream, whatever it is +// named and however closely it copies a generated one: a local ctrld.toml that +// names an upstream internal_, gives it the legacy type and the generated +// name, must still get the OS-resolver catch-all when it fails. +func TestInternalDomainsUserDefinedLookalikeIsOrdinaryUpstream(t *testing.T) { + osFixture := startDNSFixture(t) + useOSResolverFixture(t, osFixture) + + cfg := internalDomainsTestConfig() + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) + cfg.Upstream["internal_0"] = &ctrld.UpstreamConfig{ + Name: internalDomainUpstreamName, + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadLoopbackAddr(t), + Timeout: 500, + } + cfg.Listener["0"].Policy.Rules = append(cfg.Listener["0"].Policy.Rules, + ctrld.Rule{"*.corp.example": []string{"upstream.internal_0"}}) + p := newInternalDomainsProg(t, cfg) + if p.isInternalDomainUpstream("upstream.internal_0") || isGeneratedInternalDomainUpstream(cfg.Upstream["internal_0"]) { + t.Fatal("a user-defined upstream was recognized as a generated Internal Domain resolver") + } + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if got := osFixture.questions(); len(got) != 1 { + t.Errorf("OS resolver questions = %v, want the ordinary catch all", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the OS resolver's NOERROR", res) + } +} + +// captureInternalDomainProbes makes p's background re-checks run only when the +// test calls them, and returns the ones scheduled so far. +func captureInternalDomainProbes(p *prog) *[]func() { + var probes []func() + p.internalDomainProbes.start = func(run func()) { probes = append(probes, run) } + return &probes +} + +// The off-site case that fallback mode exists for: the organization resolver +// is unreachable and the network answers. Once the monitor reports the +// resolver down, a query must go to the network without waiting on the +// resolver's timeout, and a background re-check must bring it back once it +// answers again. +func TestInternalDomainsDownResolverIsSkippedAndReChecked(t *testing.T) { + network := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, network) + blackhole := blackholeUDPAddr(t) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{blackhole}}}) + p := newInternalDomainsProg(t, cfg) + probes := captureInternalDomainProbes(p) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ask := func() (*proxyResponse, time.Duration) { + t.Helper() + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + start := time.Now() + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + return res, time.Since(start) + } + + const upstream = upstreamPrefix + internalDomainUpstreamPrefix + "0" + p.um.mu.Lock() + p.um.markDown(upstream, maxFailureRequest, "immediate") + p.um.mu.Unlock() + + for i := 0; i < 2; i++ { + res, elapsed := ask() + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Fatalf("query %d: answer = %v, want the network's NOERROR", i+1, res) + } + // The blackholed resolver would cost its 2s read timeout. + if elapsed > time.Second { + t.Fatalf("query %d took %v: it waited on a resolver that is down", i+1, elapsed) + } + } + if got := len(*probes); got != 1 { + t.Fatalf("re-checks scheduled = %d, want one per interval", got) + } + + // The resolver is still unreachable: the re-check leaves it down. + (*probes)[0]() + if !p.um.isDown(upstream) { + t.Fatal("a failed re-check marked the resolver up") + } + + // Back on the organization network: the next re-check, once the interval + // has passed, finds it answering and marks it up, so queries use it again. + internal := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + cfg.Upstream["internal_0"].Endpoint = internal.addr + later := time.Now().Add(internalDomainProbeInterval) + p.internalDomainProbes.now = func() time.Time { return later } + ask() + if got := len(*probes); got != 2 { + t.Fatalf("re-checks scheduled = %d, want a second one after the interval", got) + } + (*probes)[1]() + if p.um.isDown(upstream) { + t.Fatal("the resolver answered the re-check but is still down") + } + before := len(network.questions()) + if res, _ := ask(); res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Fatalf("answer = %v, want the resolver's NOERROR", res) + } + if got := internal.questions(); len(got) != 2 { + t.Errorf("resolver questions = %v, want the re-check and the query", got) + } + if got := len(network.questions()); got != before { + t.Errorf("network asked again although the resolver answered") + } +} + +// Explicit resolver only has no fallback to skip to, so a down resolver is +// still asked: skipping it would only turn its answer into a SERVFAIL. +func TestInternalDomainsOnlyModeDoesNotSkipDownResolver(t *testing.T) { + useOSResolverFixture(t, startDNSFixture(t)) + internal := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + probes := captureInternalDomainProbes(p) + + p.um.mu.Lock() + p.um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"0", maxFailureRequest, "immediate") + p.um.mu.Unlock() + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if got := internal.questions(); len(got) != 1 { + t.Errorf("resolver questions = %v, want it asked although it is down", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the resolver's NOERROR", res) + } + if len(*probes) != 0 { + t.Errorf("explicit resolver only scheduled %d re-checks", len(*probes)) + } +} + +// fallbackTestQuery sends one query for host.corp.example through proxy(). +func fallbackTestQuery(t *testing.T, p *prog, cfg *ctrld.Config) *proxyResponse { + t.Helper() + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + return p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) +} + +// The OS step of the fallback is what keeps the private name off public +// nameservers, and it does so through the context it passes. A test that +// swaps the OS resolver for a single-endpoint fixture would pass with any +// context, so this one sees the context itself and runs the step through a +// real OS resolver whose pool holds a LAN and a public nameserver. +func TestInternalDomainsNetworkFallbackOSStepIsLanOnly(t *testing.T) { + lan := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + pool := ctrld.NewResolverWithNameserver([]string{lan.addr, "192.0.2.53:53"}) + var lanOnly []bool + prev := internalDomainOSResolve + internalDomainOSResolve = func(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + isLanOnly, _ := ctx.Value(ctrld.LanOnlyQueryCtxKey{}).(bool) + lanOnly = append(lanOnly, isLanOnly) + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + return pool.Resolve(ctx, msg) + } + t.Cleanup(func() { internalDomainOSResolve = prev }) + + internal := startRcodeDNSFixture(t, dns.RcodeNameError, false) + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + + res := fallbackTestQuery(t, p, cfg) + if len(lanOnly) != 1 || !lanOnly[0] { + t.Fatalf("OS step contexts LAN-only = %v, want one LAN-only query", lanOnly) + } + if got := lan.questions(); len(got) != 1 { + t.Errorf("LAN nameserver questions = %v, want the fallback query", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the LAN nameserver's NOERROR", res) + } +} + +// useVPNFallbackFixtures routes the fallback's VPN DNS servers to fixtures: +// real VPN DNS servers are dialed on port 53. +func useVPNFallbackFixtures(t *testing.T, servers map[string]string) { + t.Helper() + prev := internalDomainVPNUpstream + internalDomainVPNUpstream = func(_ *vpnDNSManager, server string) *ctrld.UpstreamConfig { + return &ctrld.UpstreamConfig{Name: "VPN DNS", Type: ctrld.ResolverTypeLegacy, Endpoint: servers[server], Timeout: 1000} + } + t.Cleanup(func() { internalDomainVPNUpstream = prev }) +} + +// The fallback asks VPN DNS servers that match the domain, then domain-less +// ones, then the OS resolver, and stops at the first NOERROR. +func TestInternalDomainsNetworkFallbackVPNOrder(t *testing.T) { + for _, tc := range []struct { + name string + matched, domainless int + wantMatched, wantDomless int + wantOS bool + }{ + {name: "matching VPN server answers", matched: dns.RcodeSuccess, domainless: dns.RcodeSuccess, wantMatched: 1}, + {name: "then the domain-less one", matched: dns.RcodeNameError, domainless: dns.RcodeSuccess, wantMatched: 1, wantDomless: 1}, + {name: "then the OS resolver", matched: dns.RcodeNameError, domainless: dns.RcodeRefused, wantMatched: 1, wantDomless: 1, wantOS: true}, + } { + t.Run(tc.name, func(t *testing.T) { + useDNSIntercept(t) + osFixture := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, osFixture) + matched := startRcodeDNSFixture(t, tc.matched, false) + domainless := startRcodeDNSFixture(t, tc.domainless, false) + useVPNFallbackFixtures(t, map[string]string{"10.8.0.1": matched.addr, "10.8.0.2": domainless.addr}) + + internal := startRcodeDNSFixture(t, dns.RcodeNameError, false) + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + p.vpnDNS = &vpnDNSManager{ + routes: map[string][]string{"corp.example": {"10.8.0.1"}}, + domainlessServers: []string{"10.8.0.2"}, + } + + res := fallbackTestQuery(t, p, cfg) + if got := len(matched.questions()); got != tc.wantMatched { + t.Errorf("matching VPN server asked %d times, want %d", got, tc.wantMatched) + } + if got := len(domainless.questions()); got != tc.wantDomless { + t.Errorf("domain-less VPN server asked %d times, want %d", got, tc.wantDomless) + } + if got := len(osFixture.questions()) > 0; got != tc.wantOS { + t.Errorf("OS resolver asked = %v, want %v", got, tc.wantOS) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want NOERROR", res) + } + }) + } +} + +// While Windows is serving retained VPN DNS state, a VPN transport failure +// stops the fallback before the OS resolver, as it does for VPN split routing. +func TestInternalDomainsNetworkFallbackStopsOnRetainedVPNTransportFailure(t *testing.T) { + useDNSIntercept(t) + prevSettling := vpnDNSSettlingEnabled + vpnDNSSettlingEnabled = true + t.Cleanup(func() { vpnDNSSettlingEnabled = prevSettling }) + osFixture := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, osFixture) + useVPNFallbackFixtures(t, map[string]string{"10.8.0.1": deadLoopbackAddr(t)}) + + internal := startRcodeDNSFixture(t, dns.RcodeNameError, false) + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + p.vpnDNS = &vpnDNSManager{ + routes: map[string][]string{"corp.example": {"10.8.0.1"}}, + retainedAfterEmptyDiscovery: true, + } + + res := fallbackTestQuery(t, p, cfg) + if got := osFixture.questions(); len(got) != 0 { + t.Errorf("OS resolver asked %v while retained VPN DNS state is active", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeNameError { + t.Errorf("answer = %v, want the configured resolver's NXDOMAIN", res) + } +} diff --git a/cmd/cli/internal_domains_review_test.go b/cmd/cli/internal_domains_review_test.go index d2ece04..8a65c97 100644 --- a/cmd/cli/internal_domains_review_test.go +++ b/cmd/cli/internal_domains_review_test.go @@ -139,9 +139,10 @@ func TestInternalDomainsExplicitResolverSurvivesRecoveryBypass(t *testing.T) { } } -// When the configured resolvers are unreachable the query must fail rather than -// reach the OS resolver, and the failure must be observable on the wire as an -// attempt against the configured address. +// In "explicit resolver only" mode, when the configured resolvers are +// unreachable the query must fail rather than reach the OS resolver, and the +// failure must be observable on the wire as an attempt against the configured +// address. func TestInternalDomainsUnreachableResolverDoesNotReachOSResolver(t *testing.T) { osFixture := startDNSFixture(t) useOSResolverFixture(t, osFixture) @@ -157,7 +158,7 @@ func TestInternalDomainsUnreachableResolverDoesNotReachOSResolver(t *testing.T) cfg := internalDomainsTestConfig() cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) - applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{dead}}}) + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{dead}}}) p := newInternalDomainsProg(t, cfg) addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") @@ -301,6 +302,10 @@ func TestInternalDomainsSetupKeepsResolverAddressesOutOfInfoLogs(t *testing.T) { // Domain upstream that endpoint is private, so only a classification may appear // at error level. func TestInternalDomainsResolverFailureLogsNoAddress(t *testing.T) { + // The default mode falls back to the network after the failure; keep that + // on loopback. + useOSResolverFixture(t, startDNSFixture(t)) + pc, err := net.ListenPacket("udp", "127.0.0.1:0") if err != nil { t.Fatal(err) diff --git a/cmd/cli/internal_domains_test.go b/cmd/cli/internal_domains_test.go index f96323e..243ff3e 100644 --- a/cmd/cli/internal_domains_test.go +++ b/cmd/cli/internal_domains_test.go @@ -403,6 +403,11 @@ func TestInternalDomainsEqual(t *testing.T) { } func TestInternalDomainExplicitUpstreams(t *testing.T) { + p := internalDomainUpstreamsProg(map[string]string{ + "internal_0": internalDomainUpstreamFallback, + "internal_1": internalDomainUpstreamOnly, + "0": "", + }) for _, tc := range []struct { in []string want bool @@ -415,7 +420,7 @@ func TestInternalDomainExplicitUpstreams(t *testing.T) { {nil, false}, {[]string{}, false}, } { - if got := internalDomainExplicitUpstreams(tc.in); got != tc.want { + if got := p.internalDomainExplicitUpstreams(tc.in); got != tc.want { t.Errorf("internalDomainExplicitUpstreams(%v) = %v, want %v", tc.in, got, tc.want) } } @@ -448,13 +453,13 @@ func TestInternalDomainsSummaryLogsCountsOnly(t *testing.T) { // An explicit-resolver Internal Domain must not be handed to VPN DNS split // routing: the administrator named those resolvers, and VPN suffixes are -// auto-detected. When the named resolvers are unreachable the query fails -// instead of reaching the OS resolver. +// auto-detected. In "explicit resolver only" mode, when the named resolvers +// are unreachable the query fails instead of reaching the OS resolver. func TestInternalDomainsBeatVPNSplitRoutingAndDoNotLeak(t *testing.T) { cfg := internalDomainsTestConfig() cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) applyInternalDomains(cfg, []controld.SplitDNS{ - {Domain: "aws.example.com", Resolvers: []string{"127.0.0.1:1"}}, + {Domain: "aws.example.com", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{"127.0.0.1:1"}}, }) p := &prog{cfg: cfg} @@ -473,7 +478,7 @@ func TestInternalDomainsBeatVPNSplitRoutingAndDoNotLeak(t *testing.T) { } ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.aws.example.com") - if !internalDomainExplicitUpstreams(ufr.upstreams) { + if !p.internalDomainExplicitUpstreams(ufr.upstreams) { t.Fatalf("upstreams = %v, want only internal resolvers", ufr.upstreams) } @@ -571,9 +576,18 @@ func TestApplyInternalDomainsModeIsAuthoritative(t *testing.T) { wantUpstreams: 1, }, { - // A mode ctrld does not know is not guessed at. - name: "unknown mode is dropped", + // A wire value this build does not know fails closed to the + // resolvers the administrator chose, never to the Control D upstream. + name: "unknown mode with resolvers fails closed", entry: controld.SplitDNS{Domain: "a.example.com", Mode: "forward-someday", Resolvers: []string{"10.0.0.53"}}, + wantTargets: []string{"upstream.internal_0"}, + wantRule: true, + wantUpstreams: 2, + }, + { + // With no resolvers there is nothing to fail closed to. + name: "unknown mode without resolvers is dropped", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: "forward-someday"}, wantRule: false, wantUpstreams: 1, }, diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 8bb1f77..09c3809 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -140,6 +140,7 @@ type prog struct { internalLogWriter *logWriter internalJournalWriter *logWriter querySampler errorSampler + internalDomainProbes internalDomainProber lastNetworkState atomic.Pointer[netmon.State] internalLogSent time.Time runningIface string @@ -859,7 +860,7 @@ func (p *prog) setupUpstream(cfg *ctrld.Config) { // gated on the API confirming the device is gone. managedUpstreams := 0 for n := range cfg.Upstream { - if isGeneratedInternalDomainUpstream(n, cfg.Upstream[n]) { + if isGeneratedInternalDomainUpstream(cfg.Upstream[n]) { continue } managedUpstreams++ @@ -1161,7 +1162,7 @@ func (p *prog) startNetworkJournal() func() { close(stop) }() healthDone := p.health.startLoop(stop, func() (int, bool) { - return p.upstreamMonitorNow().countDownExcept(isInternalDomainUpstream), p.recoveryBypass.Load() + return p.upstreamMonitorNow().countDownExcept(p.isInternalDomainUpstream), p.recoveryBypass.Load() }) var configDone <-chan struct{} if p.dnsConfig != nil { diff --git a/cmd/cli/query_health_prog_test.go b/cmd/cli/query_health_prog_test.go index cd0520d..0281c65 100644 --- a/cmd/cli/query_health_prog_test.go +++ b/cmd/cli/query_health_prog_test.go @@ -107,10 +107,11 @@ func Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade(t *testing.T) { cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) cfg.Upstream = map[string]*ctrld.UpstreamConfig{ internalDomainUpstreamPrefix + "1": { - Name: "dead internal resolver", - Type: ctrld.ResolverTypeLegacy, - Endpoint: deadUpstreamEndpoint, - Timeout: 500, + Name: "dead internal resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 500, + InternalDomain: internalDomainUpstreamOnly, }, } p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg), health: newQueryHealth()} @@ -140,7 +141,7 @@ func Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade(t *testing.T) { p.um.markDown(internalUpstream, 3, "immediate") p.um.mu.Unlock() - event, emit := p.health.evaluate(time.Now(), p.um.countDownExcept(isInternalDomainUpstream), false) + event, emit := p.health.evaluate(time.Now(), p.um.countDownExcept(p.isInternalDomainUpstream), false) if !emit { t.Fatal("the first evaluate did not emit") } diff --git a/cmd/cli/recovery_scope_test.go b/cmd/cli/recovery_scope_test.go index fc29426..63c71f3 100644 --- a/cmd/cli/recovery_scope_test.go +++ b/cmd/cli/recovery_scope_test.go @@ -303,7 +303,9 @@ func TestScopeHarnessRestoresRecoveryWait(t *testing.T) { func TestOSRecoveryCandidates(t *testing.T) { remote := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeDOH} - internal := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy} + internal := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy, InternalDomain: internalDomainUpstreamFallback} + // A user-defined upstream that copies a generated key carries no marker. + lookalike := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy} osConfig := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeOS} for _, tc := range []struct { name string @@ -311,9 +313,10 @@ func TestOSRecoveryCandidates(t *testing.T) { want map[string]*ctrld.UpstreamConfig }{ {"configured", map[string]*ctrld.UpstreamConfig{"0": remote, "internal_0": internal, "1": osConfig, "2": nil}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote}}, - {"custom_policy_or_unused", map[string]*ctrld.UpstreamConfig{"0": remote, "unused": internal}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote, "upstream.unused": internal}}, + {"custom_policy_or_unused", map[string]*ctrld.UpstreamConfig{"0": remote, "unused": lookalike}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote, "upstream.unused": lookalike}}, {"only_os", map[string]*ctrld.UpstreamConfig{"0": osConfig}, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}}, {"only_internal", map[string]*ctrld.UpstreamConfig{"internal_0": internal}, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}}, + {"internal_lookalike", map[string]*ctrld.UpstreamConfig{"internal_0": lookalike}, map[string]*ctrld.UpstreamConfig{"upstream.internal_0": lookalike}}, {"empty", nil, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/cli/upstream_monitor_test.go b/cmd/cli/upstream_monitor_test.go index 20f6a9b..3447144 100644 --- a/cmd/cli/upstream_monitor_test.go +++ b/cmd/cli/upstream_monitor_test.go @@ -293,7 +293,12 @@ func Test_upstreamMonitorCountDownExcept(t *testing.T) { if got := um.countDown(); got != 3 { t.Fatalf("countDown = %d, want 3", got) } - if got := um.countDownExcept(isInternalDomainUpstream); got != 1 { + p := internalDomainUpstreamsProg(map[string]string{ + "0": "", + internalDomainUpstreamPrefix + "1": internalDomainUpstreamFallback, + internalDomainUpstreamPrefix + "2": internalDomainUpstreamOnly, + }) + if got := um.countDownExcept(p.isInternalDomainUpstream); got != 1 { t.Fatalf("countDownExcept(isInternalDomainUpstream) = %d, want 1", got) } if got := um.countDownExcept(nil); got != 3 { diff --git a/config.go b/config.go index 8bcc93e..4ac52ee 100644 --- a/config.go +++ b/config.go @@ -269,8 +269,13 @@ type UpstreamConfig struct { Endpoint string `mapstructure:"endpoint" toml:"endpoint,omitempty"` BootstrapIP string `mapstructure:"bootstrap_ip" toml:"bootstrap_ip,omitempty"` Domain string `mapstructure:"-" toml:"-"` - IPStack string `mapstructure:"ip_stack" toml:"ip_stack,omitempty" validate:"ipstack"` - Timeout int `mapstructure:"timeout" toml:"timeout,omitempty" validate:"gte=0"` + // InternalDomain is set only by ctrld, on the upstreams it generates for + // organization Internal Domains, to the mode they serve. It is never read + // from or written to a configuration file, so an upstream a configuration + // defines is never mistaken for a generated one, whatever it is named. + InternalDomain string `mapstructure:"-" toml:"-"` + IPStack string `mapstructure:"ip_stack" toml:"ip_stack,omitempty" validate:"ipstack"` + Timeout int `mapstructure:"timeout" toml:"timeout,omitempty" validate:"gte=0"` // The caller should not access this field directly. // Use UpstreamSendClientInfo instead. SendClientInfo *bool `mapstructure:"send_client_info" toml:"send_client_info,omitempty"` diff --git a/docs/internal-domains.md b/docs/internal-domains.md index 1630b8d..72e70d6 100644 --- a/docs/internal-domains.md +++ b/docs/internal-domains.md @@ -15,7 +15,8 @@ The managed resolver configuration carries the list in `resolver.split_dns`: "resolver": { "split_dns": [ { "domain": "corp.example.com", "mode": "resolvers", "resolvers": ["10.0.0.53", "10.0.0.54"] }, - { "domain": "office.example.com", "mode": "os", "resolvers": [] } + { "domain": "office.example.com", "mode": "os", "resolvers": [] }, + { "domain": "vault.example.com", "mode": "resolvers_only", "resolvers": ["10.0.0.55"] } ] } } @@ -25,13 +26,15 @@ The managed resolver configuration carries the list in `resolver.split_dns`: | `mode` | Behavior | |---|---| -| `os` | The query goes to `upstream.os`, the endpoint's current system resolver, and follows DHCP and VPN resolver changes. Any addresses still present in `resolvers` are **not** read, so switching a domain back to the OS resolver cannot be undone by leftovers from a previous selection. | -| `resolvers` | The query goes only to the addresses in `resolvers`, tried in the order given. An empty or entirely unusable list does not become OS resolution: the entry is dropped instead. | -| absent | A deployment that predates the field. The mode is inferred from `resolvers`, which is what the field encodes. | -| anything else | Dropped. Routing a private domain by guess is the disclosure this feature exists to prevent. | +| `os` | **Network default.** The query goes to `upstream.os`, the endpoint's current system resolver, and follows DHCP and VPN resolver changes. Any addresses still present in `resolvers` are **not** read, so switching a domain back to the OS resolver cannot be undone by leftovers from a previous selection. | +| `resolvers` | **Explicit resolver with network fallback** (the default explicit selection). The query goes to the addresses in `resolvers` first, tried in the order given; if none of them resolves it, it falls back to the network resolvers (see [Network fallback](#network-fallback)). | +| `resolvers_only` | **Explicit resolver only.** The query goes only to the addresses in `resolvers`, and their answer or failure is final. Not offered by the dashboard yet. | +| absent | A deployment that predates the field. Treated as **explicit resolver with network fallback**, the same as `resolvers`. If the entry has no resolvers there is nothing explicit to try, so it uses the network default (`os`). | +| anything else | Fails closed. With resolvers, treated as `resolvers_only`: only the resolvers the administrator chose, so a wire value this build does not know (a renamed strict mode, a future mode) never sends the domain to the Control D upstream or the network. Without resolvers there is nothing to fail closed to, so the entry is dropped. | -A dropped entry leaves its domain on its normal path and is reported in the -`skipped` count. +In both explicit modes an empty or entirely unusable list does not become OS +resolution: the entry is dropped instead. A dropped entry leaves its domain on +its normal path and is reported in the `skipped` count. Addresses may be IPv4 or IPv6, bare or with a port; a bare address uses port 53. An address that is not a valid IP is dropped from the list. @@ -48,8 +51,15 @@ and routing a private domain to the wrong place is worse than not routing it. ## Generated configuration Each accepted domain contributes two policy rules, `` and `*.`, -which is how ctrld matches a suffix and all of its subdomains. Explicit mode -also generates one upstream per resolver address, named `internal_`: +which is how ctrld matches a suffix and all of its subdomains. The explicit +modes also generate one upstream per resolver address, named `internal_`. +Each one carries an internal marker recording its mode, which is what tells +`proxy()` that an upstream is a generated Internal Domain resolver and whether +the network fallback may run. The marker is never read from or written to a +configuration file, so the key name is only a name: an upstream a local +`ctrld.toml` or a custom configuration defines as `internal_foo`, or even as an +exact copy of a generated one, is an ordinary upstream. The generated +configuration looks like this: ```toml [upstream.internal_0] @@ -97,27 +107,75 @@ exists for a source is never overwritten. From highest precedence to lowest: **VPN DNS auto-detection** is evaluated inside `proxy()` in DNS-intercept mode, after policy matching and independently of it, so it needs its own rule: -- An **explicit-resolver** Internal Domain skips VPN routing. The administrator - named the resolvers for that suffix; VPN suffixes are discovered from the OS, - and letting a discovered route override a named one is the leak this feature - exists to prevent. +- An **explicit-resolver** Internal Domain skips VPN routing ahead of its + configured resolvers. The administrator named the resolvers for that suffix; + VPN suffixes are discovered from the OS, and a discovered route must not + override a named one. In `resolvers` mode the VPN DNS servers are asked + afterwards, as part of the [network fallback](#network-fallback). - An **OS-resolver** Internal Domain does not skip it, and behaves exactly like a Magic Folder exclude. It asks for the endpoint's default resolution, which under intercept mode includes the VPN's own resolver. Every domain that is not an Internal Domain keeps its current VPN routing. +## Network fallback + +In `resolvers` mode the configured resolvers are always asked first, even while +the DNS-intercept recovery bypass is active and ahead of VPN DNS split routing. +A configured resolver that times out, is unreachable, or answers SERVFAIL, +NXDOMAIN, REFUSED or NOTIMP hands the query to the next configured resolver, +and once all of them have, to the network resolvers: + +1. VPN DNS servers whose domains match the query, then VPN DNS servers that + advertise no domains (DNS-intercept mode only, where ctrld tracks them). +2. The OS resolver's **LAN** nameservers: private, loopback, link-local and + CGNAT addresses the active network provides. + +A configured resolver that the upstream monitor has marked down (after 10 +seconds of continuous failures, or 50 failures) is skipped rather than waited +on, so an endpoint away from the organization network reaches the fallback +without paying the resolver's timeout, about 2 seconds per resolver, on every +query. Queries in the first seconds off-site, before the resolver is marked +down, still pay it. A generated resolver comes back up only by answering, so +skipping one also re-checks it in the background, at most once every 30 +seconds, with the query that was skipped. Once it answers it is used again. +`resolvers_only` mode never skips: with no fallback to go to, skipping would +only turn the resolver's answer into SERVFAIL. + +The first NOERROR answer wins. NXDOMAIN triggers the fallback because the +configured resolver may be reachable but not know a name that the local +network's resolver does, for example on another subnet. REFUSED and NOTIMP +trigger it because they mean the resolver will not serve this endpoint, which +is how an organization resolver with an access list answers an endpoint off +the organization network. An empty NOERROR (the name exists but has no record +of the requested type) is final, and so is any other rcode. + +The OS step never sends the name to public DNS. It is a LAN-only query: a +public nameserver in the OS pool is not asked at all, whether DHCP supplied it +(a router handing out `192.168.1.1` plus `8.8.8.8`, a Wi-Fi handing out only +public resolvers) or it is ctrld's own public fallback (`76.76.2.0`). With no +LAN nameserver the OS step sends nothing. The general OS-resolver catch-all, +which does reach public nameservers, is never used either. The VPN step uses +the servers the VPN pushes as they are, including a public address a VPN +pushes as its DNS server. + +While Windows is serving retained VPN DNS state, a VPN transport failure stops +the fallback before the OS resolver, as it does for VPN split routing. + +When no resolver resolves the name, the query ends with the best negative +answer it got (NXDOMAIN before any other), or SERVFAIL if it got none. + ## Unreachable explicit resolvers -When every resolver for an explicit-mode domain fails, the query returns -SERVFAIL. It is deliberately not retried against the OS resolver, and it does -not trigger the endpoint recovery flow: +In `resolvers_only` mode, when every configured resolver fails the query returns +SERVFAIL, and an NXDOMAIN or SERVFAIL answer from them is returned as it is. +Either way the query is not retried anywhere else. Administrators who need the +configured resolvers' answer enforced select this mode. -- Sending a private name to a resolver the administrator did not select is the - leak this feature exists to prevent. -- The recovery flow exists for the loss of general DNS. One unavailable internal - server is not that, and letting it start recovery would reset endpoint DNS - settings for every other domain. +In neither explicit mode does a failure trigger the endpoint recovery flow. That +flow exists for the loss of general DNS. One unavailable internal server is not +that, and letting it start recovery would reset endpoint DNS settings for every +other domain. OS-resolver mode has no such restriction: it is the system resolver, so it follows DHCP and VPN changes and shares their failure handling. @@ -133,9 +191,10 @@ second upstream. Eligibility is recomputed on every setup and reload, so adding or removing Internal Domains, or gaining a genuinely custom second upstream, is reflected instead of latching on the first value seen. -Discounting is by upstream shape, not by key name alone: an endpoint custom -configuration replaces the generated config outright and could name an upstream -`internal_0`, which must not buy eligibility. The uninstall itself remains +Discounting is by the internal marker, not by key name or any other +configurable field: an endpoint custom configuration replaces the generated +config outright and could define an upstream `internal_0` that copies a +generated one, which must not buy eligibility. The uninstall itself remains gated on the API confirming the device is gone. ## Refresh @@ -168,6 +227,7 @@ the count. That covers every consumer, not just query handling: of recovery started for another reason. Info and warning logs report counts only — domains applied, OS-resolver and -explicit-resolver totals, generated resolvers, skipped and preempted entries. +explicit-resolver totals (with `explicit_resolver_only` for `resolvers_only`), +generated resolvers, skipped and preempted entries. Domain names and resolver addresses are organization-private and appear only at debug level. diff --git a/internal/controld/config.go b/internal/controld/config.go index adf83a7..2d8b7c3 100644 --- a/internal/controld/config.go +++ b/internal/controld/config.go @@ -64,25 +64,31 @@ type ResolverConfig struct { } // Internal Domain resolution modes. Mode is what the administrator selected; -// Resolvers is only meaningful under SplitDNSModeResolvers. +// Resolvers is only meaningful under SplitDNSModeResolvers and +// SplitDNSModeResolversOnly. const ( // SplitDNSModeOS resolves the domain through the endpoint's own OS/default // resolver, so it follows DHCP and VPN resolver changes. SplitDNSModeOS = "os" // SplitDNSModeResolvers resolves the domain through the addresses in - // Resolvers and nothing else. + // Resolvers first. When none of them answers, or they answer SERVFAIL or + // NXDOMAIN, the query falls back to the resolvers of the endpoint's active + // network and VPN. This is the default explicit selection. SplitDNSModeResolvers = "resolvers" + // SplitDNSModeResolversOnly resolves the domain through the addresses in + // Resolvers and nothing else: their answer, or their failure, is final. + SplitDNSModeResolversOnly = "resolvers_only" ) // SplitDNS is one organization Internal Domain: a domain suffix, the resolution // mode the organization administrator selected for it, and the resolver // addresses that mode may need. // -// Mode decides the routing. Resolvers is read only under -// SplitDNSModeResolvers, so addresses left behind by an earlier selection -// cannot resurrect themselves once the administrator switches back to the OS -// resolver. An empty Mode is a deployment that predates the field; the caller -// infers the mode from Resolvers in that case. +// Mode decides the routing. Resolvers is read only under the explicit modes, +// so addresses left behind by an earlier selection cannot resurrect themselves +// once the administrator switches back to the OS resolver. An empty Mode is a +// deployment that predates the field; the caller infers the mode from +// Resolvers in that case. type SplitDNS struct { Domain string `json:"domain"` Mode string `json:"mode"` diff --git a/internal/controld/split_dns_test.go b/internal/controld/split_dns_test.go index 626287a..7c2e1e7 100644 --- a/internal/controld/split_dns_test.go +++ b/internal/controld/split_dns_test.go @@ -112,6 +112,11 @@ func TestResolverConfigParsesSplitDNSMode(t *testing.T) { "domain": "office.example.com", "mode": "os", "resolvers": [] + }, + { + "domain": "vault.example.com", + "mode": "resolvers_only", + "resolvers": ["10.0.0.55"] } ] } @@ -122,7 +127,7 @@ func TestResolverConfigParsesSplitDNSMode(t *testing.T) { t.Fatal(err) } got := resp.Body.Resolver.SplitDNS - if len(got) != 2 { + if len(got) != 3 { t.Fatalf("split_dns = %+v", got) } if got[0].Domain != "corp.example.com" || got[0].Mode != SplitDNSModeResolvers { @@ -137,6 +142,9 @@ func TestResolverConfigParsesSplitDNSMode(t *testing.T) { if len(got[1].Resolvers) != 0 { t.Errorf("[1].Resolvers = %v, want empty", got[1].Resolvers) } + if got[2].Mode != SplitDNSModeResolversOnly || len(got[2].Resolvers) != 1 { + t.Errorf("[2] = %+v", got[2]) + } } // A response without the field must still decode, so a deployment that predates diff --git a/resolver.go b/resolver.go index 38e48e7..fb0f9e5 100644 --- a/resolver.go +++ b/resolver.go @@ -139,6 +139,29 @@ func LanQueryCtx(ctx context.Context) context.Context { return context.WithValue(ctx, LanQueryCtxKey{}, true) } +// LanOnlyQueryCtxKey is the context.Context key that limits an OS resolver +// query to its LAN nameservers. +type LanOnlyQueryCtxKey struct{} + +// LanOnlyQueryCtx returns a context.Context with LanOnlyQueryCtxKey set. +// +// An OS resolver query made with it goes only to LAN-classified nameservers: +// private, loopback, link-local and CGNAT addresses. Every public nameserver +// is left out, the ones DHCP or a VPN supplied as well as ctrld's own public +// fallback, and with no LAN nameserver the query fails without being sent. It +// is for names that must not reach public DNS, such as an organization's +// Internal Domains. LanQueryCtx, by contrast, only drops ctrld's own public +// fallback. +func LanOnlyQueryCtx(ctx context.Context) context.Context { + return context.WithValue(ctx, LanOnlyQueryCtxKey{}, true) +} + +// isLanOnlyQuery reports whether ctx was made by LanOnlyQueryCtx. +func isLanOnlyQuery(ctx context.Context) bool { + lanOnly, _ := ctx.Value(LanOnlyQueryCtxKey{}).(bool) + return lanOnly +} + // defaultNameservers is like nameservers with each element formed "ip:53". func defaultNameservers() []string { ns := nameservers() @@ -590,6 +613,12 @@ func (o *osResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error // the key so subnet-specific answers are neither coalesced nor hot-cached // across different subnets (RFC 7871 §7.3). key := fmt.Sprintf("%s:%d:%s", domain, qtype, dnscache.CanonicalECS(msg)) + // A LAN-only query must not join, or be answered from the hot cache of, an + // ordinary query for the same name: that answer may have come from a public + // nameserver. + if isLanOnlyQuery(ctx) { + key += ":lan-only" + } // Checking the cache first. if val, ok := o.cache.Load(key); ok { @@ -642,6 +671,10 @@ func (o *osResolver) resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error if p := o.lanServers.Load(); p != nil { nss = append(nss, (*p)...) } + // A LAN-only query is never sent to a public nameserver. + if isLanOnlyQuery(ctx) { + publicServers = nil + } numServers := len(nss) + len(publicServers) // If this is a LAN query, skip public DNS. diff --git a/resolver_test.go b/resolver_test.go index 674bef3..3237403 100644 --- a/resolver_test.go +++ b/resolver_test.go @@ -1092,3 +1092,77 @@ func TestInitializeOsResolverReportsTheUnspecifiedReason(t *testing.T) { wantLogField(t, changes[0], "reason", "unspecified") wantLogStrings(t, changes[0], "after", []string{"192.0.2.10"}) } + +// A LAN-only query must reach the LAN nameservers and nothing else. A public +// nameserver in the same pool, as DHCP hands out next to the router, is not +// asked at all, so the name is never sent to it. +func Test_osResolver_LanOnlyQuerySkipsPublicNameservers(t *testing.T) { + const lan, public = "10.0.0.53:53", "192.0.2.53:53" + resolver := newResolverWithNameserver([]string{lan, public, controldPublicDnsWithPort}) + + var mu sync.Mutex + var asked []string + resolver.exchangeDNS = func(_ context.Context, msg *dns.Msg, server string, _ net.IP) (*dns.Msg, time.Duration, error) { + mu.Lock() + asked = append(asked, server) + mu.Unlock() + answer := new(dns.Msg) + answer.SetRcode(msg, dns.RcodeNameError) + return answer, time.Millisecond, nil + } + + msg := new(dns.Msg) + msg.SetQuestion("host.corp.example.", dns.TypeA) + answer, err := resolver.Resolve(LanOnlyQueryCtx(context.Background()), msg) + if err != nil { + t.Fatal(err) + } + if answer.Rcode != dns.RcodeNameError { + t.Errorf("rcode = %s, want the LAN nameserver's NXDOMAIN", dns.RcodeToString[answer.Rcode]) + } + if len(asked) != 1 || asked[0] != lan { + t.Fatalf("nameservers asked = %v, want only %s", asked, lan) + } +} + +// With no LAN nameserver at all, a LAN-only query fails without sending +// anything, rather than falling through to the public ones. +func Test_osResolver_LanOnlyQueryWithoutLanNameserverSendsNothing(t *testing.T) { + resolver := newResolverWithNameserver([]string{"192.0.2.53:53", controldPublicDnsWithPort}) + calls := 0 + resolver.exchangeDNS = func(_ context.Context, msg *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + calls++ + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + msg := new(dns.Msg) + msg.SetQuestion("host.corp.example.", dns.TypeA) + if _, err := resolver.Resolve(LanOnlyQueryCtx(context.Background()), msg); err == nil { + t.Fatal("a LAN-only query with no LAN nameserver returned an answer") + } + if calls != 0 { + t.Fatalf("exchange calls = %d, want none", calls) + } +} + +// The hot cache is keyed separately for LAN-only queries: an answer an +// ordinary query got from a public nameserver must not serve a LAN-only one. +func Test_osResolver_LanOnlyQueryDoesNotShareOrdinaryHotCache(t *testing.T) { + resolver := newResolverWithNameserver([]string{"192.0.2.53:53"}) + resolver.exchangeDNS = func(_ context.Context, msg *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + msg := new(dns.Msg) + msg.SetQuestion("host.corp.example.", dns.TypeA) + if _, err := resolver.Resolve(context.Background(), msg); err != nil { + t.Fatalf("ordinary query: %v", err) + } + if _, err := resolver.Resolve(LanOnlyQueryCtx(context.Background()), msg); err == nil { + t.Fatal("a LAN-only query was served the ordinary query's public answer from the hot cache") + } +}