mirror of
https://github.com/Control-D-Inc/ctrld.git
synced 2026-10-12 01:58:46 +02:00
Dev Scribe
b0408e4d1c
Internal Domains now support three modes. "os" is unchanged. "resolvers" becomes explicit resolver with network fallback: the configured resolvers are tried first, and a timeout, unreachable resolver, SERVFAIL or NXDOMAIN hands the query to matching VPN DNS servers, then domain-less VPN DNS servers, then the OS resolver's LAN nameservers through the new LanOnlyQueryCtx. "resolvers_only" keeps the previous strict behavior. Down fallback-mode resolvers are skipped and re-checked in the background at most every 30s. Generated upstreams are identified by the UpstreamConfig.InternalDomain marker instead of the internal_ key prefix. Master adaptations: the fallback state lives on proxyRequest and is applied in tryUpstreams, shouldContinueWithNextUpstream and handleAllUpstreamsFailure instead of v1.0's monolithic proxy loop. Resolver construction uses master's context-aware NewResolver, and the background re-check marks its context private so transport errors that name the resolver stay at debug. Tests use master's zap JSON capture and VPN manager logger.
292 lines
9.3 KiB
Go
292 lines
9.3 KiB
Go
package cli
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/Control-D-Inc/ctrld"
|
|
)
|
|
|
|
// upstreamStateMessage is the message of every upstream transition event.
|
|
const upstreamStateMessage = "Upstream state changed"
|
|
|
|
// downMonitor returns a monitor on a clock the test owns, with the upstream down.
|
|
func downMonitor(t *testing.T, upstream string) (*upstreamMonitor, *syncBuffer, *time.Time) {
|
|
t.Helper()
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load())
|
|
now := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)
|
|
um.now = func() time.Time { return now }
|
|
for i := 0; i < maxFailureRequest; i++ {
|
|
um.increaseFailureCount(upstream)
|
|
}
|
|
if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 1 {
|
|
t.Fatalf("state events after the failures: got %d, want 1", got)
|
|
}
|
|
return um, logs, &now
|
|
}
|
|
|
|
func Test_upstreamMonitorLogsOneDownEvent(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load())
|
|
|
|
for i := 0; i < 633; i++ {
|
|
um.increaseFailureCount(upstream)
|
|
}
|
|
|
|
events := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(events) != 1 {
|
|
t.Fatalf("state events: got %d, want 1", len(events))
|
|
}
|
|
wantField(t, events[0], "level", "warn")
|
|
wantField(t, events[0], "journal", true)
|
|
wantField(t, events[0], "upstream", upstream)
|
|
wantField(t, events[0], "state", "down")
|
|
wantField(t, events[0], "reason", "immediate")
|
|
wantField(t, events[0], "failure_count", float64(maxFailureRequest))
|
|
}
|
|
|
|
func Test_upstreamMonitorTimerLogsDownEvent(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load())
|
|
var armed []func()
|
|
um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) }
|
|
|
|
um.increaseFailureCount(upstream)
|
|
if len(armed) != 1 {
|
|
t.Fatalf("armed timers: got %d, want 1", len(armed))
|
|
}
|
|
armed[0]()
|
|
|
|
events := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(events) != 1 {
|
|
t.Fatalf("state events: got %d, want 1", len(events))
|
|
}
|
|
wantField(t, events[0], "state", "down")
|
|
wantField(t, events[0], "reason", "timer")
|
|
wantField(t, events[0], "failure_count", float64(1))
|
|
}
|
|
|
|
func Test_upstreamMonitorResetLogsOneUpEvent(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
um, logs, now := downMonitor(t, upstream)
|
|
|
|
*now = now.Add(1500 * time.Millisecond)
|
|
um.reset(upstream)
|
|
|
|
events := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(events) != 2 {
|
|
t.Fatalf("state events: got %d, want 2", len(events))
|
|
}
|
|
up := events[1]
|
|
wantField(t, up, "level", "info")
|
|
wantField(t, up, "journal", true)
|
|
wantField(t, up, "upstream", upstream)
|
|
wantField(t, up, "state", "up")
|
|
wantField(t, up, "reason", "recovered")
|
|
wantField(t, up, "down_for_ms", float64(1500))
|
|
wantField(t, up, "failure_count", float64(maxFailureRequest))
|
|
}
|
|
|
|
func Test_upstreamMonitorRetireLogsTheOpenOutage(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
um, logs, now := downMonitor(t, upstream)
|
|
|
|
*now = now.Add(3 * time.Second)
|
|
um.retire()
|
|
|
|
events := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(events) != 2 {
|
|
t.Fatalf("state events: got %d, want 2", len(events))
|
|
}
|
|
up := events[1]
|
|
wantField(t, up, "upstream", upstream)
|
|
wantField(t, up, "state", "up")
|
|
wantField(t, up, "reason", "reload")
|
|
wantField(t, up, "down_for_ms", float64(3000))
|
|
|
|
um.retire()
|
|
|
|
if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 2 {
|
|
t.Fatalf("state events after the second retire: got %d, want 2", got)
|
|
}
|
|
}
|
|
|
|
func Test_upstreamMonitorNoteSuccessLogsOneUpEvent(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
um, logs, now := downMonitor(t, upstream)
|
|
|
|
*now = now.Add(2 * time.Second)
|
|
um.noteSuccess(upstream)
|
|
|
|
events := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(events) != 2 {
|
|
t.Fatalf("state events: got %d, want 2", len(events))
|
|
}
|
|
up := events[1]
|
|
wantField(t, up, "state", "up")
|
|
wantField(t, up, "reason", "recovered")
|
|
wantField(t, up, "down_for_ms", float64(2000))
|
|
wantField(t, up, "failure_count", float64(maxFailureRequest))
|
|
if got := um.failureReq[upstream]; got != 0 {
|
|
t.Fatalf("failure count after the answer: got %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
func Test_upstreamMonitorNoteSuccessOnUpUpstreamIsQuiet(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load())
|
|
|
|
um.noteSuccess(upstream)
|
|
um.noteSuccess(upstream)
|
|
|
|
if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 0 {
|
|
t.Fatalf("state events: got %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
// Test_upstreamMonitorRetiredTimerStaysSilent fires the down timer that a
|
|
// failure armed before the reload retired the monitor. A retired monitor
|
|
// owns no outage, so the timer must add no event.
|
|
func Test_upstreamMonitorRetiredTimerStaysSilent(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load())
|
|
var armed []func()
|
|
um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) }
|
|
|
|
um.increaseFailureCount(upstream)
|
|
um.retire()
|
|
for _, fire := range armed {
|
|
fire()
|
|
}
|
|
|
|
if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 0 {
|
|
t.Fatalf("state events from the retired monitor: got %d, want 0: %s", got, logs.String())
|
|
}
|
|
if um.isDown(upstream) {
|
|
t.Fatal("the retired monitor marked the upstream down")
|
|
}
|
|
}
|
|
|
|
// Test_upstreamMonitorRetiredAfterImmediateDownStaysSilent retires a monitor
|
|
// that already reported an outage. The armed timer and a late failure must
|
|
// add no event after the closing up event.
|
|
func Test_upstreamMonitorRetiredAfterImmediateDownStaysSilent(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load())
|
|
var armed []func()
|
|
um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) }
|
|
|
|
for i := 0; i < maxFailureRequest; i++ {
|
|
um.increaseFailureCount(upstream)
|
|
}
|
|
um.retire()
|
|
before := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(before) != 2 {
|
|
t.Fatalf("state events after the retire: got %d, want down and up", len(before))
|
|
}
|
|
for _, fire := range armed {
|
|
fire()
|
|
}
|
|
um.increaseFailureCount(upstream)
|
|
|
|
if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 2 {
|
|
t.Fatalf("state events after the retired timer fired: got %d, want 2: %s", got, logs.String())
|
|
}
|
|
if um.isDown(upstream) {
|
|
t.Fatal("the retired monitor marked the upstream down again")
|
|
}
|
|
}
|
|
|
|
// Test_upstreamMonitorKeepsACustomNameOutOfTheJournal marks an upstream with
|
|
// an operator-chosen key down. The key can hold a token, so the event names
|
|
// the upstream by a bounded name.
|
|
func Test_upstreamMonitorKeepsACustomNameOutOfTheJournal(t *testing.T) {
|
|
const upstream = upstreamPrefix + "dns.controld.com/org-v1-SECRET0123"
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load())
|
|
|
|
for i := 0; i < maxFailureRequest; i++ {
|
|
um.increaseFailureCount(upstream)
|
|
}
|
|
um.noteSuccess(upstream)
|
|
|
|
events := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(events) != 2 {
|
|
t.Fatalf("state events: got %d, want down and up", len(events))
|
|
}
|
|
for _, event := range events {
|
|
wantField(t, event, "upstream", upstreamPrefix+"custom")
|
|
}
|
|
for _, generated := range []string{upstreamOS, upstreamPrefix + "0", upstreamPrefix + "internal_3"} {
|
|
if got := journalUpstreamName(generated); got != generated {
|
|
t.Fatalf("journalUpstreamName(%q) = %q, want the name itself", generated, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func Test_upstreamMonitorCountDown(t *testing.T) {
|
|
captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load())
|
|
if got := um.countDown(); got != 0 {
|
|
t.Fatalf("countDown on a new monitor = %d, want 0", got)
|
|
}
|
|
um.mu.Lock()
|
|
um.markDown(upstreamPrefix+"0", 3, "immediate")
|
|
um.markDown(upstreamPrefix+"1", 3, "immediate")
|
|
um.mu.Unlock()
|
|
if got := um.countDown(); got != 2 {
|
|
t.Fatalf("countDown with two down upstreams = %d, want 2", got)
|
|
}
|
|
}
|
|
|
|
func Test_upstreamMonitorDownFor(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
um, _, now := downMonitor(t, upstream)
|
|
|
|
*now = now.Add(2500 * time.Millisecond)
|
|
|
|
if got := um.downFor(upstream); got != 2500*time.Millisecond {
|
|
t.Fatalf("downFor a down upstream = %s, want 2.5s", got)
|
|
}
|
|
if got := um.downFor(upstreamPrefix + "9"); got != 0 {
|
|
t.Fatalf("downFor an unknown upstream = %s, want 0", got)
|
|
}
|
|
|
|
um.reset(upstream)
|
|
|
|
if got := um.downFor(upstream); got != 0 {
|
|
t.Fatalf("downFor an upstream that is up = %s, want 0", got)
|
|
}
|
|
}
|
|
|
|
func Test_upstreamMonitorCountDownExcept(t *testing.T) {
|
|
captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load())
|
|
um.mu.Lock()
|
|
um.markDown(upstreamPrefix+"0", 3, "immediate")
|
|
um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"1", 3, "immediate")
|
|
um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"2", 3, "immediate")
|
|
um.mu.Unlock()
|
|
|
|
if got := um.countDown(); got != 3 {
|
|
t.Fatalf("countDown = %d, want 3", got)
|
|
}
|
|
p := internalDomainUpstreamsProg(map[string]string{
|
|
"0": "",
|
|
internalDomainUpstreamPrefix + "1": internalDomainUpstreamFallback,
|
|
internalDomainUpstreamPrefix + "2": internalDomainUpstreamOnly,
|
|
})
|
|
if got := um.countDownExcept(p.isInternalDomainUpstream); got != 1 {
|
|
t.Fatalf("countDownExcept(isInternalDomainUpstream) = %d, want 1", got)
|
|
}
|
|
if got := um.countDownExcept(nil); got != 3 {
|
|
t.Fatalf("countDownExcept(nil) = %d, want 3", got)
|
|
}
|
|
}
|