mirror of
https://github.com/Control-D-Inc/ctrld.git
synced 2026-10-12 01:58:46 +02:00
Anthony Wong
8ae09ac98e
The journal records the network state of the host and each change of that state. The support team reads one file to see the interfaces, the routes, the resolvers, and the recoveries around an incident. The same file shows the health of the query path. A change that touches only AirDrop or virtual interfaces runs no DNS work.
287 lines
9.0 KiB
Go
287 lines
9.0 KiB
Go
package cli
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/Control-D-Inc/ctrld"
|
|
)
|
|
|
|
// upstreamStateMessage is the message of every upstream transition event.
|
|
const upstreamStateMessage = "Upstream state changed"
|
|
|
|
// downMonitor returns a monitor on a clock the test owns, with the upstream down.
|
|
func downMonitor(t *testing.T, upstream string) (*upstreamMonitor, *syncBuffer, *time.Time) {
|
|
t.Helper()
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{})
|
|
now := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)
|
|
um.now = func() time.Time { return now }
|
|
for i := 0; i < maxFailureRequest; i++ {
|
|
um.increaseFailureCount(upstream)
|
|
}
|
|
if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 1 {
|
|
t.Fatalf("state events after the failures: got %d, want 1", got)
|
|
}
|
|
return um, logs, &now
|
|
}
|
|
|
|
func Test_upstreamMonitorLogsOneDownEvent(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{})
|
|
|
|
for i := 0; i < 633; i++ {
|
|
um.increaseFailureCount(upstream)
|
|
}
|
|
|
|
events := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(events) != 1 {
|
|
t.Fatalf("state events: got %d, want 1", len(events))
|
|
}
|
|
wantField(t, events[0], "level", "warn")
|
|
wantField(t, events[0], "journal", true)
|
|
wantField(t, events[0], "upstream", upstream)
|
|
wantField(t, events[0], "state", "down")
|
|
wantField(t, events[0], "reason", "immediate")
|
|
wantField(t, events[0], "failure_count", float64(maxFailureRequest))
|
|
}
|
|
|
|
func Test_upstreamMonitorTimerLogsDownEvent(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{})
|
|
var armed []func()
|
|
um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) }
|
|
|
|
um.increaseFailureCount(upstream)
|
|
if len(armed) != 1 {
|
|
t.Fatalf("armed timers: got %d, want 1", len(armed))
|
|
}
|
|
armed[0]()
|
|
|
|
events := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(events) != 1 {
|
|
t.Fatalf("state events: got %d, want 1", len(events))
|
|
}
|
|
wantField(t, events[0], "state", "down")
|
|
wantField(t, events[0], "reason", "timer")
|
|
wantField(t, events[0], "failure_count", float64(1))
|
|
}
|
|
|
|
func Test_upstreamMonitorResetLogsOneUpEvent(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
um, logs, now := downMonitor(t, upstream)
|
|
|
|
*now = now.Add(1500 * time.Millisecond)
|
|
um.reset(upstream)
|
|
|
|
events := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(events) != 2 {
|
|
t.Fatalf("state events: got %d, want 2", len(events))
|
|
}
|
|
up := events[1]
|
|
wantField(t, up, "level", "info")
|
|
wantField(t, up, "journal", true)
|
|
wantField(t, up, "upstream", upstream)
|
|
wantField(t, up, "state", "up")
|
|
wantField(t, up, "reason", "recovered")
|
|
wantField(t, up, "down_for_ms", float64(1500))
|
|
wantField(t, up, "failure_count", float64(maxFailureRequest))
|
|
}
|
|
|
|
func Test_upstreamMonitorRetireLogsTheOpenOutage(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
um, logs, now := downMonitor(t, upstream)
|
|
|
|
*now = now.Add(3 * time.Second)
|
|
um.retire()
|
|
|
|
events := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(events) != 2 {
|
|
t.Fatalf("state events: got %d, want 2", len(events))
|
|
}
|
|
up := events[1]
|
|
wantField(t, up, "upstream", upstream)
|
|
wantField(t, up, "state", "up")
|
|
wantField(t, up, "reason", "reload")
|
|
wantField(t, up, "down_for_ms", float64(3000))
|
|
|
|
um.retire()
|
|
|
|
if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 2 {
|
|
t.Fatalf("state events after the second retire: got %d, want 2", got)
|
|
}
|
|
}
|
|
|
|
func Test_upstreamMonitorNoteSuccessLogsOneUpEvent(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
um, logs, now := downMonitor(t, upstream)
|
|
|
|
*now = now.Add(2 * time.Second)
|
|
um.noteSuccess(upstream)
|
|
|
|
events := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(events) != 2 {
|
|
t.Fatalf("state events: got %d, want 2", len(events))
|
|
}
|
|
up := events[1]
|
|
wantField(t, up, "state", "up")
|
|
wantField(t, up, "reason", "recovered")
|
|
wantField(t, up, "down_for_ms", float64(2000))
|
|
wantField(t, up, "failure_count", float64(maxFailureRequest))
|
|
if got := um.failureReq[upstream]; got != 0 {
|
|
t.Fatalf("failure count after the answer: got %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
func Test_upstreamMonitorNoteSuccessOnUpUpstreamIsQuiet(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{})
|
|
|
|
um.noteSuccess(upstream)
|
|
um.noteSuccess(upstream)
|
|
|
|
if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 0 {
|
|
t.Fatalf("state events: got %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
// Test_upstreamMonitorRetiredTimerStaysSilent fires the down timer that a
|
|
// failure armed before the reload retired the monitor. A retired monitor
|
|
// owns no outage, so the timer must add no event.
|
|
func Test_upstreamMonitorRetiredTimerStaysSilent(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{})
|
|
var armed []func()
|
|
um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) }
|
|
|
|
um.increaseFailureCount(upstream)
|
|
um.retire()
|
|
for _, fire := range armed {
|
|
fire()
|
|
}
|
|
|
|
if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 0 {
|
|
t.Fatalf("state events from the retired monitor: got %d, want 0: %s", got, logs.String())
|
|
}
|
|
if um.isDown(upstream) {
|
|
t.Fatal("the retired monitor marked the upstream down")
|
|
}
|
|
}
|
|
|
|
// Test_upstreamMonitorRetiredAfterImmediateDownStaysSilent retires a monitor
|
|
// that already reported an outage. The armed timer and a late failure must
|
|
// add no event after the closing up event.
|
|
func Test_upstreamMonitorRetiredAfterImmediateDownStaysSilent(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{})
|
|
var armed []func()
|
|
um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) }
|
|
|
|
for i := 0; i < maxFailureRequest; i++ {
|
|
um.increaseFailureCount(upstream)
|
|
}
|
|
um.retire()
|
|
before := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(before) != 2 {
|
|
t.Fatalf("state events after the retire: got %d, want down and up", len(before))
|
|
}
|
|
for _, fire := range armed {
|
|
fire()
|
|
}
|
|
um.increaseFailureCount(upstream)
|
|
|
|
if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 2 {
|
|
t.Fatalf("state events after the retired timer fired: got %d, want 2: %s", got, logs.String())
|
|
}
|
|
if um.isDown(upstream) {
|
|
t.Fatal("the retired monitor marked the upstream down again")
|
|
}
|
|
}
|
|
|
|
// Test_upstreamMonitorKeepsACustomNameOutOfTheJournal marks an upstream with
|
|
// an operator-chosen key down. The key can hold a token, so the event names
|
|
// the upstream by a bounded name.
|
|
func Test_upstreamMonitorKeepsACustomNameOutOfTheJournal(t *testing.T) {
|
|
const upstream = upstreamPrefix + "dns.controld.com/org-v1-SECRET0123"
|
|
logs := captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{})
|
|
|
|
for i := 0; i < maxFailureRequest; i++ {
|
|
um.increaseFailureCount(upstream)
|
|
}
|
|
um.noteSuccess(upstream)
|
|
|
|
events := jsonLogEvents(t, logs, upstreamStateMessage)
|
|
if len(events) != 2 {
|
|
t.Fatalf("state events: got %d, want down and up", len(events))
|
|
}
|
|
for _, event := range events {
|
|
wantField(t, event, "upstream", upstreamPrefix+"custom")
|
|
}
|
|
for _, generated := range []string{upstreamOS, upstreamPrefix + "0", upstreamPrefix + "internal_3"} {
|
|
if got := journalUpstreamName(generated); got != generated {
|
|
t.Fatalf("journalUpstreamName(%q) = %q, want the name itself", generated, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func Test_upstreamMonitorCountDown(t *testing.T) {
|
|
captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{})
|
|
if got := um.countDown(); got != 0 {
|
|
t.Fatalf("countDown on a new monitor = %d, want 0", got)
|
|
}
|
|
um.mu.Lock()
|
|
um.markDown(upstreamPrefix+"0", 3, "immediate")
|
|
um.markDown(upstreamPrefix+"1", 3, "immediate")
|
|
um.mu.Unlock()
|
|
if got := um.countDown(); got != 2 {
|
|
t.Fatalf("countDown with two down upstreams = %d, want 2", got)
|
|
}
|
|
}
|
|
|
|
func Test_upstreamMonitorDownFor(t *testing.T) {
|
|
const upstream = upstreamPrefix + "0"
|
|
um, _, now := downMonitor(t, upstream)
|
|
|
|
*now = now.Add(2500 * time.Millisecond)
|
|
|
|
if got := um.downFor(upstream); got != 2500*time.Millisecond {
|
|
t.Fatalf("downFor a down upstream = %s, want 2.5s", got)
|
|
}
|
|
if got := um.downFor(upstreamPrefix + "9"); got != 0 {
|
|
t.Fatalf("downFor an unknown upstream = %s, want 0", got)
|
|
}
|
|
|
|
um.reset(upstream)
|
|
|
|
if got := um.downFor(upstream); got != 0 {
|
|
t.Fatalf("downFor an upstream that is up = %s, want 0", got)
|
|
}
|
|
}
|
|
|
|
func Test_upstreamMonitorCountDownExcept(t *testing.T) {
|
|
captureDebugMainLog(t)
|
|
um := newUpstreamMonitor(&ctrld.Config{})
|
|
um.mu.Lock()
|
|
um.markDown(upstreamPrefix+"0", 3, "immediate")
|
|
um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"1", 3, "immediate")
|
|
um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"2", 3, "immediate")
|
|
um.mu.Unlock()
|
|
|
|
if got := um.countDown(); got != 3 {
|
|
t.Fatalf("countDown = %d, want 3", got)
|
|
}
|
|
if got := um.countDownExcept(isInternalDomainUpstream); got != 1 {
|
|
t.Fatalf("countDownExcept(isInternalDomainUpstream) = %d, want 1", got)
|
|
}
|
|
if got := um.countDownExcept(nil); got != 3 {
|
|
t.Fatalf("countDownExcept(nil) = %d, want 3", got)
|
|
}
|
|
}
|