feat(cli): keep a log journal on disk with file headers and an upload limit

ctrld keeps a second log file, ctrld-journal.log, next to ctrld.log. The
journal holds all warnings and errors and each event with the field
journal=true. It stays on disk after a start of the service and after a
self-upgrade. Each log file starts with a Log header line. Each log send
upload starts with the same line, made at send time, and has a limit of
about 16 MB.

cmd/cli/dns_proxy_sampling_test.go

@@ -0,0 +1,124 @@
+package cli
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/miekg/dns"
+
+	"github.com/Control-D-Inc/ctrld"
+)
+
+// deadUpstreamEndpoint is a port that nothing listens on, so every query to it
+// fails at once.
+const deadUpstreamEndpoint = "127.0.0.1:1"
+
+// countSamplingLines counts the captured lines of one level whose message
+// starts with the given text. A request id rides in front of the message.
+func countSamplingLines(t *testing.T, logs *syncBuffer, level, message string) int {
+	t.Helper()
+	count := 0
+	for _, event := range jsonLogEvents(t, logs, "") {
+		text, _ := event["message"].(string)
+		if !strings.HasPrefix(text, message) || event["level"] != level {
+			continue
+		}
+		count++
+	}
+	return count
+}
+
+func Test_querySamplingBoundsPerQueryErrors(t *testing.T) {
+	logs := captureDebugMainLog(t)
+	p := &prog{}
+	// The zero value sampler arms a real timer. Close its window after the
+	// assertions, so no late summary line reaches another test.
+	t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) })
+
+	for i := 0; i < querySampleLimit+1; i++ {
+		ctrld.Log(context.Background(), p.querySampler.event(sampleClassResolveFailed, "upstream.0"), samplerTestMessage)
+	}
+
+	if got := countSamplingLines(t, logs, "error", samplerTestMessage); got != querySampleLimit {
+		t.Fatalf("error lines: got %d, want %d", got, querySampleLimit)
+	}
+	if got := countSamplingLines(t, logs, "debug", samplerTestMessage); got != 1 {
+		t.Fatalf("debug lines: got %d, want 1", got)
+	}
+}
+
+// askThroughProxy sends one query through the real proxy path and returns the
+// answer.
+func askThroughProxy(t *testing.T, p *prog, ctx context.Context) *dns.Msg {
+	t.Helper()
+	msg := new(dns.Msg)
+	msg.SetQuestion("sampling.test.", dns.TypeA)
+	res := p.proxy(ctx, &proxyRequest{msg: msg, ufr: &upstreamForResult{srcAddr: "192.168.0.1:1234"}})
+	if res == nil || res.answer == nil {
+		t.Fatal("the proxy returned no answer")
+	}
+	return res.answer
+}
+
+// Test_querySamplingOnTheQueryPath drives the real query path with an upstream
+// that answers nothing: the flood keeps its request id, the sampler bounds it,
+// and one state event brackets each end of the outage.
+func Test_querySamplingOnTheQueryPath(t *testing.T) {
+	logs := captureDebugMainLog(t)
+	cfg := &ctrld.Config{}
+	cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false)
+	p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg)}
+	// A new monitor holds back the failure counts for one second. The test
+	// clears the flag instead of waiting for that timer.
+	p.um.clearRecovered(upstreamOS)
+	var armed []func()
+	p.um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) }
+	// The zero value sampler arms a real timer. Close its window after the
+	// assertions, so no late summary line reaches another test.
+	t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) })
+
+	origOSUpstream := osUpstreamConfig
+	t.Cleanup(func() { osUpstreamConfig = origOSUpstream })
+	osUpstreamConfig = &ctrld.UpstreamConfig{
+		Name:     "dead resolver",
+		Type:     ctrld.ResolverTypeLegacy,
+		Endpoint: deadUpstreamEndpoint,
+		Timeout:  500,
+	}
+
+	ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, "r1")
+	for i := 0; i < querySampleLimit+2; i++ {
+		if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeServerFailure {
+			t.Fatalf("query %d: rcode %s, want SERVFAIL", i+1, dns.RcodeToString[rcode])
+		}
+	}
+
+	const failedMessage = "[r1] " + samplerTestMessage
+	if got := countSamplingLines(t, logs, "error", failedMessage); got != querySampleLimit {
+		t.Fatalf("error lines: got %d, want %d", got, querySampleLimit)
+	}
+	if got := countSamplingLines(t, logs, "debug", failedMessage); got != 2 {
+		t.Fatalf("debug lines: got %d, want 2", got)
+	}
+
+	if len(armed) != 1 {
+		t.Fatalf("armed timers: got %d, want 1", len(armed))
+	}
+	armed[0]()
+	startOSResolverStub(t)
+	if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeSuccess {
+		t.Fatalf("the answered query returned %s, want NOERROR", dns.RcodeToString[rcode])
+	}
+
+	events := jsonLogEvents(t, logs, upstreamStateMessage)
+	if len(events) != 2 {
+		t.Fatalf("state events: got %d, want 2", len(events))
+	}
+	wantField(t, events[0], "upstream", upstreamOS)
+	wantField(t, events[0], "state", "down")
+	wantField(t, events[0], "reason", "timer")
+	wantField(t, events[1], "upstream", upstreamOS)
+	wantField(t, events[1], "state", "up")
+}