cmd/cli: add upstream monitor

Some users mentioned that when there is an Internet outage, ctrld fails
to recover, crashing or locks up the router. When requests start
failing, this results in the clients emitting more queries, creating a
resource spiral of death that can brick the device entirely.

To guard against this case, this commit implement an upstream monitor
approach:

 - Marking upstream as down after 100 consecutive failed queries.
 - Start a goroutine to check when the upstream is back again.
 - When upstream is down, answer all queries with SERVFAIL.
 - The checking process uses backoff retry to reduce high requests rate.
 - As long as the query succeeded, marking the upstream as alive then
   start operate normally.
This commit is contained in:
Cuong Manh Le authored and Cuong Manh Le committed 2023-09-22 18:45:59 +07:00
1 parent bed7435b0c
commit 511c4e696f
4 files changed
+141 -19

No files matched your search

+5 -1
View File
@@ -97,8 +97,10 @@ func (r *dohResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, erro
func addHeader(ctx context.Context, req *http.Request, sendClientInfo bool) {
req.Header.Set("Content-Type", headerApplicationDNS)
req.Header.Set("Accept", headerApplicationDNS)
printed := false
if sendClientInfo {
if ci, ok := ctx.Value(ClientInfoCtxKey{}).(*ClientInfo); ok && ci != nil {
printed = ci.Mac != "" || ci.IP != "" || ci.Hostname != ""
if ci.Mac != "" {
req.Header.Set(dohMacHeader, ci.Mac)
}
@@ -110,5 +112,7 @@ func addHeader(ctx context.Context, req *http.Request, sendClientInfo bool) {
}
}
}
Log(ctx, ProxyLogger.Load().Debug().Interface("header", req.Header), "sending request header")
if printed {
Log(ctx, ProxyLogger.Load().Debug().Interface("header", req.Header), "sending request header")
}
}