mirror of
https://github.com/Control-D-Inc/ctrld.git
synced 2026-10-12 01:58:46 +02:00
fix(darwin): probe interception after the host wakes
Rule text survives a suspend unchanged, so the watchdog reports the pf anchor intact while nothing translates, and the probe monitor is only armed by interface changes with a schedule frozen through the sleep. Detect the resume from the wall-clock gap and probe from there, with a bounded repair: at most two forced reloads across the retry schedule. Refresh the OS resolver before every probe - the probe aims at the first OS nameserver and passes when it has none, so a pre-sleep list makes it meaningless. Stand down for stabilization, which verifies on completion.
This commit is contained in:
5 files changed
+972
-28
No files matched your search
@@ -0,0 +1,109 @@
|
||||
package cli
|
||||
|
||||
import "time"
|
||||
|
||||
// Suspend/resume detection without an OS notification API.
|
||||
//
|
||||
// A host sleep suspends the whole process: timers stop advancing and resume where they
|
||||
// left off, so a tick scheduled two seconds out can arrive a minute later. That
|
||||
// overshoot is the only in-process evidence that the host slept, and nothing else tells
|
||||
// the DNS-intercept repair paths that the state they proved before sleep - rule text,
|
||||
// resolver list, upstream connections - may no longer hold.
|
||||
|
||||
const (
|
||||
// suspendProbeInterval is how often the detector measures elapsed wall-clock time.
|
||||
// Short enough that a resume is noticed within a couple of seconds, which is what
|
||||
// the post-wake continuity contract measures from.
|
||||
suspendProbeInterval = 2 * time.Second
|
||||
|
||||
// suspendGapThreshold is the elapsed wall clock over one interval that counts as a
|
||||
// suspend. A pfctl storm or a loaded host can stretch a tick, so this sits at
|
||||
// several times the interval; a real sleep overshoots by far more. Erring low is
|
||||
// cheap on purpose: a false positive costs one probe, and a stall this long is
|
||||
// worth a probe either way.
|
||||
suspendGapThreshold = 8 * time.Second
|
||||
)
|
||||
|
||||
// wallNanoNow reads the wall clock in nanoseconds.
|
||||
//
|
||||
// The detector works in wall-clock nanoseconds rather than time.Time on purpose.
|
||||
// time.Time.Sub prefers the monotonic reading whenever both operands carry one, and
|
||||
// Darwin's monotonic clock stops while the host is asleep - so a monotonic difference
|
||||
// across a suspend is just the tick interval and hides the very gap this looks for.
|
||||
// Taking int64 wall nanos makes that mistake unrepresentable rather than a comment.
|
||||
func wallNanoNow() int64 { return time.Now().UnixNano() }
|
||||
|
||||
// suspendDetector turns observations of a fixed-interval tick into suspend/resume
|
||||
// events. It measures elapsed time rather than counting ticks, because a suspended
|
||||
// process gets one late tick, not the ticks it missed.
|
||||
type suspendDetector struct {
|
||||
threshold time.Duration
|
||||
wallNano func() int64
|
||||
lastNano int64
|
||||
}
|
||||
|
||||
// newSuspendDetector returns a detector that reports gaps of at least threshold.
|
||||
// wallNano is a seam for tests; nil means the real wall clock.
|
||||
func newSuspendDetector(threshold time.Duration, wallNano func() int64) *suspendDetector {
|
||||
if wallNano == nil {
|
||||
wallNano = wallNanoNow
|
||||
}
|
||||
return &suspendDetector{threshold: threshold, wallNano: wallNano, lastNano: wallNano()}
|
||||
}
|
||||
|
||||
// observe records a tick and reports the elapsed time when it reaches the threshold,
|
||||
// which means the process was not running for most of that span.
|
||||
func (d *suspendDetector) observe() (time.Duration, bool) {
|
||||
current := d.wallNano()
|
||||
gap := time.Duration(current - d.lastNano)
|
||||
d.lastNano = current
|
||||
if gap < d.threshold {
|
||||
return 0, false
|
||||
}
|
||||
return gap, true
|
||||
}
|
||||
|
||||
// reset re-baselines the detector to the current wall clock. Time that has already
|
||||
// passed is deliberately discarded rather than measured: use it after doing work
|
||||
// outside the sampling loop, which is not a suspend even though it looks like one.
|
||||
func (d *suspendDetector) reset() { d.lastNano = d.wallNano() }
|
||||
|
||||
// watchForResume calls onResume once per detected suspend/resume gap, until stopCh
|
||||
// closes or tick is closed. tick is a parameter so tests can drive it directly.
|
||||
//
|
||||
// onResume runs on this goroutine, so recovery is serialized against detection - one
|
||||
// resume cannot start a second recovery on top of the first. The cost is that no
|
||||
// sampling happens while it runs, which the re-baseline below accounts for.
|
||||
func watchForResume(stopCh <-chan struct{}, tick <-chan time.Time, d *suspendDetector, onResume func(gap time.Duration)) {
|
||||
for {
|
||||
select {
|
||||
case <-stopCh:
|
||||
return
|
||||
case _, ok := <-tick:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if gap, resumed := d.observe(); resumed {
|
||||
onResume(gap)
|
||||
// Re-baseline: the recovery just ran, and its own runtime exceeds the
|
||||
// threshold on its own (the post-wake probe schedule waits for seconds
|
||||
// before a single probe timeout or reload). The ticker keeps firing and
|
||||
// buffers a tick while it works, so without this the next observe()
|
||||
// measures the callback and reports it as a fresh suspend - which starts
|
||||
// another recovery, which is again long enough to look like a suspend.
|
||||
// Under a persistent pf failure that is an unbounded reload loop, and
|
||||
// each pass flushes pf state and kills in-flight DoH connections.
|
||||
d.reset()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runSuspendWatcher watches for suspend/resume against a real ticker.
|
||||
//
|
||||
//lint:ignore U1000 used on darwin (post-wake interception verification)
|
||||
func runSuspendWatcher(stopCh <-chan struct{}, onResume func(gap time.Duration)) {
|
||||
ticker := time.NewTicker(suspendProbeInterval)
|
||||
defer ticker.Stop()
|
||||
watchForResume(stopCh, ticker.C, newSuspendDetector(suspendGapThreshold, nil), onResume)
|
||||
}
|
||||
Reference in new issue
Block a user