cmd/ctrld: wait until ctrld listener ready to do self-check

cmd/ctrld/cli.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"net/http"
 	"net/netip"
 	"os"
 	"os/exec"
@@ -1103,13 +1104,45 @@ func selfCheckStatus(status service.Status, domain string) service.Status {
 		// Nothing to do, return the status as-is.
 		return status
 	}
-	c := new(dns.Client)
+	dir, err := userHomeDir()
+	if err != nil {
+		mainLog.Error().Err(err).Msg("failed to check ctrld listener status: could not get home directory")
+		return service.StatusUnknown
+	}
+
 	bo := backoff.NewBackoff("self-check", logf, 10*time.Second)
-	bo.LogLongerThan = 500 * time.Millisecond
+	bo.LogLongerThan = 10 * time.Second
 	ctx := context.Background()
 	maxAttempts := 20
-	mainLog.Debug().Msg("Performing self-check")
 
+	mainLog.Debug().Msg("waiting for ctrld listener to be ready")
+	cc := newControlClient(filepath.Join(dir, ctrldControlUnixSock))
+
+	// The socket control server may not start yet, so attempt to ping
+	// it until we got a response, or maxAttempts reached.
+	for i := 0; i < maxAttempts; i++ {
+		if _, err := cc.post("/", nil); err != nil {
+			bo.BackOff(ctx, err)
+			continue
+		}
+		break
+	}
+	resp, err := cc.post(startedPath, nil)
+	if err != nil {
+		mainLog.Error().Err(err).Msg("failed to connect to control server")
+		return service.StatusUnknown
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		mainLog.Error().Msg("ctrld listener is not ready")
+		return service.StatusUnknown
+	}
+
+	mainLog.Debug().Msg("ctrld listener is ready")
+	mainLog.Debug().Msg("performing self-check")
+	bo = backoff.NewBackoff("self-check", logf, 10*time.Second)
+	bo.LogLongerThan = 500 * time.Millisecond
+	c := new(dns.Client)
 	var (
 		lcChanged map[string]*ctrld.ListenerConfig
 		mu        sync.Mutex

cmd/ctrld/control_server.go

@@ -13,6 +13,7 @@ import (
 const (
 	contentTypeJson = "application/json"
 	listClientsPath = "/clients"
+	startedPath     = "/started"
 )
 
 type controlServer struct {
@@ -63,6 +64,14 @@ func (p *prog) registerControlServerHandler() {
 			return
 		}
 	}))
+	p.cs.mux.Handle(startedPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		select {
+		case <-p.onStartedDone:
+			w.WriteHeader(http.StatusOK)
+		case <-time.After(10 * time.Second):
+			w.WriteHeader(http.StatusRequestTimeout)
+		}
+	}))
 }
 
 func jsonResponse(next http.Handler) http.Handler {

cmd/ctrld/prog.go

@@ -50,9 +50,10 @@ type prog struct {
 	ciTable *clientinfo.Table
 	router  router.Router
 
-	started   chan struct{}
+	started       chan struct{}
-	onStarted []func()
+	onStartedDone chan struct{}
-	onStopped []func()
+	onStarted     []func()
+	onStopped     []func()
 }
 
 func (p *prog) Start(s service.Service) error {
@@ -67,6 +68,7 @@ func (p *prog) run() {
 	p.preRun()
 	numListeners := len(p.cfg.Listener)
 	p.started = make(chan struct{}, numListeners)
+	p.onStartedDone = make(chan struct{})
 	if p.cfg.Service.CacheEnable {
 		cacher, err := dnscache.NewLRUCache(p.cfg.Service.CacheSize)
 		if err != nil {
@@ -146,6 +148,8 @@ func (p *prog) run() {
 	for _, f := range p.onStarted {
 		f()
 	}
+	close(p.onStartedDone)
+
 	// Stop writing log to unix socket.
 	consoleWriter.Out = os.Stdout
 	initLoggingWithBackup(false)