cmd/cli, internal/controld: bound API preflight by service lifetime

processCDFlags retries the resolver-config fetch indefinitely by design: a
device that has no working network at boot must eventually come up. The loop
had no cancellation, so a stop request arriving while the API is unreachable
was ignored - the process kept retrying long after "Service stopped" was
logged, doing work on behalf of a service the OS considers stopped. The
Windows Firewall Mode incident showed this concretely: API retries continued
15 seconds after the stop completed, so stopping the service could not release
what the process was still holding.

Thread a context through processCDFlags and derive it from p.stopCh, in both
the startup preflight and the config-reload path. The loop now returns as soon
as the context is cancelled, checked both before a retry and after backoff
returns (backoff can wake up on cancellation).

A stop during preflight now exits the way a normal stop does, without Fatal,
so the service manager does not treat it as a failed start and apply its
restart policy to a service the operator just asked to stop.

Also bind the two API requests in internal/controld to the caller's context.
They were built with http.NewRequest, so an in-flight request ignored
cancellation and waited out its own timeout instead.

Covered by tests that assert what the incident needed: a stop request during
preflight ends the retry/backoff loop and reports cancellation rather than
continuing to retry after the service reports itself stopped, an
already-cancelled context makes at most one attempt, and contextFromStopCh
handles its three cases (cancelled by stopCh, released by cancel, usable with
no stop channel). Removing either cancellation check makes these tests hang
until the test timeout. The tests are also what exercise fetchResolverConfig,
the seam this commit introduces.

Sampling the stop state is the whole point of runAPIPreflight rather than doing
this inline. A stop and a failure need opposite handling - one exits quietly, the
other self-uninstalls a deleted device, surfaces the error to a mobile app, and
reports a failed start - so the two must not be confused. Reading it from the
context after cancelling would report stopped for every failure, since
CancelFunc sets ctx.Err() regardless of whether anyone asked to stop; the stop
channel is read directly instead, which also does not depend on the context
watcher goroutine having been scheduled.
This commit is contained in:
Cuong Manh Le
2026-08-14 15:27:54 +07:00
parent e5f2506199
commit 52b7aaab87
5 changed files with 672 additions and 18 deletions
+32 -6
View File
@@ -61,12 +61,35 @@ type ErrorResponse struct {
Message string `json:"message"`
Code int `json:"code"`
} `json:"error"`
// StatusCode is the HTTP status the API answered with. It is not part of the JSON
// body: this type is built for *any* non-200 whose body decodes, so the body alone
// cannot tell a permanent rejection of the request from a transient server-side
// failure, and callers that act differently on the two need the status to tell them
// apart. Zero means the status was not recorded.
StatusCode int `json:"-"`
}
func (u ErrorResponse) Error() string {
return u.ErrorField.Message
}
// apiErrorFromResponse builds the error for a non-200 API answer, recording the HTTP
// status alongside the decoded body.
//
// The status is what tells a caller whether the answer will change on a retry: this type
// is built for every non-200 whose body decodes, so a 502 from a load balancer and a 404
// for a deleted device are otherwise indistinguishable. Both response paths go through
// here so neither can decode a body and forget to record it.
func apiErrorFromResponse(statusCode int, d *json.Decoder) (*ErrorResponse, error) {
errResp := &ErrorResponse{StatusCode: statusCode}
if err := d.Decode(errResp); err != nil {
return nil, err
}
// Decode fills exported fields from the body; StatusCode is json:"-", so it survives.
errResp.StatusCode = statusCode
return errResp, nil
}
type utilityRequest struct {
UID string `json:"uid"`
ClientID string `json:"client_id,omitempty"`
@@ -171,7 +194,10 @@ func postUtilityAPI(ctx context.Context, version string, cdDev, lastUpdatedFaile
}
ctrld.Log(ctx, logger.Debug(), "Creating HTTP request")
req, err := http.NewRequest("POST", apiUrl, body)
// Context-bound so an in-flight request is abandoned when the caller is
// cancelled - a service stop during API preflight must not wait out the
// request timeout, let alone keep retrying.
req, err := http.NewRequestWithContext(ctx, "POST", apiUrl, body)
if err != nil {
ctrld.Log(ctx, logger.Error(), "Failed to create HTTP request: %v", err)
return nil, fmt.Errorf("http.NewRequest: %w", err)
@@ -206,8 +232,8 @@ func postUtilityAPI(ctx context.Context, version string, cdDev, lastUpdatedFaile
ctrld.Log(ctx, logger.Debug(), "Processing API response")
d := json.NewDecoder(resp.Body)
if resp.StatusCode != http.StatusOK {
errResp := &ErrorResponse{}
if err := d.Decode(errResp); err != nil {
errResp, err := apiErrorFromResponse(resp.StatusCode, d)
if err != nil {
ctrld.Log(ctx, logger.Error(), "Failed to decode error response: %v", err)
return nil, err
}
@@ -237,7 +263,7 @@ func SendLogs(ctx context.Context, lr *LogsRequest, cdDev bool) error {
}
ctrld.Log(ctx, logger.Debug(), "Creating HTTP request for log upload")
req, err := http.NewRequest("POST", apiUrl, lr.Data)
req, err := http.NewRequestWithContext(ctx, "POST", apiUrl, lr.Data)
if err != nil {
ctrld.Log(ctx, logger.Error(), "Failed to create HTTP request: %v", err)
return fmt.Errorf("http.NewRequest: %w", err)
@@ -265,8 +291,8 @@ func SendLogs(ctx context.Context, lr *LogsRequest, cdDev bool) error {
ctrld.Log(ctx, logger.Debug(), "Processing API response")
d := json.NewDecoder(resp.Body)
if resp.StatusCode != http.StatusOK {
errResp := &ErrorResponse{}
if err := d.Decode(errResp); err != nil {
errResp, err := apiErrorFromResponse(resp.StatusCode, d)
if err != nil {
ctrld.Log(ctx, logger.Error(), "Failed to decode error response: %v", err)
return err
}
+67
View File
@@ -1,6 +1,9 @@
package controld
import (
"encoding/json"
"net/http"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -29,3 +32,67 @@ func Test_parseUID(t *testing.T) {
})
}
}
// TestAPIErrorRecordsHTTPStatus pins the plumbing the caller's exit decision rests on.
//
// cmd/cli treats a 4xx as "this configuration is refused, restarting cannot help" and
// exits cleanly, while a 5xx keeps the abnormal exit so the service manager retries. Both
// readings need the status, and it is not in the JSON body - so a decode path that
// forgets to record it would quietly send every API error down the retry branch,
// including a deleted device that should self-uninstall and stop.
func TestAPIErrorRecordsHTTPStatus(t *testing.T) {
tests := []struct {
name string
statusCode int
body string
wantCode int
wantMsg string
}{
{
name: "deleted device",
statusCode: http.StatusNotFound,
body: `{"error":{"message":"device does not exist","code":40402}}`,
wantCode: InvalidConfigCode,
wantMsg: "device does not exist",
},
{
// A gateway error body carries no error object at all, which decodes
// cleanly into the zero value - so the status is the only thing that
// distinguishes it from a real rejection.
name: "gateway error with an empty body",
statusCode: http.StatusBadGateway,
body: `{}`,
},
{
name: "service unavailable",
statusCode: http.StatusServiceUnavailable,
body: `{"error":{"message":"try again later","code":0}}`,
wantMsg: "try again later",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
d := json.NewDecoder(strings.NewReader(tc.body))
errResp, err := apiErrorFromResponse(tc.statusCode, d)
if err != nil {
t.Fatalf("unexpected decode error: %v", err)
}
if errResp.StatusCode != tc.statusCode {
t.Errorf("StatusCode = %d, want %d: the caller cannot tell a permanent rejection from a transient failure without it", errResp.StatusCode, tc.statusCode)
}
if errResp.ErrorField.Code != tc.wantCode {
t.Errorf("code = %d, want %d", errResp.ErrorField.Code, tc.wantCode)
}
if errResp.Error() != tc.wantMsg {
t.Errorf("message = %q, want %q", errResp.Error(), tc.wantMsg)
}
})
}
t.Run("an undecodable body is reported as a decode failure", func(t *testing.T) {
d := json.NewDecoder(strings.NewReader("<html>502 Bad Gateway</html>"))
if _, err := apiErrorFromResponse(http.StatusBadGateway, d); err == nil {
t.Error("expected a decode error for a non-JSON body")
}
})
}