fix: retry transient errors on api autocomplete status GET

pkg/api/api.go

@@ -120,6 +120,30 @@ func (api *API) GET(relativeURL string, queryStringer QueryStringer, response in
 	return nil
 }
 
+// httpGetRetryBackoff is the delay between transport-error retries. It is a var
+// so tests can shorten it.
+var httpGetRetryBackoff = 5 * time.Second
+
+// httpGetWithRetry performs an HTTP GET, retrying transient transport errors
+// (such as "connection reset by peer" from the shared demo backend) that
+// otherwise fail integration runs spuriously. Each retry is logged so it is
+// visible in the run output. These GETs are idempotent, so retrying is safe.
+func httpGetWithRetry(client *http.Client, url string) (*http.Response, error) {
+	var lastErr error
+	for try := 0; try < MAX_RETRIES; try++ {
+		resp, err := client.Get(url)
+		if err == nil {
+			return resp, nil
+		}
+		lastErr = err
+		if try < MAX_RETRIES-1 {
+			log.Warnf("error getting %s: %v, retrying... (%d/%d)", url, err, try+1, MAX_RETRIES)
+			time.Sleep(httpGetRetryBackoff)
+		}
+	}
+	return nil, lastErr
+}
+
 // POST submits a POST request to the given URL, with the query string from the
 // given QueryStringer, as well as a body, and unmarshals response data into
 // the given response struct.

pkg/api/autocomplete.go

@@ -79,7 +79,7 @@ func (api *API) GetAutocompleteStatus(path string, req AutocompleteRequest) (int
 	qs := req.QueryString()
 	url := api.URL(path, qs)
 
-	httpResp, err := http.Get(url)
+	httpResp, err := httpGetWithRetry(sharedHTTPClient, url)
 	if err != nil {
 		return 0, nil, fmt.Errorf("error getting %s: %w", url, err)
 	}

pkg/api/retry_test.go

@@ -0,0 +1,66 @@
+package api
+
+import (
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+)
+
+// flakyRoundTripper fails the first failCount requests with a connection-reset
+// style error, then returns a 200. It records how many times it was called.
+type flakyRoundTripper struct {
+	failCount int
+	calls     int
+}
+
+func (f *flakyRoundTripper) RoundTrip(*http.Request) (*http.Response, error) {
+	f.calls++
+	if f.calls <= f.failCount {
+		return nil, errors.New("read tcp 10.0.0.1:1->2:443: read: connection reset by peer")
+	}
+	return &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(strings.NewReader("ok")),
+		Header:     make(http.Header),
+	}, nil
+}
+
+func TestHTTPGetWithRetrySucceedsAfterTransientErrors(t *testing.T) {
+	restore := httpGetRetryBackoff
+	httpGetRetryBackoff = time.Millisecond
+	defer func() { httpGetRetryBackoff = restore }()
+
+	rt := &flakyRoundTripper{failCount: MAX_RETRIES - 1}
+	client := &http.Client{Transport: rt}
+
+	resp, err := httpGetWithRetry(client, "http://example.invalid/allocation/autocomplete")
+	if err != nil {
+		t.Fatalf("expected success after retries, got %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("expected 200, got %d", resp.StatusCode)
+	}
+	if rt.calls != MAX_RETRIES {
+		t.Fatalf("expected %d attempts, got %d", MAX_RETRIES, rt.calls)
+	}
+}
+
+func TestHTTPGetWithRetryReturnsErrorAfterMaxRetries(t *testing.T) {
+	restore := httpGetRetryBackoff
+	httpGetRetryBackoff = time.Millisecond
+	defer func() { httpGetRetryBackoff = restore }()
+
+	rt := &flakyRoundTripper{failCount: 100}
+	client := &http.Client{Transport: rt}
+
+	if _, err := httpGetWithRetry(client, "http://example.invalid/x"); err == nil {
+		t.Fatal("expected an error after exhausting retries")
+	}
+	if rt.calls != MAX_RETRIES {
+		t.Fatalf("expected %d attempts, got %d", MAX_RETRIES, rt.calls)
+	}
+}