diff --git a/pkg/api/api.go b/pkg/api/api.go index 49613a1..a62af87 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -120,6 +120,30 @@ func (api *API) GET(relativeURL string, queryStringer QueryStringer, response in return nil } +// httpGetRetryBackoff is the delay between transport-error retries. It is a var +// so tests can shorten it. +var httpGetRetryBackoff = 5 * time.Second + +// httpGetWithRetry performs an HTTP GET, retrying transient transport errors +// (such as "connection reset by peer" from the shared demo backend) that +// otherwise fail integration runs spuriously. Each retry is logged so it is +// visible in the run output. These GETs are idempotent, so retrying is safe. +func httpGetWithRetry(client *http.Client, url string) (*http.Response, error) { + var lastErr error + for try := 0; try < MAX_RETRIES; try++ { + resp, err := client.Get(url) + if err == nil { + return resp, nil + } + lastErr = err + if try < MAX_RETRIES-1 { + log.Warnf("error getting %s: %v, retrying... (%d/%d)", url, err, try+1, MAX_RETRIES) + time.Sleep(httpGetRetryBackoff) + } + } + return nil, lastErr +} + // POST submits a POST request to the given URL, with the query string from the // given QueryStringer, as well as a body, and unmarshals response data into // the given response struct. diff --git a/pkg/api/autocomplete.go b/pkg/api/autocomplete.go index 1e338f6..e7ea013 100644 --- a/pkg/api/autocomplete.go +++ b/pkg/api/autocomplete.go @@ -79,7 +79,7 @@ func (api *API) GetAutocompleteStatus(path string, req AutocompleteRequest) (int qs := req.QueryString() url := api.URL(path, qs) - httpResp, err := http.Get(url) + httpResp, err := httpGetWithRetry(sharedHTTPClient, url) if err != nil { return 0, nil, fmt.Errorf("error getting %s: %w", url, err) } diff --git a/pkg/api/retry_test.go b/pkg/api/retry_test.go new file mode 100644 index 0000000..1be728b --- /dev/null +++ b/pkg/api/retry_test.go @@ -0,0 +1,66 @@ +package api + +import ( + "errors" + "io" + "net/http" + "strings" + "testing" + "time" +) + +// flakyRoundTripper fails the first failCount requests with a connection-reset +// style error, then returns a 200. It records how many times it was called. +type flakyRoundTripper struct { + failCount int + calls int +} + +func (f *flakyRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + f.calls++ + if f.calls <= f.failCount { + return nil, errors.New("read tcp 10.0.0.1:1->2:443: read: connection reset by peer") + } + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("ok")), + Header: make(http.Header), + }, nil +} + +func TestHTTPGetWithRetrySucceedsAfterTransientErrors(t *testing.T) { + restore := httpGetRetryBackoff + httpGetRetryBackoff = time.Millisecond + defer func() { httpGetRetryBackoff = restore }() + + rt := &flakyRoundTripper{failCount: MAX_RETRIES - 1} + client := &http.Client{Transport: rt} + + resp, err := httpGetWithRetry(client, "http://example.invalid/allocation/autocomplete") + if err != nil { + t.Fatalf("expected success after retries, got %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + if rt.calls != MAX_RETRIES { + t.Fatalf("expected %d attempts, got %d", MAX_RETRIES, rt.calls) + } +} + +func TestHTTPGetWithRetryReturnsErrorAfterMaxRetries(t *testing.T) { + restore := httpGetRetryBackoff + httpGetRetryBackoff = time.Millisecond + defer func() { httpGetRetryBackoff = restore }() + + rt := &flakyRoundTripper{failCount: 100} + client := &http.Client{Transport: rt} + + if _, err := httpGetWithRetry(client, "http://example.invalid/x"); err == nil { + t.Fatal("expected an error after exhausting retries") + } + if rt.calls != MAX_RETRIES { + t.Fatalf("expected %d attempts, got %d", MAX_RETRIES, rt.calls) + } +}