From 16b25e88ede9ca9c324bb741216a92c913c3e0ad Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Somarapu Date: Tue, 4 Aug 2026 12:51:50 -0700 Subject: [PATCH] Jitter throttling client retries to avoid lockstep The retry backoff starts at a fixed 500ms, so every client that loses the throttling server computes an identical delay sequence and retries at the same instants. The clients are already synchronised by construction: they fail together because the server they share became unreachable. Each retry round therefore arrives as a burst against a service that exists to limit request rate, and the burst repeats as the sequence doubles. Randomise the initial delay. Every subsequent delay is a multiple of it, so spreading the initial value decorrelates the whole sequence rather than just the first retry. The random amount is added to the 500ms floor rather than centred on it, so a client never retries sooner than the fixed delay already allowed. This mirrors MysqlMultiActiveLeaseArbiter, which randomises its initial delay the same way, and leaves the shared ExponentialBackoff utility untouched: its maxWait is a total budget, and pollers such as AsyncConverter1to1Test rely on that budget admitting a predictable number of attempts. Signed-off-by: Santhosh Kumar Somarapu --- .../RedirectAwareRestClientRequestSender.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-client/src/main/java/org/apache/gobblin/util/limiter/RedirectAwareRestClientRequestSender.java b/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-client/src/main/java/org/apache/gobblin/util/limiter/RedirectAwareRestClientRequestSender.java index 0863b05e4e1..54c9dd29b34 100644 --- a/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-client/src/main/java/org/apache/gobblin/util/limiter/RedirectAwareRestClientRequestSender.java +++ b/gobblin-restli/gobblin-throttling-service/gobblin-throttling-service-client/src/main/java/org/apache/gobblin/util/limiter/RedirectAwareRestClientRequestSender.java @@ -22,6 +22,7 @@ import java.net.URISyntaxException; import java.util.List; import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @@ -63,6 +64,10 @@ public class RedirectAwareRestClientRequestSender extends RestClientRequestSende private static final int MIN_RETRIES = 3; + private static final long RETRY_MAX_DELAY_MILLIS = 10000L; + private static final long RETRY_MIN_INITIAL_DELAY_MILLIS = 500L; + private static final long RETRY_INITIAL_DELAY_RANGE_MILLIS = 500L; + /** * A {@link SharedResourceFactory} that creates {@link RedirectAwareRestClientRequestSender}s. * @param @@ -195,7 +200,18 @@ void updateRestClient(String uri, String reason, Throwable errorCause) throws No private class CallbackDecorator implements Callback> { private final PermitRequest originalRequest; private final Callback> underlying; - private final ExponentialBackoff exponentialBackoff = ExponentialBackoff.builder().maxDelay(10000L).initialDelay(500L).build(); + // The initial delay is randomised so that clients which lose the throttling server do not all + // retry at the same instants. Every subsequent delay is a multiple of this one, so spreading the + // initial value decorrelates the whole sequence rather than just the first retry. + // + // The random amount is added to the 500ms floor rather than centred on it, so a client never + // retries the throttling service sooner than the fixed delay already allowed. This mirrors + // MysqlMultiActiveLeaseArbiter, which randomises its initial delay the same way. + private final ExponentialBackoff exponentialBackoff = ExponentialBackoff.builder() + .maxDelay(RETRY_MAX_DELAY_MILLIS) + .initialDelay(RETRY_MIN_INITIAL_DELAY_MILLIS + + ThreadLocalRandom.current().nextLong(RETRY_INITIAL_DELAY_RANGE_MILLIS)) + .build(); private int redirects = 0; private int retries = 0;