From 5bd5a665b0022cedb033034ea7687347218b171d Mon Sep 17 00:00:00 2001 From: Patrick Dodgen Date: Thu, 9 Jul 2026 12:54:08 -0600 Subject: [PATCH] fix: raise helm client QPS/burst to stop rate-limiter deploy failures helm upgrade --wait polls the status of every object in a release on a loop. Large releases (webstore prod: ~90+ pods plus deployments, services, ingress) exceed helm's low default client QPS, and client-go's rate limiter aborts status computation with "client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline", failing an otherwise-healthy deploy and triggering --atomic rollback. Set HELM_QPS=50 and HELM_BURST_LIMIT=300 on the deploy step. This only relaxes the client-side limiter; the API server's Priority & Fairness still protects it. Surfaced on webstore prod 2026-07-09 (DEVEX-1687). --- .github/workflows/helm-deploy-eks.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/helm-deploy-eks.yaml b/.github/workflows/helm-deploy-eks.yaml index f1a2b99..325efd7 100644 --- a/.github/workflows/helm-deploy-eks.yaml +++ b/.github/workflows/helm-deploy-eks.yaml @@ -91,6 +91,17 @@ jobs: with: token: ${{ github.token }} - name: deploy + # Raise helm's client-side API rate limits. `helm upgrade --wait` + # polls the status of every object in the release on a loop; large + # releases (e.g. webstore: ~90+ pods + deployments/services/ingress) + # exceed helm's low default QPS and the client-go rate limiter errors + # with "client rate limiter Wait returned an error: rate: Wait(n=1) + # would exceed context deadline", failing an otherwise-healthy deploy + # (webstore prod, 2026-07-09, DEVEX-1687). The API server's own + # Priority & Fairness still protects it; this only relaxes the client. + env: + HELM_QPS: "50" + HELM_BURST_LIMIT: "300" run: | echo "${{ secrets.kubeconfig }}" >> kube.config export KUBECONFIG=./kube.config