From 2f164f3c4f949448aeaa26612c8ee7387e406260 Mon Sep 17 00:00:00 2001 From: Joshua Temple Date: Fri, 26 Jun 2026 03:36:02 -0400 Subject: [PATCH] test: retry transient GitHub API errors in scenario suite Signed-off-by: Joshua Temple --- .github/workflows/scenario-suite.yaml | 216 ++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) diff --git a/.github/workflows/scenario-suite.yaml b/.github/workflows/scenario-suite.yaml index f8f318a..dcf8d47 100644 --- a/.github/workflows/scenario-suite.yaml +++ b/.github/workflows/scenario-suite.yaml @@ -25,6 +25,42 @@ jobs: # earliest job, so no run the suite causes can fall outside the window. window-start: ${{ steps.window.outputs.window-start }} steps: + - name: Install gh transient-retry wrapper + run: | + cat > "$RUNNER_TEMP/gh-retry.sh" <<'GHRETRY' + _gh_is_transient() { + local out="$1" + if printf '%s' "$out" | grep -qiE 'HTTP 5[0-9][0-9]|HTTP 429|HTTP 401|Bad credentials|was submitted too quickly|secondary rate limit'; then + return 0 + fi + if printf '%s' "$out" | grep -qiE 'HTTP 403'; then + if printf '%s' "$out" | grep -qiE 'rate limit|secondary|abuse|too quickly'; then + return 0 + fi + fi + return 1 + } + gh() { + local attempt=1 max="${GH_RETRY_MAX:-5}" delay="${GH_RETRY_BASE_DELAY:-3}" out rc + while :; do + out="$(command gh "$@" 2>&1)" && rc=0 || rc=$? + if [ "$rc" -eq 0 ]; then + printf '%s\n' "$out" + return 0 + fi + if [ "$attempt" -ge "$max" ] || ! _gh_is_transient "$out"; then + printf '%s\n' "$out" >&2 + return "$rc" + fi + printf 'gh: transient error on attempt %d/%d, retrying in %ds\n%s\n' "$attempt" "$max" "$delay" "$out" >&2 + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done + } + GHRETRY + echo "BASH_ENV=$RUNNER_TEMP/gh-retry.sh" >> "$GITHUB_ENV" + # The reconcile window opens here, before the first reset push, dispatch, # or merge. Every run the suite goes on to cause lands at or after this # timestamp, so the reconcile job sees all of them. @@ -121,6 +157,42 @@ jobs: runs-on: ubuntu-latest needs: reset steps: + - name: Install gh transient-retry wrapper + run: | + cat > "$RUNNER_TEMP/gh-retry.sh" <<'GHRETRY' + _gh_is_transient() { + local out="$1" + if printf '%s' "$out" | grep -qiE 'HTTP 5[0-9][0-9]|HTTP 429|HTTP 401|Bad credentials|was submitted too quickly|secondary rate limit'; then + return 0 + fi + if printf '%s' "$out" | grep -qiE 'HTTP 403'; then + if printf '%s' "$out" | grep -qiE 'rate limit|secondary|abuse|too quickly'; then + return 0 + fi + fi + return 1 + } + gh() { + local attempt=1 max="${GH_RETRY_MAX:-5}" delay="${GH_RETRY_BASE_DELAY:-3}" out rc + while :; do + out="$(command gh "$@" 2>&1)" && rc=0 || rc=$? + if [ "$rc" -eq 0 ]; then + printf '%s\n' "$out" + return 0 + fi + if [ "$attempt" -ge "$max" ] || ! _gh_is_transient "$out"; then + printf '%s\n' "$out" >&2 + return "$rc" + fi + printf 'gh: transient error on attempt %d/%d, retrying in %ds\n%s\n' "$attempt" "$max" "$delay" "$out" >&2 + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done + } + GHRETRY + echo "BASH_ENV=$RUNNER_TEMP/gh-retry.sh" >> "$GITHUB_ENV" + - name: Checkout uses: actions/checkout@v4 with: @@ -399,6 +471,42 @@ jobs: runs-on: ubuntu-latest needs: commit-and-build steps: + - name: Install gh transient-retry wrapper + run: | + cat > "$RUNNER_TEMP/gh-retry.sh" <<'GHRETRY' + _gh_is_transient() { + local out="$1" + if printf '%s' "$out" | grep -qiE 'HTTP 5[0-9][0-9]|HTTP 429|HTTP 401|Bad credentials|was submitted too quickly|secondary rate limit'; then + return 0 + fi + if printf '%s' "$out" | grep -qiE 'HTTP 403'; then + if printf '%s' "$out" | grep -qiE 'rate limit|secondary|abuse|too quickly'; then + return 0 + fi + fi + return 1 + } + gh() { + local attempt=1 max="${GH_RETRY_MAX:-5}" delay="${GH_RETRY_BASE_DELAY:-3}" out rc + while :; do + out="$(command gh "$@" 2>&1)" && rc=0 || rc=$? + if [ "$rc" -eq 0 ]; then + printf '%s\n' "$out" + return 0 + fi + if [ "$attempt" -ge "$max" ] || ! _gh_is_transient "$out"; then + printf '%s\n' "$out" >&2 + return "$rc" + fi + printf 'gh: transient error on attempt %d/%d, retrying in %ds\n%s\n' "$attempt" "$max" "$delay" "$out" >&2 + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done + } + GHRETRY + echo "BASH_ENV=$RUNNER_TEMP/gh-retry.sh" >> "$GITHUB_ENV" + - name: Checkout uses: actions/checkout@v4 with: @@ -536,6 +644,42 @@ jobs: runs-on: ubuntu-latest needs: promote-staging steps: + - name: Install gh transient-retry wrapper + run: | + cat > "$RUNNER_TEMP/gh-retry.sh" <<'GHRETRY' + _gh_is_transient() { + local out="$1" + if printf '%s' "$out" | grep -qiE 'HTTP 5[0-9][0-9]|HTTP 429|HTTP 401|Bad credentials|was submitted too quickly|secondary rate limit'; then + return 0 + fi + if printf '%s' "$out" | grep -qiE 'HTTP 403'; then + if printf '%s' "$out" | grep -qiE 'rate limit|secondary|abuse|too quickly'; then + return 0 + fi + fi + return 1 + } + gh() { + local attempt=1 max="${GH_RETRY_MAX:-5}" delay="${GH_RETRY_BASE_DELAY:-3}" out rc + while :; do + out="$(command gh "$@" 2>&1)" && rc=0 || rc=$? + if [ "$rc" -eq 0 ]; then + printf '%s\n' "$out" + return 0 + fi + if [ "$attempt" -ge "$max" ] || ! _gh_is_transient "$out"; then + printf '%s\n' "$out" >&2 + return "$rc" + fi + printf 'gh: transient error on attempt %d/%d, retrying in %ds\n%s\n' "$attempt" "$max" "$delay" "$out" >&2 + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done + } + GHRETRY + echo "BASH_ENV=$RUNNER_TEMP/gh-retry.sh" >> "$GITHUB_ENV" + - name: Checkout uses: actions/checkout@v4 with: @@ -781,6 +925,42 @@ jobs: runs-on: ubuntu-latest needs: rollback-check steps: + - name: Install gh transient-retry wrapper + run: | + cat > "$RUNNER_TEMP/gh-retry.sh" <<'GHRETRY' + _gh_is_transient() { + local out="$1" + if printf '%s' "$out" | grep -qiE 'HTTP 5[0-9][0-9]|HTTP 429|HTTP 401|Bad credentials|was submitted too quickly|secondary rate limit'; then + return 0 + fi + if printf '%s' "$out" | grep -qiE 'HTTP 403'; then + if printf '%s' "$out" | grep -qiE 'rate limit|secondary|abuse|too quickly'; then + return 0 + fi + fi + return 1 + } + gh() { + local attempt=1 max="${GH_RETRY_MAX:-5}" delay="${GH_RETRY_BASE_DELAY:-3}" out rc + while :; do + out="$(command gh "$@" 2>&1)" && rc=0 || rc=$? + if [ "$rc" -eq 0 ]; then + printf '%s\n' "$out" + return 0 + fi + if [ "$attempt" -ge "$max" ] || ! _gh_is_transient "$out"; then + printf '%s\n' "$out" >&2 + return "$rc" + fi + printf 'gh: transient error on attempt %d/%d, retrying in %ds\n%s\n' "$attempt" "$max" "$delay" "$out" >&2 + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done + } + GHRETRY + echo "BASH_ENV=$RUNNER_TEMP/gh-retry.sh" >> "$GITHUB_ENV" + - name: Checkout uses: actions/checkout@v4 with: @@ -910,6 +1090,42 @@ jobs: runs-on: ubuntu-latest needs: dispatch-inputs-check steps: + - name: Install gh transient-retry wrapper + run: | + cat > "$RUNNER_TEMP/gh-retry.sh" <<'GHRETRY' + _gh_is_transient() { + local out="$1" + if printf '%s' "$out" | grep -qiE 'HTTP 5[0-9][0-9]|HTTP 429|HTTP 401|Bad credentials|was submitted too quickly|secondary rate limit'; then + return 0 + fi + if printf '%s' "$out" | grep -qiE 'HTTP 403'; then + if printf '%s' "$out" | grep -qiE 'rate limit|secondary|abuse|too quickly'; then + return 0 + fi + fi + return 1 + } + gh() { + local attempt=1 max="${GH_RETRY_MAX:-5}" delay="${GH_RETRY_BASE_DELAY:-3}" out rc + while :; do + out="$(command gh "$@" 2>&1)" && rc=0 || rc=$? + if [ "$rc" -eq 0 ]; then + printf '%s\n' "$out" + return 0 + fi + if [ "$attempt" -ge "$max" ] || ! _gh_is_transient "$out"; then + printf '%s\n' "$out" >&2 + return "$rc" + fi + printf 'gh: transient error on attempt %d/%d, retrying in %ds\n%s\n' "$attempt" "$max" "$delay" "$out" >&2 + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done + } + GHRETRY + echo "BASH_ENV=$RUNNER_TEMP/gh-retry.sh" >> "$GITHUB_ENV" + - name: Checkout uses: actions/checkout@v4 with: