test: add subprocess_retry_run utility and use it for podman

test/test_manifest.py

@@ -6,6 +6,8 @@
 
 import pytest
 
+from test_utils import subprocess_retry_run
+
 # put common podman run args in once place
 podman_run = [
     "podman", "run", "--rm", "--privileged",
@@ -94,8 +96,8 @@ def test_manifest_seeded_is_the_same(build_container, use_seed_arg):
 def test_manifest_bootc_build_container(build_container):
     bootc_ref = "quay.io/centos-bootc/centos-bootc:stream9"
     bootc_build_container_ref = "quay.io/centos-bootc/centos-bootc:stream10"
-    subprocess.check_call(["podman", "pull", bootc_ref])
-    subprocess.check_call(["podman", "pull", bootc_build_container_ref])
+    subprocess_retry_run(["podman", "pull", bootc_ref])
+    subprocess_retry_run(["podman", "pull", bootc_build_container_ref])
 
     output = subprocess.check_output(podman_run + [
         build_container,
@@ -130,8 +132,8 @@ def test_container_manifest_bootc_iso_smoke(build_container):
     # that we get a manifest with the right refs its good enough.
     bootc_ref = "quay.io/centos-bootc/centos-bootc:stream9"
     bootc_payload_ref = "quay.io/centos-bootc/centos-bootc:stream10"
-    subprocess.check_call(["podman", "pull", bootc_ref])
-    subprocess.check_call(["podman", "pull", bootc_payload_ref])
+    subprocess_retry_run(["podman", "pull", bootc_ref])
+    subprocess_retry_run(["podman", "pull", bootc_payload_ref])
     output = subprocess.check_output(podman_run + [
         build_container,
         "manifest",
@@ -191,7 +193,7 @@ def test_manifest_honors_rpmmd_cache(tmp_path, build_container):
     ("quay.io/centos-bootc/centos-bootc:stream9", ""),
 ])
 def test_container_manifest_bootc_smoke(build_container, bootc_ref, defaultfs):
-    subprocess.check_call(["podman", "pull", bootc_ref])
+    subprocess_retry_run(["podman", "pull", bootc_ref])
     bootc_default_fs = []
     if defaultfs:
         bootc_default_fs = ["--bootc-default-fs", defaultfs]

test/test_utils.py

@@ -0,0 +1,59 @@
+"""Utility functions for test suite."""
+import subprocess
+import time
+
+
+def subprocess_retry_run(cmd, max_retries=3, timeout=300, **kwargs):
+    """Run a subprocess command with retry logic and timeout.
+
+    Args:
+        cmd: Command to run (list of strings)
+        max_retries: Maximum number of retry attempts (default: 3)
+        timeout: Timeout in seconds for each attempt (default: 300s = 5min)
+        **kwargs: Additional arguments to pass to subprocess.run()
+
+    Returns:
+        subprocess.CompletedProcess: The successful result
+
+    Raises:
+        subprocess.CalledProcessError: If all retry attempts fail
+        subprocess.TimeoutExpired: If timeout occurs on all attempts
+    """
+    # Set defaults for subprocess.run if not provided
+    check = kwargs.pop('check', True)
+    capture_output = kwargs.pop('capture_output', True)
+    text = kwargs.pop('text', True)
+
+    cmd_str = ' '.join(cmd) if isinstance(cmd, list) else cmd
+    last_exception = None
+
+    for attempt in range(max_retries):
+        try:
+            result = subprocess.run(
+                cmd, timeout=timeout, check=check,
+                capture_output=capture_output, text=text, **kwargs)
+            return result  # Success
+        except subprocess.TimeoutExpired as e:
+            last_exception = e
+            if attempt < max_retries - 1:
+                wait_time = 2 ** attempt  # Exponential backoff: 1s, 2s, 4s
+                print(f"Timeout running '{cmd_str}' (attempt {attempt + 1}/{max_retries}). "
+                      f"Retrying in {wait_time}s...")
+                time.sleep(wait_time)
+            else:
+                raise subprocess.CalledProcessError(
+                    1, e.cmd, f"Timeout after {max_retries} attempts"
+                ) from last_exception
+        except subprocess.CalledProcessError as e:
+            last_exception = e
+            if attempt < max_retries - 1:
+                wait_time = 2 ** attempt  # Exponential backoff: 1s, 2s, 4s
+                stderr = e.stderr if e.stderr else "(no error output)"
+                print(f"Failed to run '{cmd_str}' (attempt {attempt + 1}/{max_retries}): {stderr}. "
+                      f"Retrying in {wait_time}s...")
+                time.sleep(wait_time)
+            else:
+                raise  # Re-raise the last exception
+
+    # Should never reach here due to the raise in the else blocks above
+    return None