feat(scratch): Make scratch init faster (#1499)

Author: ZohebShaikh

src/blueapi/cli/scratch.py

@@ -3,6 +3,7 @@
 import os
 import stat
 import textwrap
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from subprocess import Popen
 
@@ -46,10 +47,27 @@ def setup_scratch(
         That is to prevent namespace clashing with the blueapi application.
         """)
             )
-    for repo in config.repositories:
-        local_directory = config.root / repo.name
-        ensure_repo(repo.remote_url, local_directory, repo.target_revision)
-        scratch_install(local_directory, timeout=install_timeout)
+
+    with ThreadPoolExecutor() as executor:
+        futures = [
+            executor.submit(
+                ensure_repo,
+                repo.remote_url,
+                config.root / repo.name,
+                repo.target_revision,
+            )
+            for repo in config.repositories
+        ]
+        for future in as_completed(futures):
+            try:
+                future.result()
+            except Exception as exc:
+                raise RuntimeError("Failed to clone repositories") from exc
+
+    scratch_install(
+        *(config.root / repo.name for repo in config.repositories),
+        timeout=install_timeout,
+    )
 
 
 def ensure_repo(
@@ -69,7 +87,12 @@ def ensure_repo(
 
     if not local_directory.exists():
         LOGGER.info(f"Cloning {remote_url}")
-        Repo.clone_from(remote_url, local_directory, branch=target_revision)
+        Repo.clone_from(
+            remote_url,
+            local_directory,
+            branch=target_revision,
+            filter="blob:none",
+        )
         LOGGER.info(f"Cloned {remote_url} -> {local_directory}")
     elif local_directory.is_dir():
         repo = Repo(local_directory)
@@ -93,34 +116,36 @@ def ensure_repo(
         )
 
 
-def scratch_install(path: Path, timeout: float = _DEFAULT_INSTALL_TIMEOUT) -> None:
+def scratch_install(*paths: Path, timeout: float = _DEFAULT_INSTALL_TIMEOUT) -> None:
     """
-    Install a scratch package. Make blueapi aware of a repository checked out in
-    the scratch area. Make it automatically follow code changes to that repository
-    (pending a restart). Do not install any of the package's dependencies as they
+    Install scratch packages. Make blueapi aware of repositories checked out in
+    the scratch area. Make it automatically follow code changes to those repositories
+    (pending a restart). Do not install any of the packages' dependencies as they
     may conflict with each other.
 
     Args:
-        path: Path to the checked out repository
+        paths: List of Paths to the checked out repositories
         timeout: Time to wait for installation subprocess
     """
+    if not paths:
+        return
+    args = [
+        "uv",
+        "pip",
+        "install",
+        "--no-deps",
+    ]
+    for path in paths:
+        _validate_directory(path)
+        args.extend(["-e", str(path)])
 
-    _validate_directory(path)
-
-    LOGGER.info(f"Installing {path}")
-    process = Popen(
-        [
-            "uv",
-            "pip",
-            "install",
-            "--no-deps",
-            "-e",
-            str(path),
-        ]
-    )
+    LOGGER.info("Installing packages")
+    process = Popen(args)
     process.wait(timeout=timeout)
     if process.returncode != 0:
-        raise RuntimeError(f"Failed to install {path}: Exit Code: {process.returncode}")
+        raise RuntimeError(
+            f"Failed to install packages: Exit Code: {process.returncode}"
+        )
 
 
 def _validate_root_directory(root_path: Path, required_gid: int | None) -> None:

tests/unit_tests/cli/test_scratch.py

@@ -123,7 +123,10 @@ def test_repo_cloned_if_not_found_locally(
     ensure_repo("http://example.com/foo.git", nonexistant_path)
     mock_repo.assert_not_called()
     mock_repo.clone_from.assert_called_once_with(
-        "http://example.com/foo.git", nonexistant_path, branch=None
+        "http://example.com/foo.git",
+        nonexistant_path,
+        branch=None,
+        filter="blob:none",
     )
 
 
@@ -140,7 +143,9 @@ def write_repo_files():
         with file_path.open("w") as stream:
             stream.write("foo")
 
-    mock_repo.clone_from.side_effect = lambda url, path, branch=None: write_repo_files()
+    mock_repo.clone_from.side_effect = lambda url, path, branch, filter: (
+        write_repo_files()
+    )
 
     ensure_repo("http://example.com/foo.git", repo_root)
     assert file_path.exists()
@@ -163,7 +168,10 @@ def test_cloned_repo_changes_to_new_branch(mock_repo, directory_path: Path):
     ensure_repo("http://example.com/foo.git", directory_path / "demo_branch", "demo")
 
     mock_repo.clone_from.assert_called_once_with(
-        "http://example.com/foo.git", ANY, branch="demo"
+        "http://example.com/foo.git",
+        ANY,
+        branch="demo",
+        filter="blob:none",
     )
 
 
@@ -345,8 +353,11 @@ def test_setup_scratch_iterates_repos(
 
     mock_scratch_install.assert_has_calls(
         [
-            call(directory_path_with_sgid / "foo", timeout=120.0),
-            call(directory_path_with_sgid / "bar", timeout=120.0),
+            call(
+                directory_path_with_sgid / "foo",
+                directory_path_with_sgid / "bar",
+                timeout=120.0,
+            ),
         ]
     )
 
@@ -376,7 +387,9 @@ def test_setup_scratch_continues_after_failure(
         ],
     )
     mock_ensure_repo.side_effect = [None, RuntimeError("bar"), None]
-    with pytest.raises(RuntimeError, match="bar"):
+    with pytest.raises(
+        RuntimeError, match="Failed to clone", check=lambda e: str(e.__cause__) == "bar"
+    ):
         setup_scratch(config)