feat(scratch): Make scratch init faster

src/blueapi/cli/scratch.py

@@ -3,6 +3,7 @@
 import os
 import stat
 import textwrap
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from subprocess import Popen
 
@@ -46,10 +47,27 @@ def setup_scratch(
         That is to prevent namespace clashing with the blueapi application.
         """)
             )
-    for repo in config.repositories:
-        local_directory = config.root / repo.name
-        ensure_repo(repo.remote_url, local_directory, repo.target_revision)
-        scratch_install(local_directory, timeout=install_timeout)
+
+    with ThreadPoolExecutor() as executor:
+        futures = [
+            executor.submit(
+                ensure_repo,
+                repo.remote_url,
+                config.root / repo.name,
+                repo.target_revision,
+            )
+            for repo in config.repositories
+        ]
+        for future in as_completed(futures):
+            try:
+                future.result()
+            except Exception as exc:
+                raise RuntimeError("Failed to clone repositories") from exc
+
+    scratch_install(
+        [config.root / repo.name for repo in config.repositories],
+        timeout=install_timeout,
+    )
 
 
 def ensure_repo(
@@ -69,7 +87,12 @@ def ensure_repo(
 
     if not local_directory.exists():
         LOGGER.info(f"Cloning {remote_url}")
-        Repo.clone_from(remote_url, local_directory, branch=target_revision)
+        repo = Repo.clone_from(
+            remote_url,
+            local_directory,
+            branch=target_revision,
+            multi_options=["--filter=blob:none"],
+        )
         LOGGER.info(f"Cloned {remote_url} -> {local_directory}")
     elif local_directory.is_dir():
         repo = Repo(local_directory)
@@ -93,34 +116,37 @@ def ensure_repo(
         )
 
 
-def scratch_install(path: Path, timeout: float = _DEFAULT_INSTALL_TIMEOUT) -> None:
+def scratch_install(
+    paths: list[Path], timeout: float = _DEFAULT_INSTALL_TIMEOUT
+) -> None:
     """
-    Install a scratch package. Make blueapi aware of a repository checked out in
+    Install all scratch package. Make blueapi aware of a repository checked out in
     the scratch area. Make it automatically follow code changes to that repository
     (pending a restart). Do not install any of the package's dependencies as they
     may conflict with each other.
 
     Args:
-        path: Path to the checked out repository
+        paths: List of Path to the checked out repositories
         timeout: Time to wait for installation subprocess
     """
-
-    _validate_directory(path)
-
-    LOGGER.info(f"Installing {path}")
-    process = Popen(
-        [
+    if paths:
+        args = [
             "uv",
             "pip",
             "install",
             "--no-deps",
-            "-e",
-            str(path),
         ]
-    )
-    process.wait(timeout=timeout)
-    if process.returncode != 0:
-        raise RuntimeError(f"Failed to install {path}: Exit Code: {process.returncode}")
+        for path in paths:
+            _validate_directory(path)
+            args.extend(["-e", str(path)])
+
+        LOGGER.info("Installing packages")
+        process = Popen(args)
+        process.wait(timeout=timeout)
+        if process.returncode != 0:
+            raise RuntimeError(
+                f"Failed to install packages: Exit Code: {process.returncode}"
+            )
 
 
 def _validate_root_directory(root_path: Path, required_gid: int | None) -> None:

tests/unit_tests/cli/test_scratch.py

@@ -63,7 +63,7 @@ def test_scratch_install_installs_path(
     mock_process.returncode = 0
     mock_popen.return_value = mock_process
 
-    scratch_install(directory_path_with_sgid, timeout=1.0)
+    scratch_install([directory_path_with_sgid], timeout=1.0)
 
     mock_popen.assert_called_once_with(
         ["uv", "pip", "install", "--no-deps", "-e", str(directory_path_with_sgid)]
@@ -72,12 +72,12 @@ def test_scratch_install_installs_path(
 
 def test_scratch_install_fails_on_file(file_path: Path):
     with pytest.raises(KeyError):
-        scratch_install(file_path, timeout=1.0)
+        scratch_install([file_path], timeout=1.0)
 
 
 def test_scratch_install_fails_on_nonexistant_path(nonexistant_path: Path):
     with pytest.raises(KeyError):
-        scratch_install(nonexistant_path, timeout=1.0)
+        scratch_install([nonexistant_path], timeout=1.0)
 
 
 @patch("blueapi.cli.scratch.Popen")
@@ -92,7 +92,7 @@ def test_scratch_install_fails_on_non_zero_exit_code(
     mock_popen.return_value = mock_process
 
     with pytest.raises(RuntimeError):
-        scratch_install(directory_path_with_sgid, timeout=1.0)
+        scratch_install([directory_path_with_sgid], timeout=1.0)
 
 
 @patch("blueapi.cli.scratch.Repo")
@@ -123,7 +123,10 @@ def test_repo_cloned_if_not_found_locally(
     ensure_repo("http://example.com/foo.git", nonexistant_path)
     mock_repo.assert_not_called()
     mock_repo.clone_from.assert_called_once_with(
-        "http://example.com/foo.git", nonexistant_path, branch=None
+        "http://example.com/foo.git",
+        nonexistant_path,
+        branch=None,
+        multi_options=["--filter=blob:none"],
     )
 
 
@@ -140,7 +143,9 @@ def write_repo_files():
         with file_path.open("w") as stream:
             stream.write("foo")
 
-    mock_repo.clone_from.side_effect = lambda url, path, branch=None: write_repo_files()
+    mock_repo.clone_from.side_effect = lambda url, path, branch, multi_options: (
+        write_repo_files()
+    )
 
     ensure_repo("http://example.com/foo.git", repo_root)
     assert file_path.exists()
@@ -163,7 +168,10 @@ def test_cloned_repo_changes_to_new_branch(mock_repo, directory_path: Path):
     ensure_repo("http://example.com/foo.git", directory_path / "demo_branch", "demo")
 
     mock_repo.clone_from.assert_called_once_with(
-        "http://example.com/foo.git", ANY, branch="demo"
+        "http://example.com/foo.git",
+        ANY,
+        branch="demo",
+        multi_options=["--filter=blob:none"],
     )
 
 
@@ -345,8 +353,10 @@ def test_setup_scratch_iterates_repos(
 
     mock_scratch_install.assert_has_calls(
         [
-            call(directory_path_with_sgid / "foo", timeout=120.0),
-            call(directory_path_with_sgid / "bar", timeout=120.0),
+            call(
+                [directory_path_with_sgid / "foo", directory_path_with_sgid / "bar"],
+                timeout=120.0,
+            ),
         ]
     )
 
@@ -376,7 +386,9 @@ def test_setup_scratch_continues_after_failure(
         ],
     )
     mock_ensure_repo.side_effect = [None, RuntimeError("bar"), None]
-    with pytest.raises(RuntimeError, match="bar"):
+    with pytest.raises(
+        RuntimeError, match="Failed to clone", check=lambda e: str(e.__cause__) == "bar"
+    ):
         setup_scratch(config)