Merge branch 'main' into chore/improve-default-error-handling-1379-1409

Author: Alexj9837

.github/workflows/backstage.yml

@@ -17,6 +17,6 @@ jobs:
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
       - name: check catalog-info.yaml file
-        uses: RoadieHQ/backstage-entity-validator@d39db2335a810aed3175c5ba9b86b89b7c7286c1 # v0.5.0
+        uses: RoadieHQ/backstage-entity-validator@421bdcd5e3e400ef1913eb16d4f9dacdc5016801 # v0.6.0
         with:
           path: "./catalog-info.yaml"

helm/blueapi/templates/statefulset.yaml

@@ -101,7 +101,7 @@ spec:
             cp -r /app/.venv/* /artefacts
         env:
         - name: UV_CACHE_DIR
-          value: {{ (.Values.worker.scratch).root }}/.uv-cache
+          value: /app/.uv-cache
         volumeMounts:
           - name: init-config
             mountPath: "/config"
@@ -205,7 +205,7 @@ spec:
                 name: {{ include "blueapi.fullname" . }}-otel-config
           env:
             - name: UV_CACHE_DIR
-              value: {{ (.Values.worker.scratch).root }}/.uv-cache
+              value: /app/.uv-cache
             {{- if .Values.debug.enabled }}
             - name: DEBUG_MODE
               value: "ON"           

src/blueapi/cli/scratch.py

@@ -3,6 +3,7 @@
 import os
 import stat
 import textwrap
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from subprocess import Popen
 
@@ -46,10 +47,27 @@ def setup_scratch(
         That is to prevent namespace clashing with the blueapi application.
         """)
             )
-    for repo in config.repositories:
-        local_directory = config.root / repo.name
-        ensure_repo(repo.remote_url, local_directory, repo.target_revision)
-        scratch_install(local_directory, timeout=install_timeout)
+
+    with ThreadPoolExecutor() as executor:
+        futures = [
+            executor.submit(
+                ensure_repo,
+                repo.remote_url,
+                config.root / repo.name,
+                repo.target_revision,
+            )
+            for repo in config.repositories
+        ]
+        for future in as_completed(futures):
+            try:
+                future.result()
+            except Exception as exc:
+                raise RuntimeError("Failed to clone repositories") from exc
+
+    scratch_install(
+        *(config.root / repo.name for repo in config.repositories),
+        timeout=install_timeout,
+    )
 
 
 def ensure_repo(
@@ -69,7 +87,12 @@ def ensure_repo(
 
     if not local_directory.exists():
         LOGGER.info(f"Cloning {remote_url}")
-        Repo.clone_from(remote_url, local_directory, branch=target_revision)
+        Repo.clone_from(
+            remote_url,
+            local_directory,
+            branch=target_revision,
+            filter="blob:none",
+        )
         LOGGER.info(f"Cloned {remote_url} -> {local_directory}")
     elif local_directory.is_dir():
         repo = Repo(local_directory)
@@ -93,34 +116,36 @@ def ensure_repo(
         )
 
 
-def scratch_install(path: Path, timeout: float = _DEFAULT_INSTALL_TIMEOUT) -> None:
+def scratch_install(*paths: Path, timeout: float = _DEFAULT_INSTALL_TIMEOUT) -> None:
     """
-    Install a scratch package. Make blueapi aware of a repository checked out in
-    the scratch area. Make it automatically follow code changes to that repository
-    (pending a restart). Do not install any of the package's dependencies as they
+    Install scratch packages. Make blueapi aware of repositories checked out in
+    the scratch area. Make it automatically follow code changes to those repositories
+    (pending a restart). Do not install any of the packages' dependencies as they
     may conflict with each other.
 
     Args:
-        path: Path to the checked out repository
+        paths: List of Paths to the checked out repositories
         timeout: Time to wait for installation subprocess
     """
+    if not paths:
+        return
+    args = [
+        "uv",
+        "pip",
+        "install",
+        "--no-deps",
+    ]
+    for path in paths:
+        _validate_directory(path)
+        args.extend(["-e", str(path)])
 
-    _validate_directory(path)
-
-    LOGGER.info(f"Installing {path}")
-    process = Popen(
-        [
-            "uv",
-            "pip",
-            "install",
-            "--no-deps",
-            "-e",
-            str(path),
-        ]
-    )
+    LOGGER.info("Installing packages")
+    process = Popen(args)
     process.wait(timeout=timeout)
     if process.returncode != 0:
-        raise RuntimeError(f"Failed to install {path}: Exit Code: {process.returncode}")
+        raise RuntimeError(
+            f"Failed to install packages: Exit Code: {process.returncode}"
+        )
 
 
 def _validate_root_directory(root_path: Path, required_gid: int | None) -> None:

tests/system_tests/compose.yaml

@@ -6,6 +6,8 @@ services:
     image: ghcr.io/diamondlightsource/numtracker:1.0.2
     ports:
       - "8406:8000"
+    security_opt:
+      - label=disable
 
   rabbitmq:
     image: docker.io/rabbitmq:4.0-management
@@ -18,6 +20,8 @@ services:
       - type: bind
         source: ./services/rabbitmq_plugins
         target: /etc/rabbitmq/enabled_plugins
+    security_opt:
+      - label=disable
 
   keycloak:
     image: keycloak/keycloak:26.4
@@ -38,6 +42,8 @@ services:
       timeout: 5s
       retries: 10
       start_period: 30s
+    security_opt:
+      - label=disable
 
   tiled:
     image: ghcr.io/bluesky/tiled:0.2.4
@@ -50,6 +56,8 @@ services:
     depends_on:
       keycloak:
         condition: service_healthy
+    security_opt:
+      - label=disable
 
   opa:
     image: openpolicyagent/opa:edge-static-debug
@@ -59,6 +67,8 @@ services:
     environment:
       - ISSUER=http://localhost:8081/realms/master
     entrypoint: "sh /mnt/entrypoint.sh"
+    security_opt:
+      - label=disable
 
   blueapi-oauth2-proxy:
     network_mode: host
@@ -73,6 +83,8 @@ services:
     depends_on:
       keycloak:
         condition: service_healthy
+    security_opt:
+      - label=disable
 
   tiled-oauth2-proxy:
     network_mode: host
@@ -87,3 +99,5 @@ services:
     depends_on:
       keycloak:
         condition: service_healthy
+    security_opt:
+      - label=disable

tests/unit_tests/cli/test_scratch.py

@@ -123,7 +123,10 @@ def test_repo_cloned_if_not_found_locally(
     ensure_repo("http://example.com/foo.git", nonexistant_path)
     mock_repo.assert_not_called()
     mock_repo.clone_from.assert_called_once_with(
-        "http://example.com/foo.git", nonexistant_path, branch=None
+        "http://example.com/foo.git",
+        nonexistant_path,
+        branch=None,
+        filter="blob:none",
     )
 
 
@@ -140,7 +143,9 @@ def write_repo_files():
         with file_path.open("w") as stream:
             stream.write("foo")
 
-    mock_repo.clone_from.side_effect = lambda url, path, branch=None: write_repo_files()
+    mock_repo.clone_from.side_effect = lambda url, path, branch, filter: (
+        write_repo_files()
+    )
 
     ensure_repo("http://example.com/foo.git", repo_root)
     assert file_path.exists()
@@ -163,7 +168,10 @@ def test_cloned_repo_changes_to_new_branch(mock_repo, directory_path: Path):
     ensure_repo("http://example.com/foo.git", directory_path / "demo_branch", "demo")
 
     mock_repo.clone_from.assert_called_once_with(
-        "http://example.com/foo.git", ANY, branch="demo"
+        "http://example.com/foo.git",
+        ANY,
+        branch="demo",
+        filter="blob:none",
     )
 
 
@@ -345,8 +353,11 @@ def test_setup_scratch_iterates_repos(
 
     mock_scratch_install.assert_has_calls(
         [
-            call(directory_path_with_sgid / "foo", timeout=120.0),
-            call(directory_path_with_sgid / "bar", timeout=120.0),
+            call(
+                directory_path_with_sgid / "foo",
+                directory_path_with_sgid / "bar",
+                timeout=120.0,
+            ),
         ]
     )
 
@@ -376,7 +387,9 @@ def test_setup_scratch_continues_after_failure(
         ],
     )
     mock_ensure_repo.side_effect = [None, RuntimeError("bar"), None]
-    with pytest.raises(RuntimeError, match="bar"):
+    with pytest.raises(
+        RuntimeError, match="Failed to clone", check=lambda e: str(e.__cause__) == "bar"
+    ):
         setup_scratch(config)