Refactor metadata_manager to move to versions provider based model

README.md

@@ -137,9 +137,9 @@ The default directory structure is established as follows:
         ├── ardupilot            (used by the web component)
         ├── artifacts
         ├── configs
-        |   └── remotes.json     (auto-generated, see examples/remotes.json.sample)
+        |   └── remotes.json     (optional, see examples/remotes.json.sample)
         ├── secrets
-        |   └── reload_token     (optional)
+        |   └── admin_token      (optional)
         ├── tmp
             └── ardupilot        (used by the builder component)
 ```

builder/requirements.txt

@@ -1,3 +1,5 @@
 jsonschema==4.20.0
 redis==5.2.1
 dill==0.3.8
+requests==2.31.0
+packaging==26.2

docker-compose.yml

@@ -20,7 +20,7 @@ services:
       CBS_LOG_LEVEL: ${CBS_LOG_LEVEL:-INFO}
       CBS_ENABLE_INBUILT_BUILDER: 0
       CBS_GITHUB_ACCESS_TOKEN: ${CBS_GITHUB_ACCESS_TOKEN}
-      CBS_REMOTES_RELOAD_TOKEN: ${CBS_REMOTES_RELOAD_TOKEN}
+      CBS_ADMIN_TOKEN: ${CBS_ADMIN_TOKEN}
       PYTHONPATH: /app
       CBS_BUILD_TIMEOUT_SEC: ${CBS_BUILD_TIMEOUT_SEC:-900}
       FORWARDED_ALLOW_IPS: ${FORWARDED_ALLOW_IPS:-*}

examples/remotes.json.sample

@@ -4,7 +4,7 @@
     "url": "https://github.com/ardupilot/ardupilot.git",
     "vehicles": [
       {
-        "name": "Copter",
+        "id": "copter",
         "releases": [
           {
             "release_type": "latest",
@@ -21,7 +21,7 @@
     "url": "https://github.com/cool-user-007/ardupilot.git",
     "vehicles": [
       {
-        "name": "Copter",
+        "id": "copter",
         "releases": [
           {
             "release_type": "Custom",
@@ -37,7 +37,7 @@
     "url": "https://github.com/ardupilot/ardupilot.git",
     "vehicles": [
       {
-        "name": "Copter",
+        "id": "copter",
         "releases": [
           {
             "release_type": "latest",
@@ -55,4 +55,4 @@
       }
     ]
   }
-]
+]

metadata_manager/__init__.py

@@ -1,21 +1,39 @@
-from .versions_fetcher import (
-    VersionsFetcher,
+from .ap_src_meta_fetcher import APSourceMetadataFetcher
+from .firmware_server import ManifestJson, ManifestFetchError, ReleaseRecord
+from .firmware_server.client import ManifestClient
+from .firmware_server.index import ManifestIndex
+from .vehicles_manager import DEFAULT_VEHICLES, Vehicle, VehiclesManager
+from .versions_manager import (
+    DEFAULT_WHITELISTED_FORK_REMOTES,
+    ForkRemoteSpec,
+    ManifestJsonVersionsProvider,
     RemoteInfo,
-)
-
-from .ap_src_meta_fetcher import (
-    APSourceMetadataFetcher,
-)
-
-from .vehicles_manager import (
-    VehiclesManager,
-    Vehicle,
+    RemotesJsonVersionsProvider,
+    VersionInfo,
+    VersionsManager,
+    VersionsProvider,
+    WhitelistedForkTagVersionsProvider,
+    build_default_providers,
 )
 
 __all__ = [
     "APSourceMetadataFetcher",
-    "VersionsFetcher",
+    "DEFAULT_VEHICLES",
+    "DEFAULT_WHITELISTED_FORK_REMOTES",
+    "ForkRemoteSpec",
+    "ManifestClient",
+    "ManifestFetchError",
+    "ManifestIndex",
+    "ManifestJson",
+    "ManifestJsonVersionsProvider",
+    "ReleaseRecord",
     "RemoteInfo",
-    "VehiclesManager",
+    "RemotesJsonVersionsProvider",
     "Vehicle",
+    "VersionInfo",
+    "VersionsManager",
+    "VersionsProvider",
+    "VehiclesManager",
+    "WhitelistedForkTagVersionsProvider",
+    "build_default_providers",
 ]

metadata_manager/ap_src_meta_fetcher.py

@@ -505,7 +505,7 @@ def get_board_defaults_from_fw_server(
 
         # Heli builds are stored under a separate folder
         artifacts_subdir = board_id
-        if vehicle_id == "Heli":
+        if vehicle_id == "heli":
             artifacts_subdir += "-heli"
 
         features_txt_url = f"{artifacts_url}/{artifacts_subdir}/features.txt"

metadata_manager/firmware_server/__init__.py

@@ -0,0 +1,9 @@
+from .manifest import ManifestJson
+from .models import ReleaseRecord
+from .exceptions import ManifestFetchError
+
+__all__ = [
+    "ManifestJson",
+    "ManifestFetchError",
+    "ReleaseRecord",
+]

metadata_manager/firmware_server/client.py

@@ -0,0 +1,137 @@
+import json
+import logging
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+import requests
+
+from .exceptions import ManifestFetchError
+
+
+@dataclass
+class _CacheMeta:
+    etag: Optional[str] = None
+    last_modified: Optional[str] = None
+    fetched_at: Optional[str] = None
+
+    def to_dict(self) -> dict:
+        return {
+            "etag": self.etag,
+            "last_modified": self.last_modified,
+            "fetched_at": self.fetched_at,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "_CacheMeta":
+        return cls(
+            etag=data.get("etag"),
+            last_modified=data.get("last_modified"),
+            fetched_at=data.get("fetched_at"),
+        )
+
+
+class ManifestClient:
+    """Fetch and cache the ArduPilot firmware manifest.json file."""
+
+    def __init__(
+        self,
+        url: str,
+        cache_dir: str,
+        timeout: int = 120,
+        user_agent: str = "CustomBuild/1.0",
+    ):
+        self.url = url
+        self.cache_dir = Path(cache_dir)
+        self.cache_path = self.cache_dir / "manifest.json"
+        self.meta_path = self.cache_dir / "manifest.json.meta"
+        self.timeout = timeout
+        self.user_agent = user_agent
+        self.logger = logging.getLogger(__name__)
+
+    def fetch_raw(self) -> bytes:
+        headers = {"User-Agent": self.user_agent}
+        meta = self._read_meta() if self._has_cache() else _CacheMeta()
+
+        if meta.etag:
+            headers["If-None-Match"] = meta.etag
+        if meta.last_modified:
+            headers["If-Modified-Since"] = meta.last_modified
+
+        try:
+            response = requests.get(
+                self.url,
+                headers=headers,
+                timeout=self.timeout,
+            )
+        except requests.RequestException as exc:
+            return self._fallback_or_raise(exc)
+
+        if response.status_code == 304:
+            self.logger.info("Manifest not modified (304), using cache")
+            self._touch_fetched_at(self._now_iso())
+            return self._read_cache_bytes()
+
+        if response.status_code != 200:
+            return self._fallback_or_raise(
+                ManifestFetchError(
+                    f"Manifest fetch failed with status {response.status_code}"
+                )
+            )
+
+        raw = response.content
+        self._write_cache(
+            raw,
+            _CacheMeta(
+                etag=response.headers.get("ETag"),
+                last_modified=response.headers.get("Last-Modified"),
+                fetched_at=self._now_iso(),
+            ),
+        )
+        self.logger.info("Downloaded manifest (%d bytes)", len(raw))
+        return raw
+
+    def fetch(self) -> dict:
+        return json.loads(self.fetch_raw().decode("utf-8"))
+
+    def _has_cache(self) -> bool:
+        return self.cache_path.is_file()
+
+    def _read_cache_bytes(self) -> bytes:
+        return self.cache_path.read_bytes()
+
+    def _read_meta(self) -> _CacheMeta:
+        if not self.meta_path.is_file():
+            return _CacheMeta()
+        return _CacheMeta.from_dict(
+            json.loads(self.meta_path.read_text(encoding="utf-8"))
+        )
+
+    def _write_cache(self, raw: bytes, meta: _CacheMeta) -> None:
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.cache_path.write_bytes(raw)
+        self.meta_path.write_text(
+            json.dumps(meta.to_dict(), indent=2),
+            encoding="utf-8",
+        )
+
+    def _touch_fetched_at(self, fetched_at: str) -> None:
+        meta = self._read_meta()
+        meta.fetched_at = fetched_at
+        self.meta_path.write_text(
+            json.dumps(meta.to_dict(), indent=2),
+            encoding="utf-8",
+        )
+
+    def _fallback_or_raise(self, exc: Exception) -> bytes:
+        if self._has_cache():
+            self.logger.warning(
+                "Manifest fetch failed (%s), using stale cache", exc
+            )
+            return self._read_cache_bytes()
+        raise ManifestFetchError(str(exc)) from exc
+
+    @staticmethod
+    def _now_iso() -> str:
+        return datetime.now(timezone.utc).isoformat()

metadata_manager/firmware_server/exceptions.py

@@ -0,0 +1,2 @@
+class ManifestFetchError(Exception):
+    """Raised when the firmware manifest cannot be fetched and no cache exists."""