From db4eb0c36a09fb9ba38c3c472ce9a3c0d8684483 Mon Sep 17 00:00:00 2001 From: Shiv Tyagi Date: Sat, 11 Jul 2026 09:31:10 +0530 Subject: [PATCH 1/8] metadata_manager: refactor to a provider based versions manager --- metadata_manager/__init__.py | 44 ++- metadata_manager/ap_src_meta_fetcher.py | 2 +- metadata_manager/firmware_server/__init__.py | 9 + metadata_manager/firmware_server/client.py | 137 +++++++ .../firmware_server/exceptions.py | 2 + metadata_manager/firmware_server/index.py | 207 ++++++++++ metadata_manager/firmware_server/manifest.py | 40 ++ metadata_manager/firmware_server/models.py | 10 + metadata_manager/remotes.schema.json | 6 +- metadata_manager/vehicles_manager.py | 10 - metadata_manager/versions_fetcher.py | 358 ------------------ metadata_manager/versions_manager/__init__.py | 23 ++ metadata_manager/versions_manager/manager.py | 153 ++++++++ metadata_manager/versions_manager/models.py | 70 ++++ .../versions_manager/providers.py | 318 ++++++++++++++++ 15 files changed, 1004 insertions(+), 385 deletions(-) create mode 100644 metadata_manager/firmware_server/__init__.py create mode 100644 metadata_manager/firmware_server/client.py create mode 100644 metadata_manager/firmware_server/exceptions.py create mode 100644 metadata_manager/firmware_server/index.py create mode 100644 metadata_manager/firmware_server/manifest.py create mode 100644 metadata_manager/firmware_server/models.py delete mode 100644 metadata_manager/versions_fetcher.py create mode 100644 metadata_manager/versions_manager/__init__.py create mode 100644 metadata_manager/versions_manager/manager.py create mode 100644 metadata_manager/versions_manager/models.py create mode 100644 metadata_manager/versions_manager/providers.py diff --git a/metadata_manager/__init__.py b/metadata_manager/__init__.py index ce0ef370..2209f6e4 100644 --- a/metadata_manager/__init__.py +++ b/metadata_manager/__init__.py @@ -1,21 +1,39 @@ -from .versions_fetcher import ( - VersionsFetcher, +from .ap_src_meta_fetcher import APSourceMetadataFetcher +from .firmware_server import ManifestJson, ManifestFetchError, ReleaseRecord +from .firmware_server.client import ManifestClient +from .firmware_server.index import ManifestIndex +from .vehicles_manager import DEFAULT_VEHICLES, Vehicle, VehiclesManager +from .versions_manager import ( + DEFAULT_WHITELISTED_FORK_REMOTES, + ForkRemoteSpec, + ManifestJsonVersionsProvider, RemoteInfo, -) - -from .ap_src_meta_fetcher import ( - APSourceMetadataFetcher, -) - -from .vehicles_manager import ( - VehiclesManager, - Vehicle, + RemotesJsonVersionsProvider, + VersionInfo, + VersionsManager, + VersionsProvider, + WhitelistedForkTagVersionsProvider, + build_default_providers, ) __all__ = [ "APSourceMetadataFetcher", - "VersionsFetcher", + "DEFAULT_VEHICLES", + "DEFAULT_WHITELISTED_FORK_REMOTES", + "ForkRemoteSpec", + "ManifestClient", + "ManifestFetchError", + "ManifestIndex", + "ManifestJson", + "ManifestJsonVersionsProvider", + "ReleaseRecord", "RemoteInfo", - "VehiclesManager", + "RemotesJsonVersionsProvider", "Vehicle", + "VersionInfo", + "VersionsManager", + "VersionsProvider", + "VehiclesManager", + "WhitelistedForkTagVersionsProvider", + "build_default_providers", ] diff --git a/metadata_manager/ap_src_meta_fetcher.py b/metadata_manager/ap_src_meta_fetcher.py index 33bfff0a..084ab343 100644 --- a/metadata_manager/ap_src_meta_fetcher.py +++ b/metadata_manager/ap_src_meta_fetcher.py @@ -505,7 +505,7 @@ def get_board_defaults_from_fw_server( # Heli builds are stored under a separate folder artifacts_subdir = board_id - if vehicle_id == "Heli": + if vehicle_id == "heli": artifacts_subdir += "-heli" features_txt_url = f"{artifacts_url}/{artifacts_subdir}/features.txt" diff --git a/metadata_manager/firmware_server/__init__.py b/metadata_manager/firmware_server/__init__.py new file mode 100644 index 00000000..0cd63054 --- /dev/null +++ b/metadata_manager/firmware_server/__init__.py @@ -0,0 +1,9 @@ +from .manifest import ManifestJson +from .models import ReleaseRecord +from .exceptions import ManifestFetchError + +__all__ = [ + "ManifestJson", + "ManifestFetchError", + "ReleaseRecord", +] diff --git a/metadata_manager/firmware_server/client.py b/metadata_manager/firmware_server/client.py new file mode 100644 index 00000000..f77f18a6 --- /dev/null +++ b/metadata_manager/firmware_server/client.py @@ -0,0 +1,137 @@ +import json +import logging +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +import requests + +from .exceptions import ManifestFetchError + + +@dataclass +class _CacheMeta: + etag: Optional[str] = None + last_modified: Optional[str] = None + fetched_at: Optional[str] = None + + def to_dict(self) -> dict: + return { + "etag": self.etag, + "last_modified": self.last_modified, + "fetched_at": self.fetched_at, + } + + @classmethod + def from_dict(cls, data: dict) -> "_CacheMeta": + return cls( + etag=data.get("etag"), + last_modified=data.get("last_modified"), + fetched_at=data.get("fetched_at"), + ) + + +class ManifestClient: + """Fetch and cache the ArduPilot firmware manifest.json file.""" + + def __init__( + self, + url: str, + cache_dir: str, + timeout: int = 120, + user_agent: str = "CustomBuild/1.0", + ): + self.url = url + self.cache_dir = Path(cache_dir) + self.cache_path = self.cache_dir / "manifest.json" + self.meta_path = self.cache_dir / "manifest.json.meta" + self.timeout = timeout + self.user_agent = user_agent + self.logger = logging.getLogger(__name__) + + def fetch_raw(self) -> bytes: + headers = {"User-Agent": self.user_agent} + meta = self._read_meta() if self._has_cache() else _CacheMeta() + + if meta.etag: + headers["If-None-Match"] = meta.etag + if meta.last_modified: + headers["If-Modified-Since"] = meta.last_modified + + try: + response = requests.get( + self.url, + headers=headers, + timeout=self.timeout, + ) + except requests.RequestException as exc: + return self._fallback_or_raise(exc) + + if response.status_code == 304: + self.logger.info("Manifest not modified (304), using cache") + self._touch_fetched_at(self._now_iso()) + return self._read_cache_bytes() + + if response.status_code != 200: + return self._fallback_or_raise( + ManifestFetchError( + f"Manifest fetch failed with status {response.status_code}" + ) + ) + + raw = response.content + self._write_cache( + raw, + _CacheMeta( + etag=response.headers.get("ETag"), + last_modified=response.headers.get("Last-Modified"), + fetched_at=self._now_iso(), + ), + ) + self.logger.info("Downloaded manifest (%d bytes)", len(raw)) + return raw + + def fetch(self) -> dict: + return json.loads(self.fetch_raw().decode("utf-8")) + + def _has_cache(self) -> bool: + return self.cache_path.is_file() + + def _read_cache_bytes(self) -> bytes: + return self.cache_path.read_bytes() + + def _read_meta(self) -> _CacheMeta: + if not self.meta_path.is_file(): + return _CacheMeta() + return _CacheMeta.from_dict( + json.loads(self.meta_path.read_text(encoding="utf-8")) + ) + + def _write_cache(self, raw: bytes, meta: _CacheMeta) -> None: + self.cache_dir.mkdir(parents=True, exist_ok=True) + self.cache_path.write_bytes(raw) + self.meta_path.write_text( + json.dumps(meta.to_dict(), indent=2), + encoding="utf-8", + ) + + def _touch_fetched_at(self, fetched_at: str) -> None: + meta = self._read_meta() + meta.fetched_at = fetched_at + self.meta_path.write_text( + json.dumps(meta.to_dict(), indent=2), + encoding="utf-8", + ) + + def _fallback_or_raise(self, exc: Exception) -> bytes: + if self._has_cache(): + self.logger.warning( + "Manifest fetch failed (%s), using stale cache", exc + ) + return self._read_cache_bytes() + raise ManifestFetchError(str(exc)) from exc + + @staticmethod + def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() diff --git a/metadata_manager/firmware_server/exceptions.py b/metadata_manager/firmware_server/exceptions.py new file mode 100644 index 00000000..ad43f7f6 --- /dev/null +++ b/metadata_manager/firmware_server/exceptions.py @@ -0,0 +1,2 @@ +class ManifestFetchError(Exception): + """Raised when the firmware manifest cannot be fetched and no cache exists.""" diff --git a/metadata_manager/firmware_server/index.py b/metadata_manager/firmware_server/index.py new file mode 100644 index 00000000..109780f6 --- /dev/null +++ b/metadata_manager/firmware_server/index.py @@ -0,0 +1,207 @@ +import logging +import re +from collections import defaultdict +from typing import Optional + +from packaging.version import InvalidVersion, Version + +from .models import ReleaseRecord + +# Minimum version for a vehicle to expose from manifest entries. +MIN_VERSION_BY_VEHICLE_ID = { + "copter": "4.3", + "plane": "4.3", + "rover": "4.3", + "sub": "4.3", + "tracker": "4.3", + "blimp": "4.3", + "heli": "4.3", + "ap-periph": "1.8.1", +} + + +def vehicle_id_for_manifest_entry(entry: dict) -> Optional[str]: + match entry.get("vehicletype", ""): + case "Copter": + return "heli" if entry.get("mav-type") == "HELICOPTER" else "copter" + case "Plane": + return "plane" + case "Rover": + return "rover" + case "Sub": + return "sub" + case "Blimp": + return "blimp" + case "AntennaTracker": + return "tracker" + case "AP_Periph": + return "ap-periph" + case _: + return None + + +def _should_skip_version( + vehicle_id: str, release_type: str, manifest_version: str +) -> bool: + if release_type == "latest": + return False + + min_version = MIN_VERSION_BY_VEHICLE_ID.get(vehicle_id) + if not min_version: + return False + + try: + return Version(manifest_version) < Version(min_version) + except InvalidVersion: + return True + + +def map_manifest_release_type(mav_firmware_version_type: str) -> Optional[str]: + if not mav_firmware_version_type: + return None + if mav_firmware_version_type.startswith("STABLE-"): + return "stable" + if mav_firmware_version_type == "OFFICIAL": + return "stable" + if mav_firmware_version_type == "BETA": + return "beta" + if mav_firmware_version_type == "DEV": + return "latest" + return mav_firmware_version_type.lower() + + +def parse_artifacts_base_url(url: str) -> Optional[str]: + match = re.match( + r"(https://firmware\.ardupilot\.org/[^/]+/[^/]+)/", + url or "", + ) + if match: + return match.group(1) + return None + + +def _release_fields_from_entry( + entry: dict, +) -> Optional[tuple[str, str, str, str, str]]: + vehicle_id = vehicle_id_for_manifest_entry(entry) + if vehicle_id is None: + return None + + release_type = map_manifest_release_type( + entry.get("mav-firmware-version-type", "") + ) + manifest_version = entry.get("mav-firmware-version") + if not release_type or not manifest_version: + return None + + if _should_skip_version(vehicle_id, release_type, manifest_version): + return None + + base_url = parse_artifacts_base_url(entry.get("url", "")) + if not base_url: + return None + + git_sha = entry.get("git-sha") + if not git_sha: + return None + + version_number = "NA" if release_type == "latest" else manifest_version + return vehicle_id, release_type, version_number, base_url, git_sha + + +def _record_release_meta( + release_meta: dict[tuple, dict], + vehicle_id: str, + release_type: str, + version_number: str, + base_url: str, + git_sha: str, + logger: logging.Logger, +) -> None: + key = (vehicle_id, release_type, version_number) + if key not in release_meta: + release_meta[key] = { + "vehicle_id": vehicle_id, + "release_type": release_type, + "version_number": version_number, + "ap_build_artifacts_url": base_url, + "git_sha": git_sha, + } + return + + if release_meta[key]["git_sha"] != git_sha: + logger.debug( + "Conflicting git-sha for %s: keeping %s, ignoring %s", + key, + release_meta[key]["git_sha"][:8], + git_sha[:8], + ) + + +def _releases_from_meta( + release_meta: dict[tuple, dict], +) -> dict[str, list[ReleaseRecord]]: + releases_by_vehicle: dict[str, list[ReleaseRecord]] = defaultdict(list) + for meta in release_meta.values(): + releases_by_vehicle[meta["vehicle_id"]].append( + ReleaseRecord( + vehicle_id=meta["vehicle_id"], + release_type=meta["release_type"], + version_number=meta["version_number"], + commit_reference=meta["git_sha"], + ap_build_artifacts_url=meta["ap_build_artifacts_url"], + ) + ) + + for vehicle_id in releases_by_vehicle: + releases_by_vehicle[vehicle_id].sort( + key=lambda r: ( + 0 if r.release_type == "latest" else 1, + r.release_type, + _version_sort_key(r.version_number), + ) + ) + + return dict(releases_by_vehicle) + + +class ManifestIndex: + """In-memory index built from a parsed manifest document.""" + + def __init__(self, releases_by_vehicle: dict[str, list[ReleaseRecord]]): + self.releases_by_vehicle = releases_by_vehicle + + @classmethod + def build(cls, manifest: dict) -> "ManifestIndex": + logger = logging.getLogger(__name__) + release_meta: dict[tuple, dict] = {} + + for entry in manifest.get("firmware") or []: + fields = _release_fields_from_entry(entry) + if fields is None: + continue + vehicle_id, release_type, version_number, base_url, git_sha = fields + _record_release_meta( + release_meta, + vehicle_id, + release_type, + version_number, + base_url, + git_sha, + logger, + ) + + return cls(_releases_from_meta(release_meta)) + + def get_releases(self, vehicle_id: str) -> list[ReleaseRecord]: + return list(self.releases_by_vehicle.get(vehicle_id, [])) + + +def _version_sort_key(version_number: str) -> tuple: + if version_number == "NA": + return (0,) + try: + parsed = Version(version_number) + return (1, parsed.major, parsed.minor, parsed.micro) + except InvalidVersion: + return (2, version_number) diff --git a/metadata_manager/firmware_server/manifest.py b/metadata_manager/firmware_server/manifest.py new file mode 100644 index 00000000..d96f7c75 --- /dev/null +++ b/metadata_manager/firmware_server/manifest.py @@ -0,0 +1,40 @@ +import logging +from typing import Optional + +from .client import ManifestClient +from .exceptions import ManifestFetchError +from .index import ManifestIndex + + +class ManifestJson: + """Facade over cached manifest data for version and firmware-server lookups.""" + + def __init__(self, url: str, cache_dir: str): + self._client = ManifestClient(url=url, cache_dir=cache_dir) + self._index: Optional[ManifestIndex] = None + self.logger = logging.getLogger(__name__) + + @property + def is_available(self) -> bool: + return self._index is not None + + def refresh(self) -> None: + try: + manifest = self._client.fetch() + self._index = ManifestIndex.build(manifest) + self.logger.info( + "Manifest index built with releases for %d vehicles", + len(self._index.releases_by_vehicle), + ) + except ManifestFetchError: + if self._index is not None: + self.logger.warning( + "Manifest refresh failed, continuing with stale index" + ) + return + raise + + def get_releases(self, vehicle_id: str) -> list: + if not self.is_available: + return [] + return self._index.get_releases(vehicle_id) diff --git a/metadata_manager/firmware_server/models.py b/metadata_manager/firmware_server/models.py new file mode 100644 index 00000000..3ce2455d --- /dev/null +++ b/metadata_manager/firmware_server/models.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ReleaseRecord: + vehicle_id: str + release_type: str + version_number: str + commit_reference: str + ap_build_artifacts_url: str diff --git a/metadata_manager/remotes.schema.json b/metadata_manager/remotes.schema.json index 8e385325..d6e676b9 100644 --- a/metadata_manager/remotes.schema.json +++ b/metadata_manager/remotes.schema.json @@ -21,9 +21,9 @@ "type": "object", "description": "Vehicle object", "properties": { - "name": { + "id": { "type": "string", - "description": "Name of vehicle" + "description": "Vehicle id" }, "releases": { "type": "array", @@ -55,7 +55,7 @@ } }, "required": [ - "name", + "id", "releases" ] } diff --git a/metadata_manager/vehicles_manager.py b/metadata_manager/vehicles_manager.py index 6641a1fc..ddf4bb52 100644 --- a/metadata_manager/vehicles_manager.py +++ b/metadata_manager/vehicles_manager.py @@ -3,13 +3,11 @@ def __init__(self, id: str, name: str, ap_source_subdir: str, - fw_server_vehicle_sdir: str, waf_build_command: str, ) -> None: self.id = id self.name = name self.ap_source_subdir = ap_source_subdir - self.fw_server_vehicle_sdir = fw_server_vehicle_sdir self.waf_build_command = waf_build_command def __eq__(self, other): @@ -27,56 +25,48 @@ def __hash__(self): id="copter", name="Copter", ap_source_subdir="ArduCopter", - fw_server_vehicle_sdir="Copter", waf_build_command="copter" ), Vehicle( id="plane", name="Plane", ap_source_subdir="ArduPlane", - fw_server_vehicle_sdir="Plane", waf_build_command="plane" ), Vehicle( id="rover", name="Rover", ap_source_subdir="Rover", - fw_server_vehicle_sdir="Rover", waf_build_command="rover" ), Vehicle( id="sub", name="Sub", ap_source_subdir="ArduSub", - fw_server_vehicle_sdir="Sub", waf_build_command="sub" ), Vehicle( id="heli", name="Heli", ap_source_subdir="ArduCopter", - fw_server_vehicle_sdir="Copter", waf_build_command="heli" ), Vehicle( id="blimp", name="Blimp", ap_source_subdir="Blimp", - fw_server_vehicle_sdir="Blimp", waf_build_command="blimp" ), Vehicle( id="tracker", name="Tracker", ap_source_subdir="AntennaTracker", - fw_server_vehicle_sdir="AntennaTracker", waf_build_command="antennatracker" ), Vehicle( id="ap-periph", name="AP_Periph", ap_source_subdir="Tools/AP_Periph", - fw_server_vehicle_sdir="AP_Periph", waf_build_command="AP_Periph" ), ] diff --git a/metadata_manager/versions_fetcher.py b/metadata_manager/versions_fetcher.py deleted file mode 100644 index 8a75c0cf..00000000 --- a/metadata_manager/versions_fetcher.py +++ /dev/null @@ -1,358 +0,0 @@ -import logging -import os -import ap_git -import json -import jsonschema -import hashlib -from pathlib import Path -from threading import Lock -from utils import TaskRunner -from .vehicles_manager import VehiclesManager as vehm - - -class VersionInfo: - """ - Class to wrap version info properties. - """ - def __init__(self, - remote_info: 'RemoteInfo', - commit_ref: str, - release_type: str, - version_number: str, - ap_build_artifacts_url) -> None: - self.remote_info = remote_info - self.commit_ref = commit_ref - self.release_type = release_type - self.version_number = version_number - self.ap_build_artifacts_url = ap_build_artifacts_url - - # Generate version_id as remote-sanitized_commit_ref-hash - # Commit ref is sanitized for URL safety by replacing '/' with '-' - # Hash is used to ensure unique ID after sanitization of commit_ref, - # as different commit refs may become identical after replacing '/' - commit_ref_sanitized = commit_ref.replace('/', '-') - commit_ref_hash = hashlib.md5(commit_ref.encode()).hexdigest()[:8] - self.version_id = ( - f"{remote_info.name}-{commit_ref_sanitized}-{commit_ref_hash}" - ) - - -class RemoteInfo: - """ - Class to wrap remote info properties. - """ - def __init__(self, - name: str, - url: str) -> None: - self.name = name - self.url = url - - def to_dict(self): - return { - 'name': self.name, - 'url': self.url, - } - - -class VersionsFetcher: - """ - Class to fetch the version-to-build metadata from remotes.json - and provide methods to view the same - """ - - __singleton = None - - def __init__(self, remotes_json_path: str, - ap_repo: ap_git.GitRepo): - """ - Initializes the VersionsFetcher instance - with a given remotes.json path. - - Parameters: - remotes_json_path (str): Path to the remotes.json file. - ap_repo (GitRepo): ArduPilot local git repository. This local - repository is shared between the VersionsFetcher - and the APSourceMetadataFetcher. - - Raises: - RuntimeError: If an instance of this class already exists, - enforcing a singleton pattern. - """ - if vehm.get_singleton() is None: - raise RuntimeError("VehiclesManager should be initialised first") - - # Enforce singleton pattern by raising an error if - # an instance already exists. - if VersionsFetcher.__singleton: - raise RuntimeError("VersionsFetcher must be a singleton.") - - self.logger = logging.getLogger(__name__) - - self.__remotes_json_path = remotes_json_path - self.__ensure_remotes_json() - self.__access_lock_versions_metadata = Lock() - self.__versions_metadata = [] - tasks = ( - (self.fetch_ap_releases, 1200), - (self.fetch_whitelisted_tags, 1200), - ) - self.__task__runner = TaskRunner(tasks=tasks) - self.repo = ap_repo - VersionsFetcher.__singleton = self - - def start(self) -> None: - """ - Start auto-fetch jobs. - """ - self.logger.info( - "Starting VersionsFetcher background auto-fetch jobs." - ) - self.__task__runner.start() - - def stop(self) -> None: - """ - Stop auto-fetch jobs. - """ - self.logger.info( - "Stopping VersionsFetcher background auto-fetch jobs." - ) - self.__task__runner.stop() - - def get_all_remotes_info(self) -> list[RemoteInfo]: - """ - Return the list of RemoteInfo objects constructed from the - information in the remotes.json file - - Returns: - list: RemoteInfo objects for all remotes mentioned in remotes.json - """ - return [ - RemoteInfo( - name=remote.get('name', None), - url=remote.get('url', None) - ) - for remote in self.__get_versions_metadata() - ] - - def get_remote_info(self, remote_name: str) -> RemoteInfo: - """ - Return the RemoteInfo for the given remote name, None otherwise. - - Returns: - RemoteInfo: The remote information object. - """ - return next( - ( - remote for remote in self.get_all_remotes_info() - if remote.name == remote_name - ), - None - ) - - def get_versions_for_vehicle(self, vehicle_id: str) -> list[VersionInfo]: - """ - Return the list of dictionaries containing the info about the - versions listed to be built for a particular vehicle. - - Parameters: - vehicle_id (str): the vehicle ID to fetch versions list for - - Returns: - list: VersionInfo objects for all versions allowed to be - built for the said vehicle. - - """ - if vehicle_id is None: - raise ValueError("Vehicle ID is a required parameter.") - - vehicle = vehm.get_singleton().get_vehicle_by_id(vehicle_id) - if vehicle is None: - raise ValueError(f"Invalid vehicle ID '{vehicle_id}'.") - - vehicle_name = vehicle.name - - versions_list = [] - for remote in self.__get_versions_metadata(): - remote_info = RemoteInfo( - name=remote.get('name', None), - url=remote.get('url', None) - ) - for vehicle in remote['vehicles']: - if vehicle['name'] != vehicle_name: - continue - - for release in vehicle['releases']: - versions_list.append(VersionInfo( - remote_info=remote_info, - commit_ref=release.get('commit_reference', None), - release_type=release.get('release_type', None), - version_number=release.get('version_number', None), - ap_build_artifacts_url=release.get( - 'ap_build_artifacts_url', - None - ) - )) - return versions_list - - def is_version_listed(self, vehicle_id: str, version_id: str) -> bool: - """ - Check if a version with given properties mentioned in remotes.json - - Parameters: - vehicle_id (str): ID of the vehicle for which version is listed - version_id (str): version ID - - Returns: - bool: True if the said version is mentioned in remotes.json, - False otherwise - - """ - if vehicle_id is None: - raise ValueError("vehicle_id is a required parameter.") - - if version_id is None: - raise ValueError("version_id is a required parameter.") - - return version_id in [ - version_info.version_id - for version_info in - self.get_versions_for_vehicle(vehicle_id=vehicle_id) - ] - - def get_version_info(self, vehicle_id: str, - version_id: str) -> VersionInfo: - """ - Find first version matching the given properties in remotes.json - - Parameters: - vehicle_id (str): ID of the vehicle for which version is listed - version_id (str): version ID - - Returns: - VersionInfo: Object for the version matching the properties, - None if not found - - """ - return next( - ( - version - for version in self.get_versions_for_vehicle( - vehicle_id=vehicle_id - ) - if version.version_id == version_id - ), - None - ) - - def reload_remotes_json(self) -> None: - """ - Read remotes.json, validate its structure against the schema - and cache it in memory - """ - # load file containing vehicles listed to be built for each - # remote along with the branches/tags/commits on which the - # firmware can be built - remotes_json_schema_path = os.path.join( - os.path.dirname(__file__), - 'remotes.schema.json' - ) - with open(self.__remotes_json_path, 'r') as f, \ - open(remotes_json_schema_path, 'r') as s: - f_content = f.read() - - # Early return if file is empty - if not f_content: - return - versions_metadata = json.loads(f_content) - schema = json.loads(s.read()) - # validate schema - jsonschema.validate(instance=versions_metadata, schema=schema) - self.__set_versions_metadata(versions_metadata=versions_metadata) - - # update git repo with latest remotes list - self.__sync_remotes_with_ap_repo() - - def __ensure_remotes_json(self) -> None: - """ - Ensures remotes.json exists and is a valid JSON file. - """ - p = Path(self.__remotes_json_path) - - if not p.exists(): - # Ensure parent directory exists - Path.mkdir(p.parent, parents=True, exist_ok=True) - - # write empty json list - with open(p, 'w') as f: - f.write('[]') - - def __set_versions_metadata(self, versions_metadata: list) -> None: - """ - Set versions metadata property with the one passed as parameter - This requires to acquire the access lock to avoid overwriting the - object while it is being read - """ - if versions_metadata is None: - raise ValueError("versions_metadata is a required parameter. " - "Cannot be None.") - - with self.__access_lock_versions_metadata: - self.__versions_metadata = versions_metadata - - def __get_versions_metadata(self) -> list: - """ - Read versions metadata property - This requires to acquire the access lock to avoid reading the list - while it is being modified - - Returns: - list: the versions metadata list - """ - with self.__access_lock_versions_metadata: - return self.__versions_metadata - - def __sync_remotes_with_ap_repo(self): - """ - Update the remotes in ArduPilot local repository with the latest - remotes list. - """ - remotes = tuple( - (remote.name, remote.url) - for remote in self.get_all_remotes_info() - ) - self.repo.remote_add_bulk(remotes=remotes, force=True) - - def fetch_ap_releases(self) -> None: - """ - Execute the fetch_releases.py script to update remotes.json - with Ardupilot's official releases - """ - from scripts import fetch_releases - fetch_releases.run( - base_dir=os.path.join( - os.path.dirname(self.__remotes_json_path), - '..', - ), - remote_name="ardupilot", - ) - self.reload_remotes_json() - return - - def fetch_whitelisted_tags(self) -> None: - """ - Execute the fetch_whitelisted_tags.py script to update - remotes.json with tags from whitelisted repos - """ - from scripts import fetch_whitelisted_tags - fetch_whitelisted_tags.run( - base_dir=os.path.join( - os.path.dirname(self.__remotes_json_path), - '..', - ) - ) - self.reload_remotes_json() - return - - @staticmethod - def get_singleton(): - return VersionsFetcher.__singleton diff --git a/metadata_manager/versions_manager/__init__.py b/metadata_manager/versions_manager/__init__.py new file mode 100644 index 00000000..f15cc775 --- /dev/null +++ b/metadata_manager/versions_manager/__init__.py @@ -0,0 +1,23 @@ +from .manager import VersionsManager +from .models import ForkRemoteSpec, RemoteInfo, VersionInfo +from .providers import ( + WhitelistedForkTagVersionsProvider, + ManifestJsonVersionsProvider, + RemotesJsonVersionsProvider, + VersionsProvider, + build_default_providers, + DEFAULT_WHITELISTED_FORK_REMOTES, +) + +__all__ = [ + "VersionsManager", + "ForkRemoteSpec", + "RemoteInfo", + "VersionInfo", + "VersionsProvider", + "ManifestJsonVersionsProvider", + "WhitelistedForkTagVersionsProvider", + "RemotesJsonVersionsProvider", + "build_default_providers", + "DEFAULT_WHITELISTED_FORK_REMOTES", +] diff --git a/metadata_manager/versions_manager/manager.py b/metadata_manager/versions_manager/manager.py new file mode 100644 index 00000000..6f6e86fc --- /dev/null +++ b/metadata_manager/versions_manager/manager.py @@ -0,0 +1,153 @@ +import logging +from pathlib import Path +from threading import Lock + +import ap_git +from utils import TaskRunner + +from metadata_manager.versions_manager.models import RemoteInfo, VersionInfo +from metadata_manager.versions_manager.providers import ( + VersionsProvider, + build_default_providers, +) +from metadata_manager.vehicles_manager import VehiclesManager as vehm + +REMOTES_SCHEMA_PATH = Path(__file__).resolve().parent.parent / "remotes.schema.json" + + +def _release_type_dedup_priority(release_type: str) -> int: + """Lower value wins when multiple releases share the same version_id.""" + match release_type: + case "stable": + return 0 + case "beta": + return 1 + case "latest": + return 2 + case "tag": + return 3 + case _: + return 99 + + +class VersionsManager: + """ + Version manager merges version metadata from different pluggable providers. + """ + + __singleton = None + + def __init__( + self, + ap_repo: ap_git.GitRepo, + remotes_json_path: str, + manifest_json=None, + providers: list[VersionsProvider] | None = None, + ): + if vehm.get_singleton() is None: + raise RuntimeError("VehiclesManager should be initialised first") + + if VersionsManager.__singleton: + raise RuntimeError("VersionsManager must be a singleton.") + + self.logger = logging.getLogger(__name__) + if providers is None: + if manifest_json is None: + raise ValueError("manifest_json is required when providers is omitted") + providers = build_default_providers( + manifest_json=manifest_json, + remotes_json_path=remotes_json_path, + schema_path=str(REMOTES_SCHEMA_PATH), + ) + self._providers = providers + self.repo = ap_repo + self._remotes_json_path = remotes_json_path + self._access_lock = Lock() + self.__task__runner = TaskRunner(tasks=((self.refresh_all, 1200),)) + VersionsManager.__singleton = self + + def start(self) -> None: + self.logger.info("Starting VersionsManager background refresh jobs.") + self.__task__runner.start() + + def stop(self) -> None: + self.logger.info("Stopping VersionsManager background refresh jobs.") + self.__task__runner.stop() + + def refresh_all(self) -> None: + for provider in self._providers: + try: + provider.refresh() + except Exception as exc: + self.logger.error( + "Provider %s refresh failed: %s", provider.name, exc + ) + self._sync_remotes_with_ap_repo() + + def get_all_remotes_info(self) -> list[RemoteInfo]: + remotes = {} + for provider in self._providers: + for remote in provider.get_remotes(): + remotes[remote.name] = remote + return list(remotes.values()) + + def get_remote_info(self, remote_name: str) -> RemoteInfo | None: + return next( + (remote for remote in self.get_all_remotes_info() + if remote.name == remote_name), + None, + ) + + def get_versions_for_vehicle(self, vehicle_id: str) -> list[VersionInfo]: + if vehicle_id is None: + raise ValueError("Vehicle ID is a required parameter.") + + vehicle = vehm.get_singleton().get_vehicle_by_id(vehicle_id) + if vehicle is None: + raise ValueError(f"Invalid vehicle ID '{vehicle_id}'.") + + by_version_id: dict[str, VersionInfo] = {} + for provider in self._providers: + for version in provider.get_versions(vehicle_id): + existing = by_version_id.get(version.version_id) + if existing is None: + by_version_id[version.version_id] = version + continue + if _release_type_dedup_priority(version.release_type) < ( + _release_type_dedup_priority(existing.release_type) + ): + by_version_id[version.version_id] = version + return list(by_version_id.values()) + + def is_version_listed(self, vehicle_id: str, version_id: str) -> bool: + if vehicle_id is None: + raise ValueError("vehicle_id is a required parameter.") + if version_id is None: + raise ValueError("version_id is a required parameter.") + + return version_id in [ + version.version_id + for version in self.get_versions_for_vehicle(vehicle_id=vehicle_id) + ] + + def get_version_info(self, vehicle_id: str, version_id: str) -> VersionInfo | None: + return next( + ( + version + for version in self.get_versions_for_vehicle(vehicle_id=vehicle_id) + if version.version_id == version_id + ), + None, + ) + + def _sync_remotes_with_ap_repo(self) -> None: + remotes = tuple( + (remote.name, remote.url) + for remote in self.get_all_remotes_info() + ) + if remotes: + self.repo.remote_add_bulk(remotes=remotes, force=True) + + @staticmethod + def get_singleton(): + return VersionsManager.__singleton diff --git a/metadata_manager/versions_manager/models.py b/metadata_manager/versions_manager/models.py new file mode 100644 index 00000000..3e9a7800 --- /dev/null +++ b/metadata_manager/versions_manager/models.py @@ -0,0 +1,70 @@ +import hashlib +from dataclasses import dataclass + +from metadata_manager.firmware_server.models import ReleaseRecord + + +@dataclass(frozen=True) +class ForkRemoteSpec: + """GitHub owner and repo for a whitelisted fork.""" + + owner: str + repo: str + + @property + def github_repo(self) -> str: + return f"{self.owner}/{self.repo}" + + @property + def url(self) -> str: + return f"https://github.com/{self.github_repo}.git" + + +@dataclass(frozen=True) +class RemoteInfo: + name: str + url: str + + def to_dict(self): + return { + "name": self.name, + "url": self.url, + } + + +class VersionInfo: + """Version metadata exposed to the rest of the application.""" + + def __init__( + self, + remote_info: RemoteInfo, + commit_ref: str, + release_type: str, + version_number: str, + ap_build_artifacts_url, + ) -> None: + self.remote_info = remote_info + self.commit_ref = commit_ref + self.release_type = release_type + self.version_number = version_number + self.ap_build_artifacts_url = ap_build_artifacts_url + + commit_ref_sanitized = commit_ref.replace("/", "-") + commit_ref_hash = hashlib.md5(commit_ref.encode()).hexdigest()[:8] + self.version_id = ( + f"{remote_info.name}-{commit_ref_sanitized}-{commit_ref_hash}" + ) + + @classmethod + def from_release_record( + cls, + remote_info: RemoteInfo, + release: ReleaseRecord, + ) -> "VersionInfo": + return cls( + remote_info=remote_info, + commit_ref=release.commit_reference, + release_type=release.release_type, + version_number=release.version_number, + ap_build_artifacts_url=release.ap_build_artifacts_url, + ) diff --git a/metadata_manager/versions_manager/providers.py b/metadata_manager/versions_manager/providers.py new file mode 100644 index 00000000..784bde12 --- /dev/null +++ b/metadata_manager/versions_manager/providers.py @@ -0,0 +1,318 @@ +import json +import logging +import os +from abc import ABC, abstractmethod +from pathlib import Path +from threading import Lock +from typing import Optional + +import jsonschema +import requests + +from metadata_manager.vehicles_manager import DEFAULT_VEHICLES + +from metadata_manager.firmware_server import ManifestJson +from metadata_manager.firmware_server.models import ReleaseRecord + +from .models import ForkRemoteSpec, RemoteInfo, VersionInfo + +OFFICIAL_REMOTE_NAME = "ardupilot" +OFFICIAL_REMOTE_URL = "https://github.com/ardupilot/ardupilot.git" +FIRMWARE_SERVER_BASE = "https://firmware.ardupilot.org" + +# CBS vehicle id -> firmware.ardupilot.org top-level directory. +_FIRMWARE_SERVER_DIR_BY_VEHICLE_ID = { + "copter": "Copter", + "plane": "Plane", + "rover": "Rover", + "sub": "Sub", + "heli": "Copter", + "blimp": "Blimp", + "tracker": "AntennaTracker", + "ap-periph": "AP_Periph", +} + + +def _firmware_server_dir(vehicle_id: str) -> str: + try: + return _FIRMWARE_SERVER_DIR_BY_VEHICLE_ID[vehicle_id] + except KeyError as exc: + raise ValueError(f"Unknown vehicle id: {vehicle_id}") from exc + + +def _vehicle_id_for_tag_segment(segment: str) -> Optional[str]: + for vehicle in DEFAULT_VEHICLES: + if vehicle.name == segment or vehicle.id == segment: + return vehicle.id + return None + + +DEFAULT_WHITELISTED_FORK_REMOTES = [ + ForkRemoteSpec(owner="tridge", repo="ardupilot"), + ForkRemoteSpec(owner="peterbarker", repo="ardupilot"), + ForkRemoteSpec(owner="rmackay9", repo="rmackay9-ardupilot"), + ForkRemoteSpec(owner="shiv-tyagi", repo="ardupilot"), + ForkRemoteSpec(owner="andyp1per", repo="ardupilot"), +] + + +class VersionsProvider(ABC): + @property + @abstractmethod + def name(self) -> str: + ... + + @abstractmethod + def get_versions(self, vehicle_id: str) -> list[VersionInfo]: + ... + + @abstractmethod + def get_remotes(self) -> list[RemoteInfo]: + ... + + def refresh(self) -> None: + """Optional hook to refresh provider data.""" + + @property + def is_available(self) -> bool: + return True + + +class ManifestJsonVersionsProvider(VersionsProvider): + name = "official-ardupilot" + + def __init__(self, manifest_json: ManifestJson): + self._manifest_json = manifest_json + self._remote = RemoteInfo(OFFICIAL_REMOTE_NAME, OFFICIAL_REMOTE_URL) + + @property + def is_available(self) -> bool: + return self._manifest_json.is_available + + def refresh(self) -> None: + self._manifest_json.refresh() + + def get_remotes(self) -> list[RemoteInfo]: + if not self.is_available: + return [] + return [self._remote] + + def get_versions(self, vehicle_id: str) -> list[VersionInfo]: + if not self.is_available: + return [] + + return [ + VersionInfo.from_release_record(self._remote, release) + for release in self._manifest_json.get_releases(vehicle_id) + ] + + +class WhitelistedForkTagVersionsProvider(VersionsProvider): + """ + Discover custom-build tags from whitelisted GitHub forks. + + Only tags with the ``custom-build/`` prefix are included. Vehicle-specific + tag segments may use a vehicle name (e.g. ``Copter``) or id (e.g. ``copter``). + + Tag format examples: + - ``custom-build/my-feature`` — listed for all vehicles + - ``custom-build/Copter/my-feature`` — listed for Copter only + - ``custom-build/copter/my-feature`` — listed for Copter only + """ + + name = "whitelisted-fork-tags" + + def __init__( + self, + fork_remotes: Optional[list[ForkRemoteSpec]] = None, + ): + self.logger = logging.getLogger(__name__) + self._fork_remotes = fork_remotes or DEFAULT_WHITELISTED_FORK_REMOTES + self._vehicle_ids = [v.id for v in DEFAULT_VEHICLES] + self._versions_by_remote_vehicle: dict[str, dict[str, list[ReleaseRecord]]] = {} + self._available = False + + @property + def is_available(self) -> bool: + return self._available + + def get_remotes(self) -> list[RemoteInfo]: + if not self.is_available: + return [] + fork_by_owner = {spec.owner: spec for spec in self._fork_remotes} + return [ + RemoteInfo(name=remote_name, url=fork_by_owner[remote_name].url) + for remote_name in self._versions_by_remote_vehicle + if remote_name != OFFICIAL_REMOTE_NAME and remote_name in fork_by_owner + ] + + def refresh(self) -> None: + versions_map = { + spec.owner: {vehicle_id: [] for vehicle_id in self._vehicle_ids} + for spec in self._fork_remotes + } + + for spec in self._fork_remotes: + if spec.owner == OFFICIAL_REMOTE_NAME: + continue + try: + tag_objs = self._fetch_tags_from_github(spec.github_repo) + except Exception as exc: + self.logger.warning( + "Skipping remote %s (%s): %s", + spec.owner, + spec.github_repo, + exc, + ) + continue + + for tag_info in tag_objs: + ref = tag_info["ref"].replace("refs/tags/", "") + parts = ref.split("/", 3) + if len(parts) <= 1 or parts[0] != "custom-build": + continue + + tag_vehicle_id = _vehicle_id_for_tag_segment(parts[1]) + if tag_vehicle_id is not None: + if len(parts) == 2: + continue + vehicles_for_tag = [tag_vehicle_id] + else: + vehicles_for_tag = self._vehicle_ids + + for vid in vehicles_for_tag: + versions_map[spec.owner][vid].append( + ReleaseRecord( + vehicle_id=vid, + release_type="tag", + version_number=parts[-1], + commit_reference=tag_info["object"]["sha"], + ap_build_artifacts_url=( + f"{FIRMWARE_SERVER_BASE}/" + f"{_firmware_server_dir(vid)}/latest" + ), + ) + ) + + self._versions_by_remote_vehicle = versions_map + self._available = True + + def get_versions(self, vehicle_id: str) -> list[VersionInfo]: + if not self.is_available: + return [] + + fork_by_owner = {spec.owner: spec for spec in self._fork_remotes} + versions = [] + for remote_name, vehicles_map in self._versions_by_remote_vehicle.items(): + if remote_name == OFFICIAL_REMOTE_NAME: + continue + spec = fork_by_owner.get(remote_name) + if spec is None: + continue + remote = RemoteInfo(name=spec.owner, url=spec.url) + for release in vehicles_map.get(vehicle_id, []): + versions.append(VersionInfo.from_release_record(remote, release)) + return versions + + def _fetch_tags_from_github(self, github_repo: str) -> list: + url = f"https://api.github.com/repos/{github_repo}/git/refs/tags" + headers = { + "X-GitHub-Api-Version": "2022-11-28", + "Accept": "application/vnd.github+json", + } + token = os.getenv("CBS_GITHUB_ACCESS_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + + response = requests.get(url=url, headers=headers, timeout=60) + response.raise_for_status() + return response.json() + + +class RemotesJsonVersionsProvider(VersionsProvider): + name = "remotes-json" + + def __init__(self, remotes_json_path: str, schema_path: str): + self._remotes_json_path = remotes_json_path + self._schema_path = schema_path + self._lock = Lock() + self._metadata: list = [] + + @property + def is_available(self) -> bool: + return bool(self._metadata) + + def refresh(self) -> None: + self.reload() + + def reload(self) -> None: + path = Path(self._remotes_json_path) + if not path.is_file(): + with self._lock: + self._metadata = [] + return + + content = path.read_text(encoding="utf-8") + if not content.strip(): + with self._lock: + self._metadata = [] + return + + metadata = json.loads(content) + schema = json.loads(Path(self._schema_path).read_text(encoding="utf-8")) + jsonschema.validate(instance=metadata, schema=schema) + + with self._lock: + self._metadata = metadata + + def get_remotes(self) -> list[RemoteInfo]: + with self._lock: + metadata = list(self._metadata) + return [ + RemoteInfo(name=remote.get("name"), url=remote.get("url")) + for remote in metadata + if remote.get("name") and remote.get("url") + ] + + def get_versions(self, vehicle_id: str) -> list[VersionInfo]: + versions = [] + with self._lock: + metadata = list(self._metadata) + + for remote in metadata: + remote_info = RemoteInfo( + name=remote.get("name"), + url=remote.get("url"), + ) + for remote_vehicle in remote.get("vehicles", []): + if remote_vehicle.get("id") != vehicle_id: + continue + for release in remote_vehicle.get("releases", []): + versions.append( + VersionInfo( + remote_info=remote_info, + commit_ref=release.get("commit_reference"), + release_type=release.get("release_type"), + version_number=release.get("version_number"), + ap_build_artifacts_url=release.get( + "ap_build_artifacts_url" + ), + ) + ) + return versions + + +def build_default_providers( + manifest_json: ManifestJson, + remotes_json_path: str, + schema_path: str, + fork_remotes: Optional[list[ForkRemoteSpec]] = None, +) -> list[VersionsProvider]: + return [ + ManifestJsonVersionsProvider(manifest_json), + WhitelistedForkTagVersionsProvider(fork_remotes=fork_remotes), + RemotesJsonVersionsProvider( + remotes_json_path=remotes_json_path, + schema_path=schema_path, + ), + ] From cd2e41d8ce8d1db836d079452c01ffb6fa7356ec Mon Sep 17 00:00:00 2001 From: Shiv Tyagi Date: Sat, 11 Jul 2026 09:35:15 +0530 Subject: [PATCH 2/8] scripts: drop cron based version updation scripts --- scripts/fetch_releases.py | 300 -------------------- scripts/fetch_whitelisted_tags.py | 296 ------------------- scripts/update_remotes_json_cronjob_beta.sh | 89 ------ scripts/update_remotes_json_cronjob_main.sh | 89 ------ 4 files changed, 774 deletions(-) delete mode 100755 scripts/fetch_releases.py delete mode 100644 scripts/fetch_whitelisted_tags.py delete mode 100755 scripts/update_remotes_json_cronjob_beta.sh delete mode 100755 scripts/update_remotes_json_cronjob_main.sh diff --git a/scripts/fetch_releases.py b/scripts/fetch_releases.py deleted file mode 100755 index 826dd6b8..00000000 --- a/scripts/fetch_releases.py +++ /dev/null @@ -1,300 +0,0 @@ -import json -import optparse -import os -import re -import requests -from packaging.version import Version - - -def version_number_and_type(git_hash, ap_source_subdir): - url = ( - "https://raw.githubusercontent.com/ArduPilot/ardupilot/" - f"{git_hash}/{ap_source_subdir}/version.h" - ) - response = requests.get(url=url) - - if response.status_code != 200: - print(response.text) - print(url) - raise Exception( - "Couldn't fetch version.h from github server. " - f"Got status code {response.status_code}" - ) - - exp = re.compile( - r'#define FIRMWARE_VERSION (\d+),(\d+),(\d+),FIRMWARE_VERSION_TYPE_(\w+)' # noqa - ) - matches = re.findall(exp, response.text) - major, minor, patch, fw_type = matches[0] - fw_type = fw_type.lower() - - if fw_type == 'official': - # to avoid any confusion, beta is also 'official' ;-) - fw_type = 'stable' - return f'{major}.{minor}.{patch}', fw_type - - -def fetch_tags_from_github(): - url = 'https://api.github.com/repos/ardupilot/ardupilot/git/refs/tags' - headers = { - 'X-GitHub-Api-Version': '2022-11-28', - 'Accept': 'application/vnd.github+json' - } - - token = os.getenv("CBS_GITHUB_ACCESS_TOKEN") - if token: - headers['Authorization'] = f"Bearer {token}" - - response = requests.get(url=url, headers=headers) - if response.status_code != 200: - print(response.text) - print(url) - raise Exception( - "Couldn't fetch tags from github server. " - f"Got status code {response.status_code}" - ) - - tags_objs = response.json() - return tags_objs - - -def remove_duplicate_entries(releases): - temp = {} - for release in releases: - # if we have already seen a version with similar hash - # and we now see a beta release with same hash, we skip it - if temp.get(release['commit_reference']) and \ - release['release_type'] == 'beta': - continue - - temp[release['commit_reference']] = release - - return list(temp.values()) - - -def construct_vehicle_versions_list(vehicle, ap_source_subdir, - fw_server_vehicle_sdir, - tag_filter_exps, tags, - ignore_versions_before=''): - ret = [] - for tag_info in tags: - tag = tag_info['ref'].replace('refs/tags/', '') - - matches = [] - for exp in tag_filter_exps: - # the regexes capture two groups - # first group is the matched substring itself - # second group is the tag body (e.g., beta, stable, 4.5.1 etc) - matches.extend(re.findall(re.compile(exp), tag)) - - if matches: - matched_string, tag_body = matches[0] - - if len(matched_string) < len(tag): - print( - f"Partial match. Ignoring. Matched '{matched_string}' " - f"in '{tag}'." - ) - continue - - try: - v_num, v_type = version_number_and_type( - tag_info['object']['sha'], - ap_source_subdir - ) - except Exception as e: - print(f'Cannot determine version number for tag {tag}') - print(e) - continue - - if Version(v_num) < Version(ignore_versions_before): - print(f"{v_num} Version too old. Ignoring.") - continue - - if re.search(r'\d+\.\d+\.\d+', tag_body): - # we do stable version tags in this format - # e.g. Rover-4.5.1, Copter-4.5.1, where Rover and Copter - # are prefixes and 4.5.1 is the tag body - # artifacts for these versions are stored in firmware - # server at stable-x.y.z subdirs - afcts_url = ( - f'https://firmware.ardupilot.org/{fw_server_vehicle_sdir}' - f'/stable-{tag_body}' - ) - else: - afcts_url = ( - f'https://firmware.ardupilot.org/{fw_server_vehicle_sdir}' - f'/{tag_body}' - ) - - ret.append({ - 'release_type': v_type, - 'version_number': v_num, - 'ap_build_artifacts_url': afcts_url, - 'commit_reference': tag_info['object']['sha'] - }) - - ret = remove_duplicate_entries(ret) - - # entry for master - ret.append({ - 'release_type': 'latest', - 'version_number': 'NA', - 'ap_build_artifacts_url': ( - f'https://firmware.ardupilot.org/{fw_server_vehicle_sdir}/latest' - ), - 'commit_reference': 'refs/heads/master' - }) - - return { - 'name': vehicle, - 'releases': ret - } - - -def run(base_dir, remote_name): - remotes_json_path = os.path.join(base_dir, 'configs', 'remotes.json') - - tags = fetch_tags_from_github() - vehicles = [] - - vehicles.append(construct_vehicle_versions_list( - "Copter", - "ArduCopter", - "Copter", - [ - "(ArduCopter-(beta-4.3|beta|stable))", - "(Copter-(\d+\.\d+\.\d+))" # noqa - ], - tags, - '4.3', - )) - - vehicles.append(construct_vehicle_versions_list( - "Plane", - "ArduPlane", - "Plane", - [ - "(ArduPlane-(beta-4.3|beta|stable))", - "(Plane-(\d+\.\d+\.\d+))" # noqa - ], - tags, - '4.3', - )) - - vehicles.append(construct_vehicle_versions_list( - "Rover", - "Rover", - "Rover", - [ - "(APMrover2-(beta-4.3|beta|stable))", - "(Rover-(\d+\.\d+\.\d+))" # noqa - ], - tags, - '4.3', - )) - - vehicles.append(construct_vehicle_versions_list( - "Sub", - "ArduSub", - "Sub", - [ - "(ArduSub-(beta-4.3|beta|stable))", - "(Sub-(\d+\.\d+\.\d+))" # noqa - ], - tags, - '4.3', - )) - - vehicles.append(construct_vehicle_versions_list( - "Tracker", - "AntennaTracker", - "AntennaTracker", - [ - "(AntennaTracker-(beta-4.3|beta|stable))", - "(Tracker-(\d+\.\d+\.\d+))" # noqa - ], - tags, - '4.3', - )) - - vehicles.append(construct_vehicle_versions_list( - "Blimp", - "Blimp", - "Blimp", - [ - "(Blimp-(beta-4.3|beta|stable|\d+\.\d+\.\d+))" # noqa - ], - tags, - '4.3', - )) - - vehicles.append(construct_vehicle_versions_list( - "Heli", - "ArduCopter", - "Copter", - [ - "(ArduCopter-(beta-4.3|beta|stable)-heli)" - ], - tags, - '4.3', - )) - - vehicles.append(construct_vehicle_versions_list( - "AP_Periph", - "Tools/AP_Periph", - "AP_Periph", - [ - "(AP_Periph-(beta|stable))" - ], - tags, - '1.8.1', - )) - - remotes_json = { - "name": remote_name, - "url": "https://github.com/ardupilot/ardupilot.git", - "vehicles": vehicles - } - - try: - with open(remotes_json_path, 'r') as f: - remotes = json.loads(f.read()) - - # remove existing remote entry from the list - temp = [] - for remote in remotes: - if remote['name'] != remote_name: - temp.append(remote) - remotes = temp - except Exception as e: - print(e) - print("Writing to empty file") - remotes = [] - - with open(remotes_json_path, 'w') as f: - remotes.append(remotes_json) - f.write(json.dumps(remotes, indent=2)) - print(f"Wrote {remotes_json_path}") - - -if __name__ == "__main__": - parser = optparse.OptionParser("fetch_releases.py") - parser.add_option( - "", "--basedir", type="string", - default=os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "base") - ), - help="base directory" - ) - - parser.add_option( - "", "--remotename", type="string", - default="ardupilot", - help="Remote name to write in json file" - ) - - cmd_opts, cmd_args = parser.parse_args() - basedir = os.path.abspath(cmd_opts.basedir) - remotename = cmd_opts.remotename - run(base_dir=basedir, remote_name=remotename) diff --git a/scripts/fetch_whitelisted_tags.py b/scripts/fetch_whitelisted_tags.py deleted file mode 100644 index 90d59a49..00000000 --- a/scripts/fetch_whitelisted_tags.py +++ /dev/null @@ -1,296 +0,0 @@ -""" -This script automates the listing of tagged versions of -the ArduPilot source code from various white-listed forks -from GitHub on the Custom Build Server. -It performs the following tasks: - -1. **Fetch Tags**: Utilizes the GitHub API to retrieve the list of tags from - the specified remote repository. -2. **Filter Tags**: Identifies tags that start with the prefix 'custom-build/'. -3. **Update remotes.json**: Adds information about these custom-build tags to - the `remotes.json` file, enabling building these versions. - -Tag Naming Conventions: -- Tags named in the format `custom-build/xyz` are allowed to be built - for all vehicles. -- Tags in the format `custom-build/vehicle/xyz` are allowed to be built - for the specified vehicle only. - -Examples: -- A tag named `custom-build/some-test-work` will be listed under all - the vehicles. -- A tag named `custom-build/Copter/some-test-work` will be listed under - Copter only. - -Note: -- Vehicle names are case-sensitive. e.g., copter is not Copter. -""" -import json -import optparse -import os -import requests - - -# TODO: move this to base/configs/whitelisted_custom_tag_remotes.json -remotes = [ - "ardupilot", - "tridge", - "peterbarker", - "rmackay9", - "shiv-tyagi", - "andyp1per", -] - -vehicles = [ - 'Copter', - 'Plane', - 'Rover', - 'Sub', - 'AntennaTracker', - 'Blimp', - 'Heli', - 'AP_Periph', -] - - -def fetch_tags_from_github(remote): - """Returns a list of dictionaries with tag details (name and commit SHA) - for the ardupilot repo in the specified remote using the GitHub API - See https://docs.github.com/en/rest/git/refs?apiVersion=2022-11-28 - for more details - """ - url = f'https://api.github.com/repos/{remote}/ardupilot/git/refs/tags' - headers = { - 'X-GitHub-Api-Version': '2022-11-28', - 'Accept': 'application/vnd.github+json' - } - - token = os.getenv("CBS_GITHUB_ACCESS_TOKEN") - if token: - headers['Authorization'] = f"Bearer {token}" - - response = requests.get(url=url, headers=headers) - if response.status_code != 200: - print(response.text) - print(url) - raise Exception( - "Couldn't fetch tags from github server. " - f"Got status code {response.status_code}" - ) - - tags_objs = response.json() - return tags_objs - - -def construct_versions_map(remotes, vehicles): - """ - Returns a dictionary containing the list of versions - allowed to be built for different remotes and vehicles. - - The returned dictionary is structured as follows: - - The first-level keys represent the remote names. - - Each first-level key maps to a nested dictionary. - - The nested dictionary has keys representing vehicle names. - - Each vehicle name key maps to a list of dictionaries - containing the information about the versions allowed - to be built for that particular remote and vehicle. - - Refer remotes.schema.json to see the schema of the - dictionaries about the version information. - - Example: - { - 'remote1': { - 'vehicleA': [{...}, {...}], - 'vehicleB': [{...}, {...}, {...}] - }, - 'remote2': { - 'vehicleA': [{...}, {...}], - 'vehicleB': [{...}, {...}, {...}] - 'vehicleC': [{...}, {...}], - } - } - """ - - ret = { - remote: { - vehicle: [] for vehicle in vehicles - } for remote in remotes - } - - for remote in remotes: - try: - # fetch tag info for ardupilot repo in the remote from github - tag_objs = fetch_tags_from_github(remote) - except Exception as e: - print(e) - print("Skipping this remote...") - continue - - for tag_info in tag_objs: - ref = tag_info['ref'] - ref = ref.replace('refs/tags/', '') - - s = ref.split('/', 3) - - # skip if tag name does not start with custom-build/ - if len(s) <= 1 or s[0] != 'custom-build': - continue - - vehicles_for_tag = [] - if s[1] in vehicles: - if len(s) == 2: - print(f'Found {ref}. Incomplete tag. Skipping.') - # tag is named like custom-build/vehicle - # this format is incorrect - continue - - # tag is in the format custom-build/vehicle/xyz - # list the tag only under this vehicle - vehicles_for_tag = [s[1],] - print(f'Found {ref}. Adding to {s[1]}.') - else: - # tag is in the format custom-build/xyz - # no vehicle is specified in tag name - # list the tag under all vehicles - vehicles_for_tag = vehicles - print(f'Found {ref}. Adding to all vehicles.') - - for vehicle in vehicles_for_tag: - # append an entry to the list of versions listed to be built - # for the remote and vehicle - ret[remote][vehicle].append( - { - 'release_type': 'tag', - 'version_number': s[-1], - 'ap_build_artifacts_url': f'https://firmware.ardupilot.org/{vehicle}/latest', # noqa # use master defaults for auto-fetched tags - 'commit_reference': tag_info['object']['sha'] - } - ) - - return ret - - -def read_remotes_json_file(path): - """Returns python object constructed from the contents of - remotes.json file - - If the file cannot be read, it returns an empty list - """ - try: - with open(path, 'r') as f: - remotes = json.loads(f.read()) - except Exception as e: - print(e) - print("Returning empty list") - remotes = [] - - return remotes - - -def write_remotes_json_file(path, remotes_json_obj): - """Serialize the remotes_json_obj object and - write to the remotes.json file - """ - with open(path, 'w') as f: - f.write(json.dumps(remotes_json_obj, indent=2)) - print(f"Wrote {path}") - - -def update_remotes_json(path, new_versions_map): - """Update remotes.json with the versions listed in new_versions_map - """ - remotes_json_obj = read_remotes_json_file(path) - - # create a dict remote names mapped to the objects containing the - # information about that remote in remotes_json_obj to avoid - # iterating over the list to access the object every time - rname_obj_map = { - remote['name']: remote for remote in remotes_json_obj - } - - # create another dict with remote names mapped to a dict. - # In the secondary dict, the vehicle names are mapped - # to the objects in the remotes_json_obj containing - # the information about the vehicles listed under the remote - rname_vname_obj_map = { - remote['name']: { - vehicle['name']: vehicle for vehicle in remote['vehicles'] - } for remote in remotes_json_obj - } - - for remote_name, vehicles_obj_dict in new_versions_map.items(): - # we do not have the remote listed in existing remotes_json_obj - if not rname_obj_map.get(remote_name): - print(f'Remote {remote_name} ' - f'does not exist in existing remotes.json. Adding.') - # create object containing the information about the remote - remote_obj = { - 'name': remote_name, - 'url': f'https://github.com/{remote_name}/ardupilot.git', - 'vehicles': [] - } - - # append to the remotes_json_obj - remotes_json_obj.append(remote_obj) - # add a reference to the object in rname_obj_map - rname_obj_map[remote_name] = remote_obj - # no vehicles are listed for this remote yet, - # we populate them as we see them ahead - rname_vname_obj_map[remote_name] = dict() - - for vehicle_name, versions in vehicles_obj_dict.items(): - # the vehicle is not listed under this remote - if not rname_vname_obj_map[remote_name].get(vehicle_name): - print(f'Vehicle {vehicle_name} does not exist ' - f'for remote {remote_name} ' - f'in existing remotes.json. Adding.') - # create vehicle object - vehicle_obj = { - 'name': vehicle_name, - 'releases': [] - } - - # append the vehicle object to the list of vehicle - # objects under the remote - rname_obj_map[remote_name]['vehicles'].append(vehicle_obj) - # add a reference to this object under the remote - # and the vehicle name - rname_vname_obj_map[remote_name][vehicle_name] = vehicle_obj - - # remove duplicates and merge lists - existing_list = rname_vname_obj_map[remote_name][vehicle_name]['releases'] # noqa - new_versions_list = [] - for i in range(len(existing_list)): - prefix = "tag" - if existing_list[i]['release_type'][:len(prefix)] != prefix: - new_versions_list.append(existing_list[i]) - new_versions_list.extend(versions) - - # add the versions listed for this vehicle and - # the remote in remotes_json_obj as mentioned above - rname_vname_obj_map[remote_name][vehicle_name]['releases'] = new_versions_list # noqa - - # write the updated obj to the remotes.json file - write_remotes_json_file(path, remotes_json_obj) - return - - -def run(base_dir): - remotes_json_path = os.path.join(base_dir, 'configs', 'remotes.json') - new_versions_map = construct_versions_map(remotes, vehicles) - update_remotes_json(remotes_json_path, new_versions_map) - - -if __name__ == "__main__": - parser = optparse.OptionParser("fetch_releases.py") - parser.add_option( - "", "--basedir", type="string", - default=os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "base") - ), - help="base directory" - ) - - cmd_opts, cmd_args = parser.parse_args() - basedir = os.path.abspath(cmd_opts.basedir) - run(base_dir=basedir) diff --git a/scripts/update_remotes_json_cronjob_beta.sh b/scripts/update_remotes_json_cronjob_beta.sh deleted file mode 100755 index 2656759d..00000000 --- a/scripts/update_remotes_json_cronjob_beta.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -# this script is intended to be run from a crontab owned by the -# custom user on ArduPilot's autotest server -# to do a hot reload of remotes.json on custom-beta.ardupilot.org - -# CBS_REMOTES_RELOAD_TOKEN can be supplied as an environment -# variable in the crontab line, for example: - -# 0 * * * * CBS_REMOTES_RELOAD_TOKEN=8d64ed06945 /home/custom/beta/CustomBuild/scripts/fetch_release_cronjob.sh - -# or can be read from base/secrets/reload_token (the token from the file gets the preference) - -CUSTOM_HOME=/home/custom -TOPDIR=$CUSTOM_HOME/beta -LOGDIR=$TOPDIR/cron -BASEDIR=$TOPDIR/base - -# Maximum number of log files to retain -MAX_LOG_FILES=50 - -# Get the current timestamp -TIMESTAMP=$(date +"%Y%m%d_%H%M%S") - -# Function to clean up old log files -cleanup_old_logs() { - # Find and sort log files by modification time, oldest first - LOG_FILES=($(ls -1t ${LOGDIR}/fetch_releases_*.log \ - ${LOGDIR}/fetch_whitelisted_tags_*.log \ - 2>/dev/null)) - - # Count the number of log files - NUM_LOG_FILES=${#LOG_FILES[@]} - - # If the number of log files is greater than the maximum allowed - if [ $NUM_LOG_FILES -gt $MAX_LOG_FILES ]; then - - # Loop through and delete the oldest files - for ((i = $MAX_LOG_FILES ; i < $NUM_LOG_FILES ; i++ )); do - rm -f "${LOG_FILES[$i]}" - done - - fi -} - -# Method to reload remotes on custom build server app -reload_remotes_on_app() { - local token_file="$BASEDIR/secrets/reload_token" - local token - - # Check if the token file exists and is readable - if [[ -f "$token_file" && -r "$token_file" ]]; then - # Read the token from the file - token=$(cat "$token_file") - else - echo "Token can't be retrieved from the file." - # Try to get the token from the environment variable - token=$CBS_REMOTES_RELOAD_TOKEN - fi - - # Check if token is still empty - if [[ -z "$token" ]]; then - echo "Error: Token could not be retrieved." - return 1 - fi - - # Send the curl request - curl -X POST https://custom-beta.ardupilot.org/refresh_remotes \ - -H "Content-Type: application/json" \ - -d "{\"token\": \"$token\"}" - - return 0 -} - -# Call the cleanup function before executing the main script -cleanup_old_logs - -# Run fetch_releases.py to add official releases from AP -python3 $TOPDIR/CustomBuild/scripts/fetch_releases.py \ - --basedir $BASEDIR \ - >> $LOGDIR/fetch_releases_${TIMESTAMP}.log 2>&1 - -# Run fetch_whitelisted_tags.py to add tags from whitelisted repos -python3 $TOPDIR/CustomBuild/scripts/fetch_whitelisted_tags.py \ - --basedir $BASEDIR \ - >> $LOGDIR/fetch_whitelisted_tags_${TIMESTAMP}.log 2>&1 - -# Call the reload_remotes_on_app function to refresh remotes.json on app -reload_remotes_on_app diff --git a/scripts/update_remotes_json_cronjob_main.sh b/scripts/update_remotes_json_cronjob_main.sh deleted file mode 100755 index 9d3072f1..00000000 --- a/scripts/update_remotes_json_cronjob_main.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -# this script is intended to be run from a crontab owned by the -# custom user on ArduPilot's autotest server -# to do a hot reload of remotes.json on custom.ardupilot.org - -# CBS_REMOTES_RELOAD_TOKEN can be supplied as an environment -# variable in the crontab line, for example: - -# 0 * * * * CBS_REMOTES_RELOAD_TOKEN=8d64ed06945 /home/custom/CustomBuild/scripts/fetch_release_cronjob.sh - -# or can be read from base/secrets/reload_token (the token from the file gets the preference) - -CUSTOM_HOME=/home/custom -TOPDIR=$CUSTOM_HOME -LOGDIR=$TOPDIR/cron -BASEDIR=$TOPDIR/base - -# Maximum number of log files to retain -MAX_LOG_FILES=50 - -# Get the current timestamp -TIMESTAMP=$(date +"%Y%m%d_%H%M%S") - -# Function to clean up old log files -cleanup_old_logs() { - # Find and sort log files by modification time, oldest first - LOG_FILES=($(ls -1t ${LOGDIR}/fetch_releases_*.log \ - ${LOGDIR}/fetch_whitelisted_tags_*.log \ - 2>/dev/null)) - - # Count the number of log files - NUM_LOG_FILES=${#LOG_FILES[@]} - - # If the number of log files is greater than the maximum allowed - if [ $NUM_LOG_FILES -gt $MAX_LOG_FILES ]; then - - # Loop through and delete the oldest files - for ((i = $MAX_LOG_FILES ; i < $NUM_LOG_FILES ; i++ )); do - rm -f "${LOG_FILES[$i]}" - done - - fi -} - -# Method to reload remotes on custom build server app -reload_remotes_on_app() { - local token_file="$BASEDIR/secrets/reload_token" - local token - - # Check if the token file exists and is readable - if [[ -f "$token_file" && -r "$token_file" ]]; then - # Read the token from the file - token=$(cat "$token_file") - else - echo "Token can't be retrieved from the file." - # Try to get the token from the environment variable - token=$CBS_REMOTES_RELOAD_TOKEN - fi - - # Check if token is still empty - if [[ -z "$token" ]]; then - echo "Error: Token could not be retrieved." - return 1 - fi - - # Send the curl request - curl -X POST https://custom.ardupilot.org/refresh_remotes \ - -H "Content-Type: application/json" \ - -d "{\"token\": \"$token\"}" - - return 0 -} - -# Call the cleanup function before executing the main script -cleanup_old_logs - -# Run fetch_releases.py to add official releases from AP -python3 $TOPDIR/CustomBuild/scripts/fetch_releases.py \ - --basedir $BASEDIR \ - >> $LOGDIR/fetch_releases_${TIMESTAMP}.log 2>&1 - -# Run fetch_whitelisted_tags.py to add tags from whitelisted repos -python3 $TOPDIR/CustomBuild/scripts/fetch_whitelisted_tags.py \ - --basedir $BASEDIR \ - >> $LOGDIR/fetch_whitelisted_tags_${TIMESTAMP}.log 2>&1 - -# Call the reload_remotes_on_app function to refresh remotes.json on app -reload_remotes_on_app From 990f8a64e629233aeaab9c262434f6b5317d392c Mon Sep 17 00:00:00 2001 From: Shiv Tyagi Date: Sat, 11 Jul 2026 09:39:05 +0530 Subject: [PATCH 3/8] web: use new provider based versions manager --- web/api/v1/admin.py | 26 ++++++++++----------- web/core/config.py | 26 ++++++++++++++------- web/core/startup.py | 49 ++------------------------------------- web/main.py | 30 ++++++++++++++++-------- web/schemas/__init__.py | 4 ++-- web/schemas/admin.py | 7 +++--- web/services/admin.py | 50 +++++++++++++++++++--------------------- web/services/builds.py | 10 ++++---- web/services/vehicles.py | 12 +++++----- 9 files changed, 94 insertions(+), 120 deletions(-) diff --git a/web/api/v1/admin.py b/web/api/v1/admin.py index f1ea036e..f1a63a0f 100644 --- a/web/api/v1/admin.py +++ b/web/api/v1/admin.py @@ -1,7 +1,7 @@ from fastapi import APIRouter, HTTPException, Depends, status from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials -from web.schemas import RefreshRemotesResponse +from web.schemas import RefreshVersionsResponse from web.services.admin import get_admin_service, AdminService @@ -9,12 +9,12 @@ security = HTTPBearer(auto_error=False) -async def verify_remote_reload_token( +async def verify_admin_token( credentials: HTTPAuthorizationCredentials = Depends(security), admin_service: AdminService = Depends(get_admin_service) ) -> None: """ - Verify the bearer token for remote reload authentication. + Verify the bearer token for admin API authentication. Args: credentials: HTTP authorization credentials from request header @@ -32,7 +32,7 @@ async def verify_remote_reload_token( token = credentials.credentials try: - if not await admin_service.verify_remote_reload_token(token): + if not await admin_service.verify_admin_token(token): raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid authentication token" @@ -45,8 +45,8 @@ async def verify_remote_reload_token( @router.post( - "/refresh_remotes", - response_model=RefreshRemotesResponse, + "/refresh_versions", + response_model=RefreshVersionsResponse, responses={ 401: {"description": "Invalid or missing authentication token"}, 500: { @@ -57,12 +57,12 @@ async def verify_remote_reload_token( } } ) -async def refresh_remotes( - _: None = Depends(verify_remote_reload_token), +async def refresh_versions( + _: None = Depends(verify_admin_token), admin_service: AdminService = Depends(get_admin_service) ): """ - Trigger a hot reset/refresh of remote metadata. + Trigger a refresh of all version metadata providers. This endpoint requires bearer token authentication in the Authorization header: @@ -71,17 +71,17 @@ async def refresh_remotes( ``` Returns: - RefreshRemotesResponse: List of remotes that were refreshed + RefreshVersionsResponse: Git remotes synced after the refresh Raises: 401: Invalid or missing authentication token 500: Refresh operation failed """ try: - remotes = await admin_service.refresh_remotes() - return RefreshRemotesResponse(remotes=remotes) + remotes = await admin_service.refresh_versions() + return RefreshVersionsResponse(remotes=remotes) except Exception as e: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to refresh remotes: {str(e)}" + detail=f"Failed to refresh versions: {str(e)}" ) diff --git a/web/core/config.py b/web/core/config.py index 93225ebd..187168e1 100644 --- a/web/core/config.py +++ b/web/core/config.py @@ -88,33 +88,43 @@ def enable_inbuilt_builder(self) -> bool: return os.getenv('CBS_ENABLE_INBUILT_BUILDER', '1') == '1' @property - def remote_reload_token(self) -> Optional[str]: + def manifest_cache_dir(self) -> str: + """Directory for cached firmware manifest files.""" + return os.path.dirname(self.remotes_json_path) + + @property + def ap_firmware_manifest_url(self) -> str: + """URL for the ArduPilot firmware manifest.""" + return os.getenv( + 'AP_FIRMWARE_MANIFEST_URL', + 'https://firmware.ardupilot.org/manifest.json', + ) + + @property + def admin_token(self) -> Optional[str]: """ - Get remote reload token from file or environment variable. + Get admin API token from file or environment variable. Tries to read token from file first, falls back to environment variable. Returns: The authorization token if found, None otherwise """ - token_file_path = os.path.join(self.base_dir, 'secrets', 'reload_token') + token_file_path = os.path.join(self.base_dir, 'secrets', 'admin_token') try: - # Try to read the secret token from the file with open(token_file_path, 'r') as file: token = file.read().strip() return token except (FileNotFoundError, PermissionError): - # If the file does not exist or no permission, check environment - env_token = os.getenv('CBS_REMOTES_RELOAD_TOKEN', '') + env_token = os.getenv('CBS_ADMIN_TOKEN', '') return env_token if env_token != '' else None except Exception as e: logger.error( f"Unexpected error reading token file at {token_file_path}: {e}. " "Checking environment for token." ) - # For any other error, fall back to environment variable - env_token = os.getenv('CBS_REMOTES_RELOAD_TOKEN', None) + env_token = os.getenv('CBS_ADMIN_TOKEN', None) return env_token if env_token != '' else None diff --git a/web/core/startup.py b/web/core/startup.py index 08c7f7dd..39b6c69c 100644 --- a/web/core/startup.py +++ b/web/core/startup.py @@ -25,7 +25,6 @@ def ensure_base_structure(base_dir: str) -> None: logger.warning("Base directory not specified, skipping initialization") return - # Define required subdirectories subdirs = [ 'artifacts', 'configs', @@ -39,52 +38,12 @@ def ensure_base_structure(base_dir: str) -> None: logger.debug(f"Ensured directory exists: {path}") -def ensure_remotes_json(base_dir: str, remote_name: str = "ardupilot") -> None: - """ - Ensure remotes.json configuration file exists. - - If the remotes.json file doesn't exist, creates it by fetching release - information from the specified remote. - - Args: - base_dir: The base directory path (typically from CBS_BASEDIR) - remote_name: The remote repository name to fetch releases from - """ - if not base_dir: - logger.warning( - "Base directory not specified, " - "skipping remotes.json initialization" - ) - return - - remotes_json_path = os.path.join(base_dir, 'configs', 'remotes.json') - - if not os.path.isfile(remotes_json_path): - logger.info( - f"remotes.json not found at {remotes_json_path}, " - f"creating it..." - ) - try: - from scripts import fetch_releases - fetch_releases.run( - base_dir=base_dir, - remote_name=remote_name, - ) - logger.info("Successfully created remotes.json") - except Exception as e: - logger.error(f"Failed to create remotes.json: {e}") - raise - else: - logger.debug(f"remotes.json already exists at {remotes_json_path}") - - def initialize_application(base_dir: str) -> None: """ Initialize the application environment. - Performs all necessary setup operations including: - - Creating required directory structure - - Ensuring remotes.json configuration exists + Performs all necessary setup operations including creating the required + directory structure. Args: base_dir: The base directory path (typically from CBS_BASEDIR) @@ -95,10 +54,6 @@ def initialize_application(base_dir: str) -> None: logger.info(f"Initializing application with base directory: {base_dir}") - # Ensure directory structure ensure_base_structure(base_dir) - # Ensure remotes.json exists - ensure_remotes_json(base_dir) - logger.info("Application initialization complete") diff --git a/web/main.py b/web/main.py index 71107798..10c34df2 100755 --- a/web/main.py +++ b/web/main.py @@ -23,8 +23,13 @@ from web.core.limiter import limiter, rate_limit_exceeded_handler import ap_git -import metadata_manager import build_manager +from metadata_manager import ( + APSourceMetadataFetcher, + ManifestJson, + VehiclesManager, + VersionsManager, +) setup_logging() @@ -45,20 +50,26 @@ async def lifespan(app: FastAPI): recurse_submodules=True, ) - vehicles_manager = metadata_manager.VehiclesManager() + vehicles_manager = VehiclesManager() + + manifest_json = ManifestJson( + url=settings.ap_firmware_manifest_url, + cache_dir=settings.manifest_cache_dir, + ) - ap_src_metadata_fetcher = metadata_manager.APSourceMetadataFetcher( + ap_src_metadata_fetcher = APSourceMetadataFetcher( ap_repo=repo, caching_enabled=True, redis_host=settings.redis_host, redis_port=settings.redis_port, ) - versions_fetcher = metadata_manager.VersionsFetcher( + versions_manager = VersionsManager( + ap_repo=repo, remotes_json_path=settings.remotes_json_path, - ap_repo=repo + manifest_json=manifest_json, ) - versions_fetcher.reload_remotes_json() + versions_manager.refresh_all() build_mgr = build_manager.BuildManager( outdir=settings.outdir_parent, @@ -83,13 +94,14 @@ async def lifespan(app: FastAPI): ) inbuilt_builder_thread.start() - versions_fetcher.start() + versions_manager.start() cleaner.start() progress_updater.start() app.state.repo = repo app.state.ap_src_metadata_fetcher = ap_src_metadata_fetcher - app.state.versions_fetcher = versions_fetcher + app.state.manifest_json = manifest_json + app.state.versions_manager = versions_manager app.state.vehicles_manager = vehicles_manager app.state.build_manager = build_mgr app.state.inbuilt_builder = inbuilt_builder @@ -99,7 +111,7 @@ async def lifespan(app: FastAPI): yield # Shutdown - versions_fetcher.stop() + versions_manager.stop() cleaner.stop() progress_updater.stop() if inbuilt_builder is not None: diff --git a/web/schemas/__init__.py b/web/schemas/__init__.py index 22f58bad..dbeeaa5b 100644 --- a/web/schemas/__init__.py +++ b/web/schemas/__init__.py @@ -7,7 +7,7 @@ # Admin schemas from web.schemas.admin import ( - RefreshRemotesResponse, + RefreshVersionsResponse, ) # Build schemas @@ -35,7 +35,7 @@ __all__ = [ # Admin - "RefreshRemotesResponse", + "RefreshVersionsResponse", # Builds "BuildVersionInfo", "RemoteInfo", diff --git a/web/schemas/admin.py b/web/schemas/admin.py index ab8068e9..b04cc83d 100644 --- a/web/schemas/admin.py +++ b/web/schemas/admin.py @@ -3,10 +3,9 @@ from pydantic import BaseModel, Field -# --- Refresh Remotes Response --- -class RefreshRemotesResponse(BaseModel): - """Response schema for remote refresh operation.""" +class RefreshVersionsResponse(BaseModel): + """Response schema for version metadata refresh operation.""" remotes: List[str] = Field( ..., - description="List of remotes discovered in remotes.json file" + description="Git remotes synced after refreshing all version providers", ) diff --git a/web/services/admin.py b/web/services/admin.py index 6ba95c46..e8d1745c 100644 --- a/web/services/admin.py +++ b/web/services/admin.py @@ -14,20 +14,20 @@ class AdminService: """Service for managing administrative operations.""" - def __init__(self, remote_reload_token: str, versions_fetcher=None): + def __init__(self, admin_token: str, versions_manager=None): """ Initialize the admin service. Args: - remote_reload_token: Remote reload authentication token - versions_fetcher: VersionsFetcher instance for managing remotes + admin_token: Admin API authentication token + versions_manager: VersionsManager instance for managing remotes """ - self.remote_reload_token = remote_reload_token - self.versions_fetcher = versions_fetcher + self.admin_token = admin_token + self.versions_manager = versions_manager - async def verify_remote_reload_token(self, token: str) -> bool: + async def verify_admin_token(self, token: str) -> bool: """ - Verify that the provided token matches the expected remote reload token. + Verify that the provided token matches the expected admin token. Args: token: The token to verify @@ -38,33 +38,31 @@ async def verify_remote_reload_token(self, token: str) -> bool: if not token: return False - return token == self.remote_reload_token + return token == self.admin_token - async def refresh_remotes(self) -> List[str]: + async def refresh_versions(self) -> List[str]: """ - Trigger a refresh of remote metadata. + Refresh all version providers and sync git remotes. Returns: - List of remote names that were refreshed + List of git remote names synced after the refresh Raises: Exception: If refresh operation fails """ - logger.info("Triggering remote metadata refresh") + logger.info("Triggering version provider refresh") - # Reload remotes.json - self.versions_fetcher.reload_remotes_json() + self.versions_manager.refresh_all() - # Get list of remotes that are now available - remotes_info = self.versions_fetcher.get_all_remotes_info() - remotes_refreshed = [remote.name for remote in remotes_info] + remotes_info = self.versions_manager.get_all_remotes_info() + remotes_synced = [remote.name for remote in remotes_info] logger.info( - f"Successfully refreshed {len(remotes_refreshed)} remotes: " - f"{remotes_refreshed}" + f"Successfully refreshed version providers; " + f"synced {len(remotes_synced)} remotes: {remotes_synced}" ) - return remotes_refreshed + return remotes_synced def get_admin_service( @@ -82,14 +80,14 @@ def get_admin_service( AdminService instance initialized with app state dependencies Raises: - RuntimeError: If remote reload token is not configured + RuntimeError: If admin token is not configured """ - remote_reload_token = settings.remote_reload_token + admin_token = settings.admin_token - if remote_reload_token is None: - raise RuntimeError("Remote reload token not configured on server") + if admin_token is None: + raise RuntimeError("Admin token not configured on server") return AdminService( - remote_reload_token=remote_reload_token, - versions_fetcher=request.app.state.versions_fetcher + admin_token=admin_token, + versions_manager=request.app.state.versions_manager ) diff --git a/web/services/builds.py b/web/services/builds.py index 2725f667..9cc6b671 100644 --- a/web/services/builds.py +++ b/web/services/builds.py @@ -29,13 +29,13 @@ class BuildsService: def __init__( self, build_manager=None, - versions_fetcher=None, + versions_manager=None, ap_src_metadata_fetcher=None, repo=None, vehicles_manager=None ): self.manager = build_manager - self.versions_fetcher = versions_fetcher + self.versions_manager = versions_manager self.ap_src_metadata_fetcher = ap_src_metadata_fetcher self.repo = repo self.vehicles_manager = vehicles_manager @@ -66,7 +66,7 @@ def create_build( raise ValueError("vehicle_id is required") # Get version info using version_id - version_info = self.versions_fetcher.get_version_info( + version_info = self.versions_manager.get_version_info( vehicle_id=vehicle_id, version_id=build_request.version_id ) @@ -77,7 +77,7 @@ def create_build( commit_ref = version_info.commit_ref # Validate remote - remote_info = self.versions_fetcher.get_remote_info(remote_name) + remote_info = self.versions_manager.get_remote_info(remote_name) if remote_info is None: raise ValueError(f"Remote {remote_name} is not whitelisted") @@ -395,7 +395,7 @@ def get_builds_service(request: Request) -> BuildsService: """ return BuildsService( build_manager=request.app.state.build_manager, - versions_fetcher=request.app.state.versions_fetcher, + versions_manager=request.app.state.versions_manager, ap_src_metadata_fetcher=request.app.state.ap_src_metadata_fetcher, repo=request.app.state.repo, vehicles_manager=request.app.state.vehicles_manager, diff --git a/web/services/vehicles.py b/web/services/vehicles.py index 1ebe6662..ecf4d147 100644 --- a/web/services/vehicles.py +++ b/web/services/vehicles.py @@ -23,11 +23,11 @@ class VehiclesService: """Service for managing vehicles, versions, boards, and features.""" def __init__(self, vehicle_manager=None, - versions_fetcher=None, + versions_manager=None, ap_src_metadata_fetcher=None, repo=None): self.vehicles_manager = vehicle_manager - self.versions_fetcher = versions_fetcher + self.versions_manager = versions_manager self.ap_src_metadata_fetcher = ap_src_metadata_fetcher self.repo = repo @@ -58,7 +58,7 @@ def get_versions( """Get all versions available for a specific vehicle.""" versions = [] - for version_info in self.versions_fetcher.get_versions_for_vehicle( + for version_info in self.versions_manager.get_versions_for_vehicle( vehicle_id=vehicle_id ): # Apply type filter if provided @@ -107,7 +107,7 @@ def get_boards( ) -> List[BoardOut]: """Get all boards available for a specific vehicle version.""" # Get version info - version_info = self.versions_fetcher.get_version_info( + version_info = self.versions_manager.get_version_info( vehicle_id=vehicle_id, version_id=version_id ) @@ -162,7 +162,7 @@ def get_features( vehicle version/board. """ # Get version info - version_info = self.versions_fetcher.get_version_info( + version_info = self.versions_manager.get_version_info( vehicle_id=vehicle_id, version_id=version_id ) @@ -267,7 +267,7 @@ def get_vehicles_service(request: Request) -> VehiclesService: """ return VehiclesService( vehicle_manager=request.app.state.vehicles_manager, - versions_fetcher=request.app.state.versions_fetcher, + versions_manager=request.app.state.versions_manager, ap_src_metadata_fetcher=request.app.state.ap_src_metadata_fetcher, repo=request.app.state.repo, ) From 02d606cce9de4d8fb5158dceee2266b2c4eb413c Mon Sep 17 00:00:00 2001 From: Shiv Tyagi Date: Sat, 11 Jul 2026 09:40:46 +0530 Subject: [PATCH 4/8] docker-compose: rename CBS_REMOTES_RELOAD_TOKEN to CBS_ADMIN_TOKEN --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 794a7bba..4ae3fe1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: CBS_LOG_LEVEL: ${CBS_LOG_LEVEL:-INFO} CBS_ENABLE_INBUILT_BUILDER: 0 CBS_GITHUB_ACCESS_TOKEN: ${CBS_GITHUB_ACCESS_TOKEN} - CBS_REMOTES_RELOAD_TOKEN: ${CBS_REMOTES_RELOAD_TOKEN} + CBS_ADMIN_TOKEN: ${CBS_ADMIN_TOKEN} PYTHONPATH: /app CBS_BUILD_TIMEOUT_SEC: ${CBS_BUILD_TIMEOUT_SEC:-900} FORWARDED_ALLOW_IPS: ${FORWARDED_ALLOW_IPS:-*} From 83f5a28ffc9b311c2ab505f1a6a1395b97798808 Mon Sep 17 00:00:00 2001 From: Shiv Tyagi Date: Sat, 11 Jul 2026 09:41:42 +0530 Subject: [PATCH 5/8] tests: add/update tests for provider based versions manager --- .../fixtures/manifest_sample.json | 62 ++++++ .../metadata_manager/test_firmware_server.py | 54 +++++ .../metadata_manager/test_versions_manager.py | 123 +++++++++++ tests/web/conftest.py | 43 ++-- tests/web/test_admin_api.py | 48 ++--- tests/web/test_admin_service.py | 34 +-- tests/web/test_builds_service.py | 25 ++- tests/web/test_config.py | 28 +-- tests/web/test_vehicles_service.py | 202 +++++++++--------- 9 files changed, 426 insertions(+), 193 deletions(-) create mode 100644 tests/metadata_manager/fixtures/manifest_sample.json create mode 100644 tests/metadata_manager/test_firmware_server.py create mode 100644 tests/metadata_manager/test_versions_manager.py diff --git a/tests/metadata_manager/fixtures/manifest_sample.json b/tests/metadata_manager/fixtures/manifest_sample.json new file mode 100644 index 00000000..3cf4a734 --- /dev/null +++ b/tests/metadata_manager/fixtures/manifest_sample.json @@ -0,0 +1,62 @@ +{ + "format-version": "1.0.0", + "firmware": [ + { + "vehicletype": "Copter", + "platform": "CubeOrange", + "git-sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "url": "https://firmware.ardupilot.org/Copter/stable-4.6.3/CubeOrange/arducopter.apj", + "mav-firmware-version-type": "STABLE-4.6.3", + "mav-firmware-version": "4.6.3", + "format": "apj", + "latest": 0, + "mav-type": "QUADROTOR", + "image_size": 100 + }, + { + "vehicletype": "Copter", + "platform": "CubeOrange-heli", + "git-sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "url": "https://firmware.ardupilot.org/Copter/stable-4.6.3/CubeOrange-heli/arducopter-heli.apj", + "mav-firmware-version-type": "STABLE-4.6.3", + "mav-firmware-version": "4.6.3", + "format": "apj", + "latest": 0, + "mav-type": "HELICOPTER", + "image_size": 100 + }, + { + "vehicletype": "Copter", + "platform": "CubeOrange", + "git-sha": "cccccccccccccccccccccccccccccccccccccccc", + "url": "https://firmware.ardupilot.org/Copter/beta/CubeOrange/arducopter.apj", + "mav-firmware-version-type": "BETA", + "mav-firmware-version": "4.7.0", + "format": "apj", + "latest": 0, + "mav-type": "QUADROTOR" + }, + { + "vehicletype": "AntennaTracker", + "platform": "CubeOrange", + "git-sha": "dddddddddddddddddddddddddddddddddddddddd", + "url": "https://firmware.ardupilot.org/AntennaTracker/stable-4.6.3/CubeOrange/antennatracker.apj", + "mav-firmware-version-type": "STABLE-4.6.3", + "mav-firmware-version": "4.6.3", + "format": "apj", + "latest": 0, + "mav-type": "ANTENNA_TRACKER" + }, + { + "vehicletype": "Copter", + "platform": "CubeOrange", + "git-sha": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "url": "https://firmware.ardupilot.org/Copter/latest/CubeOrange/arducopter.apj", + "mav-firmware-version-type": "DEV", + "mav-firmware-version": "4.8.0", + "format": "apj", + "latest": 1, + "mav-type": "QUADROTOR" + } + ] +} diff --git a/tests/metadata_manager/test_firmware_server.py b/tests/metadata_manager/test_firmware_server.py new file mode 100644 index 00000000..518720a5 --- /dev/null +++ b/tests/metadata_manager/test_firmware_server.py @@ -0,0 +1,54 @@ +import json +from pathlib import Path +from unittest.mock import Mock, patch + +from metadata_manager import ManifestClient, ManifestIndex +from metadata_manager.firmware_server.client import _CacheMeta + + +FIXTURES_DIR = Path(__file__).parent / "fixtures" +SAMPLE_MANIFEST = json.loads((FIXTURES_DIR / "manifest_sample.json").read_text()) + + +class TestManifestIndex: + def test_builds_releases_for_copter_heli_and_tracker(self): + index = ManifestIndex.build(SAMPLE_MANIFEST) + + copter = index.get_releases("copter") + stable = [r for r in copter if r.release_type == "stable" and r.version_number == "4.6.3"] + assert len(stable) == 1 + assert stable[0].commit_reference.startswith("aaaa") + + heli = index.get_releases("heli") + heli_stable = [r for r in heli if r.release_type == "stable" and r.version_number == "4.6.3"] + assert len(heli_stable) == 1 + assert heli_stable[0].commit_reference.startswith("bbbb") + + tracker = index.get_releases("tracker") + assert any(r.version_number == "4.6.3" for r in tracker) + + latest = [r for r in copter if r.release_type == "latest"] + assert len(latest) == 1 + assert latest[0].commit_reference.startswith("eeee") + assert latest[0].version_number == "NA" + assert latest[0].ap_build_artifacts_url == ( + "https://firmware.ardupilot.org/Copter/latest" + ) + + +class TestManifestClientCache: + def test_uses_cache_on_304(self, tmp_path): + client = ManifestClient( + url="https://example.com/manifest.json", + cache_dir=str(tmp_path), + ) + client._write_cache( + b'{"format-version":"1.0.0","firmware":[]}', + _CacheMeta(etag='"abc"', last_modified="Mon, 01 Jan 2024 00:00:00 GMT"), + ) + + response = Mock(status_code=304, headers={}, content=b"") + with patch("metadata_manager.firmware_server.client.requests.get", return_value=response): + raw = client.fetch_raw() + + assert raw == b'{"format-version":"1.0.0","firmware":[]}' diff --git a/tests/metadata_manager/test_versions_manager.py b/tests/metadata_manager/test_versions_manager.py new file mode 100644 index 00000000..6fac5d96 --- /dev/null +++ b/tests/metadata_manager/test_versions_manager.py @@ -0,0 +1,123 @@ +from unittest.mock import Mock, PropertyMock + +import pytest + +from metadata_manager import ( + DEFAULT_WHITELISTED_FORK_REMOTES, + ForkRemoteSpec, + ManifestJson, + ManifestJsonVersionsProvider, + RemoteInfo, + VehiclesManager, + VersionInfo, + VersionsManager, +) + + +@pytest.fixture +def vehicles_manager(): + existing = VehiclesManager.get_singleton() + if existing is not None: + return existing + return VehiclesManager() + + +@pytest.fixture +def versions_manager(vehicles_manager, tmp_path): + VersionsManager._VersionsManager__singleton = None + repo = Mock() + manager = VersionsManager( + ap_repo=repo, + remotes_json_path=str(tmp_path / "missing-remotes.json"), + providers=[], + ) + yield manager + VersionsManager._VersionsManager__singleton = None + + +class TestManifestJsonVersionsProvider: + def test_is_available_when_manifest_is_available(self, vehicles_manager): + manifest_json = Mock(spec=ManifestJson) + type(manifest_json).is_available = PropertyMock(return_value=True) + + provider = ManifestJsonVersionsProvider(manifest_json) + + assert provider.is_available is True + + def test_is_unavailable_when_manifest_is_unavailable(self, vehicles_manager): + manifest_json = Mock(spec=ManifestJson) + type(manifest_json).is_available = PropertyMock(return_value=False) + + provider = ManifestJsonVersionsProvider(manifest_json) + + assert provider.is_available is False + assert provider.get_versions("copter") == [] + assert provider.get_remotes() == [] + + def test_refresh_delegates_to_manifest_json(self, vehicles_manager): + manifest_json = Mock(spec=ManifestJson) + provider = ManifestJsonVersionsProvider(manifest_json) + + provider.refresh() + + manifest_json.refresh.assert_called_once() + + +class TestVersionsManagerDedup: + def _make_version(self, release_type: str, version_number: str, commit_ref: str): + remote = RemoteInfo("ardupilot", "https://github.com/ardupilot/ardupilot.git") + return VersionInfo( + remote_info=remote, + commit_ref=commit_ref, + release_type=release_type, + version_number=version_number, + ap_build_artifacts_url=None, + ) + + def test_stable_wins_when_beta_is_seen_first(self, versions_manager): + commit = "03c12698df56c600f5b3d39f8b17d414ea6d2a48" + beta = self._make_version("beta", "4.5.0", commit) + stable = self._make_version("stable", "4.5.0", commit) + + beta_provider = Mock() + beta_provider.get_versions.return_value = [beta] + stable_provider = Mock() + stable_provider.get_versions.return_value = [stable] + versions_manager._providers = [beta_provider, stable_provider] + + versions = versions_manager.get_versions_for_vehicle("sub") + + assert len(versions) == 1 + assert versions[0].release_type == "stable" + assert versions[0].version_number == "4.5.0" + + def test_stable_wins_when_stable_is_seen_first(self, versions_manager): + commit = "03c12698df56c600f5b3d39f8b17d414ea6d2a48" + beta = self._make_version("beta", "4.5.0", commit) + stable = self._make_version("stable", "4.5.0", commit) + + stable_provider = Mock() + stable_provider.get_versions.return_value = [stable] + beta_provider = Mock() + beta_provider.get_versions.return_value = [beta] + versions_manager._providers = [stable_provider, beta_provider] + + versions = versions_manager.get_versions_for_vehicle("sub") + + assert len(versions) == 1 + assert versions[0].release_type == "stable" + + +class TestForkRemoteSpec: + def test_default_rmackay9_uses_custom_repo_name(self): + rmackay9 = next( + spec for spec in DEFAULT_WHITELISTED_FORK_REMOTES if spec.owner == "rmackay9" + ) + assert rmackay9.repo == "rmackay9-ardupilot" + assert rmackay9.github_repo == "rmackay9/rmackay9-ardupilot" + assert rmackay9.url == "https://github.com/rmackay9/rmackay9-ardupilot.git" + + def test_fork_remote_spec_url(self): + spec = ForkRemoteSpec(owner="example", repo="my-ardupilot-fork") + assert spec.github_repo == "example/my-ardupilot-fork" + assert spec.url == "https://github.com/example/my-ardupilot-fork.git" diff --git a/tests/web/conftest.py b/tests/web/conftest.py index 41e4690a..51b931dc 100644 --- a/tests/web/conftest.py +++ b/tests/web/conftest.py @@ -25,7 +25,7 @@ "url": "https://github.com/test/ardupilot.git", "vehicles": [ { - "name": "Copter", + "id": "copter", "releases": [ { "release_type": "latest", @@ -40,7 +40,7 @@ ] }, { - "name": "Plane", + "id": "plane", "releases": [ { "release_type": "latest", @@ -56,7 +56,7 @@ "url": "https://github.com/another/ardupilot.git", "vehicles": [ { - "name": "Rover", + "id": "rover", "releases": [ { "release_type": "Custom", @@ -90,10 +90,10 @@ def test_base_dir() -> Generator[str, None, None]: with open(remotes_json_path, "w") as f: json.dump(TEST_REMOTES_JSON, f, indent=2) - # Create remote reload token file - token_file_path = os.path.join(temp_dir, "secrets", "reload_token") + # Create admin token file + token_file_path = os.path.join(temp_dir, "secrets", "admin_token") with open(token_file_path, "w") as f: - f.write("test-remote-reload-token-12345") + f.write("test-admin-token-12345") yield temp_dir @@ -130,9 +130,9 @@ def mock_ap_src_metadata_fetcher(): @pytest.fixture -def mock_versions_fetcher(test_base_dir): +def mock_versions_manager(test_base_dir): """ - Create a mock VersionsFetcher that doesn't actually fetch versions. + Create a mock VersionsManager that doesn't actually fetch versions. This allows tests to run without starting background threads or making actual git operations. @@ -141,27 +141,27 @@ def mock_versions_fetcher(test_base_dir): test_base_dir: Test base directory fixture Returns: - Mock: Mock VersionsFetcher instance + Mock: Mock VersionsManager instance """ - from metadata_manager.versions_fetcher import RemoteInfo + from metadata_manager import RemoteInfo - mock_fetcher = Mock() + mock_manager = Mock() - # Mock the reload_remotes_json method - mock_fetcher.reload_remotes_json = Mock(return_value=None) + # Mock the refresh methods + mock_manager.refresh_all = Mock(return_value=None) # Mock get_all_remotes_info to return test remotes test_remotes = [ RemoteInfo(name="test-remote-1", url="https://github.com/test/ardupilot.git"), RemoteInfo(name="test-remote-2", url="https://github.com/another/ardupilot.git") ] - mock_fetcher.get_all_remotes_info = Mock(return_value=test_remotes) + mock_manager.get_all_remotes_info = Mock(return_value=test_remotes) # Mock start/stop methods (no-op for tests) - mock_fetcher.start = Mock() - mock_fetcher.stop = Mock() + mock_manager.start = Mock() + mock_manager.stop = Mock() - return mock_fetcher + return mock_manager @pytest.fixture @@ -191,7 +191,6 @@ def mock_vehicles_manager(): Mock: Mock VehiclesManager instance """ mock_manager = Mock() - mock_manager.get_vehicle_names = Mock(return_value=["Copter", "Plane", "Rover"]) return mock_manager @@ -199,7 +198,7 @@ def mock_vehicles_manager(): def app_with_mocked_dependencies( test_base_dir, mock_git_repo, - mock_versions_fetcher, + mock_versions_manager, mock_build_manager, mock_vehicles_manager, ): @@ -215,7 +214,7 @@ def app_with_mocked_dependencies( Args: test_base_dir: Test base directory mock_git_repo: Mock git repository - mock_versions_fetcher: Mock versions fetcher + mock_versions_manager: Mock versions manager mock_build_manager: Mock build manager mock_vehicles_manager: Mock vehicles manager @@ -240,7 +239,7 @@ async def test_lifespan(app: FastAPI): """Test lifespan that doesn't start background tasks.""" # Setup: Attach mocked dependencies to app state app.state.repo = mock_git_repo - app.state.versions_fetcher = mock_versions_fetcher + app.state.versions_manager = mock_versions_manager app.state.vehicles_manager = mock_vehicles_manager app.state.build_manager = mock_build_manager app.state.limiter = limiter @@ -250,7 +249,7 @@ async def test_lifespan(app: FastAPI): app.state.ap_src_metadata_fetcher = mock_ap_src_fetcher # Don't start background tasks in test mode - # versions_fetcher.start() + # versions_manager.start() # cleaner.start() # progress_updater.start() diff --git a/tests/web/test_admin_api.py b/tests/web/test_admin_api.py index 49160241..9f24a8a9 100644 --- a/tests/web/test_admin_api.py +++ b/tests/web/test_admin_api.py @@ -9,11 +9,11 @@ from web.core.config import get_settings -class TestAdminRefreshRemotesEndpoint: - """Test suite for the /admin/refresh_remotes endpoint.""" +class TestAdminRefreshVersionsEndpoint: + """Test suite for the /admin/refresh_versions endpoint.""" - AUTH_HEADERS = {"Authorization": "Bearer test-remote-reload-token-12345"} - TEST_TOKEN = "test-remote-reload-token-12345" + AUTH_HEADERS = {"Authorization": "Bearer test-admin-token-12345"} + TEST_TOKEN = "test-admin-token-12345" @staticmethod @contextmanager @@ -25,8 +25,8 @@ def override_settings(client, mock_settings): finally: client.app.dependency_overrides.pop(get_settings, None) - def test_refresh_remotes_success(self, client, test_base_dir): - """Test successful refresh of remotes with valid auth and verifies against remotes.json.""" + def test_refresh_versions_success(self, client, test_base_dir): + """Test successful refresh with valid auth and verifies against remotes.json.""" import os import json @@ -37,10 +37,10 @@ def test_refresh_remotes_success(self, client, test_base_dir): initial_remotes = json.load(f) mock_settings = Mock() - mock_settings.remote_reload_token = self.TEST_TOKEN + mock_settings.admin_token = self.TEST_TOKEN with self.override_settings(client, mock_settings): response = client.post( - "/api/v1/admin/refresh_remotes", + "/api/v1/admin/refresh_versions", headers=self.AUTH_HEADERS ) @@ -54,22 +54,22 @@ def test_refresh_remotes_success(self, client, test_base_dir): for name in expected_names: assert name in data["remotes"] - def test_refresh_remotes_no_auth(self, client): + def test_refresh_versions_no_auth(self, client): """Test refresh without authentication - should fail.""" mock_settings = Mock() - mock_settings.remote_reload_token = self.TEST_TOKEN + mock_settings.admin_token = self.TEST_TOKEN with self.override_settings(client, mock_settings): - response = client.post("/api/v1/admin/refresh_remotes") + response = client.post("/api/v1/admin/refresh_versions") assert response.status_code == status.HTTP_401_UNAUTHORIZED - def test_refresh_remotes_invalid_token(self, client): + def test_refresh_versions_invalid_token(self, client): """Test refresh with invalid token - should fail.""" mock_settings = Mock() - mock_settings.remote_reload_token = self.TEST_TOKEN + mock_settings.admin_token = self.TEST_TOKEN with self.override_settings(client, mock_settings): response = client.post( - "/api/v1/admin/refresh_remotes", + "/api/v1/admin/refresh_versions", headers={"Authorization": "Bearer invalid-token-xyz"} ) @@ -79,31 +79,31 @@ def test_refresh_remotes_invalid_token(self, client): assert "detail" in data assert "Invalid authentication token" in data["detail"] - def test_refresh_remotes_malformed_auth_header(self, client): + def test_refresh_versions_malformed_auth_header(self, client): """Test refresh with malformed authorization header.""" mock_settings = Mock() - mock_settings.remote_reload_token = self.TEST_TOKEN + mock_settings.admin_token = self.TEST_TOKEN with self.override_settings(client, mock_settings): response = client.post( - "/api/v1/admin/refresh_remotes", - headers={"Authorization": "test-remote-reload-token-12345"} + "/api/v1/admin/refresh_versions", + headers={"Authorization": "test-admin-token-12345"} ) assert response.status_code == status.HTTP_401_UNAUTHORIZED - def test_refresh_remotes_empty_token(self, client): + def test_refresh_versions_empty_token(self, client): """Test refresh with empty token.""" mock_settings = Mock() - mock_settings.remote_reload_token = self.TEST_TOKEN + mock_settings.admin_token = self.TEST_TOKEN with self.override_settings(client, mock_settings): response = client.post( - "/api/v1/admin/refresh_remotes", + "/api/v1/admin/refresh_versions", headers={"Authorization": "Bearer "} ) assert response.status_code == status.HTTP_401_UNAUTHORIZED - def test_refresh_remotes_method_not_allowed(self, client): + def test_refresh_versions_method_not_allowed(self, client): """Test that only POST method is allowed.""" disallowed_methods = [ ("GET", client.get), @@ -113,11 +113,11 @@ def test_refresh_remotes_method_not_allowed(self, client): ] mock_settings = Mock() - mock_settings.remote_reload_token = self.TEST_TOKEN + mock_settings.admin_token = self.TEST_TOKEN with self.override_settings(client, mock_settings): for method_name, method_func in disallowed_methods: response = method_func( - "/api/v1/admin/refresh_remotes", + "/api/v1/admin/refresh_versions", headers=self.AUTH_HEADERS ) assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED, \ diff --git a/tests/web/test_admin_service.py b/tests/web/test_admin_service.py index dc9271c1..a54e864e 100644 --- a/tests/web/test_admin_service.py +++ b/tests/web/test_admin_service.py @@ -12,46 +12,46 @@ class TestAdminService: @pytest.mark.asyncio async def test_verify_token_success(self): """Test successful token verification.""" - admin_service = AdminService(remote_reload_token="valid-token") - result = await admin_service.verify_remote_reload_token("valid-token") + admin_service = AdminService(admin_token="valid-token") + result = await admin_service.verify_admin_token("valid-token") assert result is True @pytest.mark.asyncio async def test_verify_token_failure(self): """Test token verification with incorrect token.""" - admin_service = AdminService(remote_reload_token="valid-token") - result = await admin_service.verify_remote_reload_token("invalid-token") + admin_service = AdminService(admin_token="valid-token") + result = await admin_service.verify_admin_token("invalid-token") assert result is False @pytest.mark.asyncio - async def test_refresh_remotes_success(self, mock_versions_fetcher): - """Test successful refresh of remote metadata.""" + async def test_refresh_versions_success(self, mock_versions_manager): + """Test successful refresh of version metadata.""" admin_service = AdminService( - remote_reload_token="some-token", - versions_fetcher=mock_versions_fetcher + admin_token="some-token", + versions_manager=mock_versions_manager ) - remotes = await admin_service.refresh_remotes() + remotes = await admin_service.refresh_versions() assert len(remotes) == 2 assert "test-remote-1" in remotes assert "test-remote-2" in remotes - mock_versions_fetcher.reload_remotes_json.assert_called_once() - mock_versions_fetcher.get_all_remotes_info.assert_called_once() + mock_versions_manager.refresh_all.assert_called_once() + mock_versions_manager.get_all_remotes_info.assert_called_once() @pytest.mark.asyncio - async def test_refresh_remotes_empty_result(self): + async def test_refresh_versions_empty_result(self): """Test refresh when no remotes are configured.""" mock_fetcher = Mock() - mock_fetcher.reload_remotes_json = Mock() + mock_fetcher.refresh_all = Mock() mock_fetcher.get_all_remotes_info = Mock(return_value=[]) admin_service = AdminService( - remote_reload_token="some-token", - versions_fetcher=mock_fetcher + admin_token="some-token", + versions_manager=mock_fetcher ) - remotes = await admin_service.refresh_remotes() + remotes = await admin_service.refresh_versions() assert len(remotes) == 0 - mock_fetcher.reload_remotes_json.assert_called_once() + mock_fetcher.refresh_all.assert_called_once() diff --git a/tests/web/test_builds_service.py b/tests/web/test_builds_service.py index b9648dbb..03b2453d 100644 --- a/tests/web/test_builds_service.py +++ b/tests/web/test_builds_service.py @@ -7,7 +7,7 @@ import build_manager as bm from metadata_manager import RemoteInfo as ManagerRemoteInfo -from metadata_manager.versions_fetcher import RemoteInfo, VersionInfo +from metadata_manager import RemoteInfo, VersionInfo from web.schemas import BuildRequest from web.services.builds import BuildsService @@ -15,14 +15,14 @@ @pytest.fixture def service( mock_build_manager, - mock_versions_fetcher, + mock_versions_manager, mock_ap_src_metadata_fetcher, mock_git_repo, mock_vehicles_manager, ): """Create a BuildsService instance with mocked dependencies.""" - mock_versions_fetcher.get_version_info.return_value = make_version_info() - mock_versions_fetcher.get_remote_info.return_value = RemoteInfo( + mock_versions_manager.get_version_info.return_value = make_version_info() + mock_versions_manager.get_remote_info.return_value = RemoteInfo( name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git" ) mock_ap_src_metadata_fetcher.get_boards.return_value = ["MatekH743", "CubeOrange"] @@ -37,11 +37,10 @@ def service( mock_vehicles_manager.get_vehicle_by_id = Mock( side_effect=lambda vid: vehicles.get(vid) ) - mock_vehicles_manager.get_vehicle_names = Mock(return_value=[v.name for v in vehicles.values()]) return BuildsService( build_manager=mock_build_manager, - versions_fetcher=mock_versions_fetcher, + versions_manager=mock_versions_manager, ap_src_metadata_fetcher=mock_ap_src_metadata_fetcher, repo=mock_git_repo, vehicles_manager=mock_vehicles_manager, @@ -179,10 +178,10 @@ def test_create_build_raises_value_error_for_missing_board_id( service.create_build(request) def test_create_build_raises_value_error_for_invalid_version( - self, service, mock_versions_fetcher + self, service, mock_versions_manager ): """ValueError is raised when the version_id is not found.""" - mock_versions_fetcher.get_version_info.return_value = None + mock_versions_manager.get_version_info.return_value = None request = BuildRequest( vehicle_id="copter", board_id="MatekH743", @@ -194,10 +193,10 @@ def test_create_build_raises_value_error_for_invalid_version( service.create_build(request) def test_create_build_queries_version_info_with_correct_params( - self, service, mock_versions_fetcher + self, service, mock_versions_manager ): """get_version_info is called with the correct vehicle_id and version_id.""" - mock_versions_fetcher.get_version_info.return_value = None + mock_versions_manager.get_version_info.return_value = None request = BuildRequest( vehicle_id="plane", board_id="CubeOrange", @@ -208,16 +207,16 @@ def test_create_build_queries_version_info_with_correct_params( with pytest.raises(ValueError): service.create_build(request) - mock_versions_fetcher.get_version_info.assert_called_once_with( + mock_versions_manager.get_version_info.assert_called_once_with( vehicle_id="plane", version_id="plane-4.4.0-stable", ) def test_create_build_raises_value_error_when_remote_not_found( - self, service, mock_versions_fetcher + self, service, mock_versions_manager ): """ValueError is raised when the remote is not found.""" - mock_versions_fetcher.get_remote_info.return_value = None + mock_versions_manager.get_remote_info.return_value = None request = BuildRequest( vehicle_id="some-vehicle", board_id="some-board", diff --git a/tests/web/test_config.py b/tests/web/test_config.py index 3456546f..c4604757 100644 --- a/tests/web/test_config.py +++ b/tests/web/test_config.py @@ -44,27 +44,27 @@ def test_env_var_overrides(self): assert settings.enable_inbuilt_builder is False -class TestRemoteReloadToken: - """Test suite for remote_reload_token property.""" +class TestAdminToken: + """Test suite for admin_token property.""" def test_token_from_file(self, tmp_path): """Test that token is read from file when it exists.""" secrets_dir = tmp_path / 'secrets' secrets_dir.mkdir() - token_file = secrets_dir / 'reload_token' + token_file = secrets_dir / 'admin_token' expected_token = "test-token-from-file" token_file.write_text(f" {expected_token} \n") # Test whitespace stripping with patch.dict(os.environ, {"CBS_BASEDIR": str(tmp_path)}, clear=True): settings = Settings() - assert settings.remote_reload_token == expected_token + assert settings.admin_token == expected_token def test_token_file_takes_precedence_over_env(self, tmp_path): """Test that token from file takes precedence over environment variable.""" secrets_dir = tmp_path / 'secrets' secrets_dir.mkdir() - token_file = secrets_dir / 'reload_token' + token_file = secrets_dir / 'admin_token' file_token = "token-from-file" env_token = "token-from-env" @@ -73,10 +73,10 @@ def test_token_file_takes_precedence_over_env(self, tmp_path): with patch.dict(os.environ, { "CBS_BASEDIR": str(tmp_path), - "CBS_REMOTES_RELOAD_TOKEN": env_token + "CBS_ADMIN_TOKEN": env_token }, clear=True): settings = Settings() - assert settings.remote_reload_token == file_token + assert settings.admin_token == file_token def test_token_from_env_when_file_not_found(self, tmp_path): """Test that token falls back to environment variable when file doesn't exist.""" @@ -84,10 +84,10 @@ def test_token_from_env_when_file_not_found(self, tmp_path): with patch.dict(os.environ, { "CBS_BASEDIR": str(tmp_path), - "CBS_REMOTES_RELOAD_TOKEN": expected_token + "CBS_ADMIN_TOKEN": expected_token }, clear=True): settings = Settings() - assert settings.remote_reload_token == expected_token + assert settings.admin_token == expected_token def test_token_from_env_on_file_read_error(self, tmp_path): """Test that token falls back to env var when file cannot be read.""" @@ -95,23 +95,23 @@ def test_token_from_env_on_file_read_error(self, tmp_path): with patch.dict(os.environ, { "CBS_BASEDIR": str(tmp_path), - "CBS_REMOTES_RELOAD_TOKEN": env_token + "CBS_ADMIN_TOKEN": env_token }, clear=True): with patch("builtins.open", side_effect=PermissionError("No access")): settings = Settings() - assert settings.remote_reload_token == env_token + assert settings.admin_token == env_token def test_token_none_when_not_configured(self, tmp_path): """Test that token is None when neither file nor env var is set.""" with patch.dict(os.environ, {"CBS_BASEDIR": str(tmp_path)}, clear=True): settings = Settings() - assert settings.remote_reload_token is None + assert settings.admin_token is None def test_token_none_when_env_is_empty_string(self, tmp_path): """Test that token is None when env var is empty string.""" with patch.dict(os.environ, { "CBS_BASEDIR": str(tmp_path), - "CBS_REMOTES_RELOAD_TOKEN": "" + "CBS_ADMIN_TOKEN": "" }, clear=True): settings = Settings() - assert settings.remote_reload_token is None + assert settings.admin_token is None diff --git a/tests/web/test_vehicles_service.py b/tests/web/test_vehicles_service.py index 35eb60b2..bb8f31c3 100644 --- a/tests/web/test_vehicles_service.py +++ b/tests/web/test_vehicles_service.py @@ -4,16 +4,19 @@ import pytest from unittest.mock import Mock -from metadata_manager import Vehicle -from metadata_manager.versions_fetcher import VersionInfo, RemoteInfo +from metadata_manager import ( + RemoteInfo, + Vehicle, + VersionInfo, +) from web.services.vehicles import VehiclesService @pytest.fixture -def service(mock_vehicles_manager, mock_versions_fetcher, mock_ap_src_metadata_fetcher, mock_git_repo): +def service(mock_vehicles_manager, mock_versions_manager, mock_ap_src_metadata_fetcher, mock_git_repo): return VehiclesService( vehicle_manager=mock_vehicles_manager, - versions_fetcher=mock_versions_fetcher, + versions_manager=mock_versions_manager, ap_src_metadata_fetcher=mock_ap_src_metadata_fetcher, repo=mock_git_repo, ) @@ -31,14 +34,12 @@ def test_get_all_vehicles_returns_all(self, service, mock_vehicles_manager): id="copter", name="Copter", ap_source_subdir="ArduCopter", - fw_server_vehicle_sdir="Copter", waf_build_command="copter" ), Vehicle( id="plane", name="Plane", ap_source_subdir="ArduPlane", - fw_server_vehicle_sdir="Plane", waf_build_command="plane" ), ] @@ -64,7 +65,6 @@ def test_get_all_vehicles_single(self, service, mock_vehicles_manager): id="copter", name="Copter", ap_source_subdir="ArduCopter", - fw_server_vehicle_sdir="Copter", waf_build_command="copter" ), ] @@ -80,21 +80,18 @@ def test_get_all_vehicles_sorted_by_name(self, service, mock_vehicles_manager): id="plane", name="Plane", ap_source_subdir="ArduPlane", - fw_server_vehicle_sdir="Plane", waf_build_command="plane" ), Vehicle( id="copter", name="Copter", ap_source_subdir="ArduCopter", - fw_server_vehicle_sdir="Copter", waf_build_command="copter" ), Vehicle( id="rover", name="Rover", ap_source_subdir="ArduRover", - fw_server_vehicle_sdir="Rover", waf_build_command="rover" ), ] @@ -118,7 +115,6 @@ def test_get_vehicle_found(self, service, mock_vehicles_manager): id="copter", name="Copter", ap_source_subdir="ArduCopter", - fw_server_vehicle_sdir="Copter", waf_build_command="copter" ) vehicle = service.get_vehicle("copter") @@ -143,16 +139,16 @@ def test_get_vehicle_calls_manager_with_correct_id(self, service, mock_vehicles_ # Tests for get_versions - def test_get_versions_empty(self, service, mock_versions_fetcher): + def test_get_versions_empty(self, service, mock_versions_manager): """Test that an empty list is returned when no versions exist.""" - mock_versions_fetcher.get_versions_for_vehicle.return_value = [] + mock_versions_manager.get_versions_for_vehicle.return_value = [] versions = service.get_versions("copter") assert versions == [] - def test_get_versions_single(self, service, mock_versions_fetcher): + def test_get_versions_single(self, service, mock_versions_manager): """Test fetching versions when only one version exists.""" - mock_versions_fetcher.get_versions_for_vehicle.return_value = [ + mock_versions_manager.get_versions_for_vehicle.return_value = [ VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/heads/master", @@ -165,9 +161,9 @@ def test_get_versions_single(self, service, mock_versions_fetcher): assert len(versions) == 1 - def test_get_versions_many(self, service, mock_versions_fetcher): + def test_get_versions_many(self, service, mock_versions_manager): """Test fetching versions when multiple versions exist.""" - mock_versions_fetcher.get_versions_for_vehicle.return_value = [ + mock_versions_manager.get_versions_for_vehicle.return_value = [ VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/heads/master", @@ -194,9 +190,9 @@ def test_get_versions_many(self, service, mock_versions_fetcher): assert len(versions) == 3 - def test_get_versions_sorted_by_name(self, service, mock_versions_fetcher): + def test_get_versions_sorted_by_name(self, service, mock_versions_manager): """Test that versions are returned sorted by their display name.""" - mock_versions_fetcher.get_versions_for_vehicle.return_value = [ + mock_versions_manager.get_versions_for_vehicle.return_value = [ VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.5.0", @@ -225,21 +221,21 @@ def test_get_versions_sorted_by_name(self, service, mock_versions_fetcher): assert names == sorted(names) def test_get_versions_calls_fetcher_once_with_correct_vehicle_id( - self, service, mock_versions_fetcher + self, service, mock_versions_manager ): """Test that get_versions calls the fetcher exactly once with the correct vehicle_id.""" - mock_versions_fetcher.get_versions_for_vehicle.return_value = [] + mock_versions_manager.get_versions_for_vehicle.return_value = [] service.get_versions("copter") - mock_versions_fetcher.get_versions_for_vehicle.assert_called_once_with( + mock_versions_manager.get_versions_for_vehicle.assert_called_once_with( vehicle_id="copter" ) def test_get_versions_type_filter_keeps_matching( - self, service, mock_versions_fetcher + self, service, mock_versions_manager ): """Test that type_filter returns only versions of the specified type.""" - mock_versions_fetcher.get_versions_for_vehicle.return_value = [ + mock_versions_manager.get_versions_for_vehicle.return_value = [ VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.5.0", @@ -268,10 +264,10 @@ def test_get_versions_type_filter_keeps_matching( assert versions[0].type == "stable" def test_get_versions_type_filter_excludes_non_matching( - self, service, mock_versions_fetcher + self, service, mock_versions_manager ): """Test that type_filter excludes versions that do not match.""" - mock_versions_fetcher.get_versions_for_vehicle.return_value = [ + mock_versions_manager.get_versions_for_vehicle.return_value = [ VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.5.0", @@ -292,10 +288,10 @@ def test_get_versions_type_filter_excludes_non_matching( assert versions == [] def test_get_versions_type_filter_none_returns_all( - self, service, mock_versions_fetcher + self, service, mock_versions_manager ): """Test that passing no type_filter returns all versions.""" - mock_versions_fetcher.get_versions_for_vehicle.return_value = [ + mock_versions_manager.get_versions_for_vehicle.return_value = [ VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.5.0", @@ -323,10 +319,10 @@ def test_get_versions_type_filter_none_returns_all( assert len(versions) == 3 def test_get_versions_type_filter_multiple_matches( - self, service, mock_versions_fetcher + self, service, mock_versions_manager ): """Test that type_filter returns all versions matching the type when there are multiple.""" - mock_versions_fetcher.get_versions_for_vehicle.return_value = [ + mock_versions_manager.get_versions_for_vehicle.return_value = [ VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.4.0", @@ -355,10 +351,10 @@ def test_get_versions_type_filter_multiple_matches( assert all(v.type == "stable" for v in versions) def test_get_versions_latest_name_format( - self, service, mock_versions_fetcher + self, service, mock_versions_manager ): """Test that latest versions have the correct display name format.""" - mock_versions_fetcher.get_versions_for_vehicle.return_value = [ + mock_versions_manager.get_versions_for_vehicle.return_value = [ VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/heads/master", @@ -372,10 +368,10 @@ def test_get_versions_latest_name_format( assert versions[0].name == "Latest (ardupilot)" def test_get_versions_non_latest_name_format( - self, service, mock_versions_fetcher + self, service, mock_versions_manager ): """Test that non-latest versions have the correct display name format.""" - mock_versions_fetcher.get_versions_for_vehicle.return_value = [ + mock_versions_manager.get_versions_for_vehicle.return_value = [ VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.5.0", @@ -390,7 +386,7 @@ def test_get_versions_non_latest_name_format( # Tests for get_version - def test_get_version_found(self, service, mock_versions_fetcher): + def test_get_version_found(self, service, mock_versions_manager): """Test that the correct version is returned when it exists.""" version_info = VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), @@ -399,16 +395,16 @@ def test_get_version_found(self, service, mock_versions_fetcher): version_number="4.5.0", ap_build_artifacts_url=None, ) - mock_versions_fetcher.get_versions_for_vehicle.return_value = [version_info] + mock_versions_manager.get_versions_for_vehicle.return_value = [version_info] result = service.get_version("copter", version_info.version_id) assert result is not None assert result.id == version_info.version_id - def test_get_version_not_found(self, service, mock_versions_fetcher): + def test_get_version_not_found(self, service, mock_versions_manager): """Test that None is returned when the version does not exist.""" - mock_versions_fetcher.get_versions_for_vehicle.return_value = [ + mock_versions_manager.get_versions_for_vehicle.return_value = [ VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), commit_ref="refs/tags/Copter-4.5.0", @@ -422,15 +418,15 @@ def test_get_version_not_found(self, service, mock_versions_fetcher): assert result is None - def test_get_version_no_versions_available(self, service, mock_versions_fetcher): + def test_get_version_no_versions_available(self, service, mock_versions_manager): """Test that None is returned when there are no versions at all.""" - mock_versions_fetcher.get_versions_for_vehicle.return_value = [] + mock_versions_manager.get_versions_for_vehicle.return_value = [] result = service.get_version("copter", "any-version-id") assert result is None - def test_get_version_returns_correct_match_among_many(self, service, mock_versions_fetcher): + def test_get_version_returns_correct_match_among_many(self, service, mock_versions_manager): """Test that only the matching version is returned when multiple exist.""" stable_info = VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), @@ -453,7 +449,7 @@ def test_get_version_returns_correct_match_among_many(self, service, mock_versio version_number="NA", ap_build_artifacts_url=None, ) - mock_versions_fetcher.get_versions_for_vehicle.return_value = [ + mock_versions_manager.get_versions_for_vehicle.return_value = [ stable_info, beta_info, latest_info, ] @@ -465,29 +461,29 @@ def test_get_version_returns_correct_match_among_many(self, service, mock_versio # Tests for get_boards - def test_get_boards_version_not_found_returns_empty(self, service, mock_versions_fetcher): + def test_get_boards_version_not_found_returns_empty(self, service, mock_versions_manager): """Test that an empty list is returned when the version does not exist.""" - mock_versions_fetcher.get_version_info.return_value = None + mock_versions_manager.get_version_info.return_value = None result = service.get_boards("copter", "nonexistent-version-id") assert result == [] def test_get_boards_version_info_queried_with_correct_params( - self, service, mock_versions_fetcher + self, service, mock_versions_manager ): """Test that get_version_info is called with the correct vehicle and version IDs.""" - mock_versions_fetcher.get_version_info.return_value = None + mock_versions_manager.get_version_info.return_value = None service.get_boards("copter", "some-version-id") - mock_versions_fetcher.get_version_info.assert_called_once_with( + mock_versions_manager.get_version_info.assert_called_once_with( vehicle_id="copter", version_id="some-version-id", ) def test_get_boards_empty( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that an empty list is returned when there are no boards for a version.""" version_info = VersionInfo( @@ -497,7 +493,7 @@ def test_get_boards_empty( version_number="4.5.0", ap_build_artifacts_url=None, ) - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = [] result = service.get_boards("copter", version_info.version_id) @@ -505,7 +501,7 @@ def test_get_boards_empty( assert result == [] def test_get_boards_single( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that a single board is returned correctly.""" version_info = VersionInfo( @@ -515,7 +511,7 @@ def test_get_boards_single( version_number="4.5.0", ap_build_artifacts_url=None, ) - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeRed"] result = service.get_boards("copter", version_info.version_id) @@ -525,7 +521,7 @@ def test_get_boards_single( assert result[0].name == "CubeRed" def test_get_boards_many( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that multiple boards are returned correctly.""" version_info = VersionInfo( @@ -535,7 +531,7 @@ def test_get_boards_many( version_number="4.5.0", ap_build_artifacts_url=None, ) - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = [ "CubeRed", "CubeOrange", "MatekF405", ] @@ -546,7 +542,7 @@ def test_get_boards_many( assert [b.id for b in result] == ["CubeRed", "CubeOrange", "MatekF405"] def test_get_boards_sets_correct_vehicle_and_version_ids( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that returned boards carry the correct vehicle_id and version_id.""" version_info = VersionInfo( @@ -556,7 +552,7 @@ def test_get_boards_sets_correct_vehicle_and_version_ids( version_number="4.5.0", ap_build_artifacts_url=None, ) - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeRed"] result = service.get_boards("copter", version_info.version_id) @@ -565,7 +561,7 @@ def test_get_boards_sets_correct_vehicle_and_version_ids( assert result[0].version_id == version_info.version_id def test_get_boards_fetcher_called_with_correct_params( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that the metadata fetcher is called with remote name, commit ref, and vehicle ID from version info.""" version_info = VersionInfo( @@ -575,7 +571,7 @@ def test_get_boards_fetcher_called_with_correct_params( version_number="4.5.0", ap_build_artifacts_url=None, ) - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = [] service.get_boards("copter", version_info.version_id) @@ -588,7 +584,7 @@ def test_get_boards_fetcher_called_with_correct_params( # Tests for get_board - def test_get_board_found(self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher): + def test_get_board_found(self, service, mock_versions_manager, mock_ap_src_metadata_fetcher): """Test that the correct board is returned when it exists.""" version_info = VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), @@ -597,7 +593,7 @@ def test_get_board_found(self, service, mock_versions_fetcher, mock_ap_src_metad version_number="4.5.0", ap_build_artifacts_url=None, ) - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeRed", "CubeOrange"] result = service.get_board("copter", version_info.version_id, "CubeRed") @@ -608,7 +604,7 @@ def test_get_board_found(self, service, mock_versions_fetcher, mock_ap_src_metad assert result.vehicle_id == "copter" assert result.version_id == version_info.version_id - def test_get_board_not_found(self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher): + def test_get_board_not_found(self, service, mock_versions_manager, mock_ap_src_metadata_fetcher): """Test that None is returned when the board does not exist.""" version_info = VersionInfo( remote_info=RemoteInfo(name="ardupilot", url="https://github.com/ArduPilot/ardupilot.git"), @@ -617,7 +613,7 @@ def test_get_board_not_found(self, service, mock_versions_fetcher, mock_ap_src_m version_number="4.5.0", ap_build_artifacts_url=None, ) - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = ["CubeRed", "CubeOrange"] result = service.get_board("copter", version_info.version_id, "NonExistentBoard") @@ -625,7 +621,7 @@ def test_get_board_not_found(self, service, mock_versions_fetcher, mock_ap_src_m assert result is None def test_get_board_returns_correct_match_among_many( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that only the matching board is returned when multiple boards exist.""" version_info = VersionInfo( @@ -635,7 +631,7 @@ def test_get_board_returns_correct_match_among_many( version_number="4.5.0", ap_build_artifacts_url=None, ) - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_boards.return_value = [ "CubeRed", "CubeOrange", "MatekF405", ] @@ -648,17 +644,17 @@ def test_get_board_returns_correct_match_among_many( # Tests for get_features def test_get_features_version_not_found_returns_empty( - self, service, mock_versions_fetcher + self, service, mock_versions_manager ): """Test that an empty list is returned when the version does not exist.""" - mock_versions_fetcher.get_version_info.return_value = None + mock_versions_manager.get_version_info.return_value = None result = service.get_features("copter", "nonexistent-version-id", "CubeRed") assert result == [] def test_get_features_zero_options_returns_empty( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that an empty list is returned when there are no build options.""" version_info = VersionInfo( @@ -668,7 +664,7 @@ def test_get_features_zero_options_returns_empty( version_number="4.5.0", ap_build_artifacts_url=None, ) - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [] result = service.get_features("copter", version_info.version_id, "CubeRed") @@ -676,7 +672,7 @@ def test_get_features_zero_options_returns_empty( assert result == [] def test_get_features_one_option( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that a single feature is returned correctly.""" version_info = VersionInfo( @@ -693,7 +689,7 @@ def test_get_features_one_option( opt.description = "" opt.default = 1 opt.dependency = None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -704,7 +700,7 @@ def test_get_features_one_option( assert result[0].name == "HAL_LOGGING_ENABLED" def test_get_features_many_options( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that all features are returned when multiple options exist.""" version_info = VersionInfo( @@ -723,7 +719,7 @@ def test_get_features_many_options( opt_sensors = Mock() opt_sensors.label, opt_sensors.define, opt_sensors.category = "HAL_BEACON_ENABLED", "HAL_BEACON_ENABLED", "Sensors" opt_sensors.description, opt_sensors.default, opt_sensors.dependency = "", 1, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_logging, opt_ekf, opt_sensors] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -732,7 +728,7 @@ def test_get_features_many_options( assert len(result) == 3 def test_get_features_sorted_by_category( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that features are sorted by category name.""" version_info = VersionInfo( @@ -751,7 +747,7 @@ def test_get_features_sorted_by_category( opt_m = Mock() opt_m.label, opt_m.define, opt_m.category = "FEATURE_M", "DEFINE_M", "Logging" opt_m.description, opt_m.default, opt_m.dependency = "", 1, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_z, opt_a, opt_m] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -760,7 +756,7 @@ def test_get_features_sorted_by_category( assert [f.category.name for f in result] == ["EKF", "Logging", "Sensors"] def test_get_features_uses_fallback_defaults_when_no_artifacts_url( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that build-options-py defaults are used when ap_build_artifacts_url is None.""" version_info = VersionInfo( @@ -776,7 +772,7 @@ def test_get_features_uses_fallback_defaults_when_no_artifacts_url( opt_off = Mock() opt_off.label, opt_off.define, opt_off.category = "FEATURE_OFF", "DEFINE_OFF", "Cat" opt_off.description, opt_off.default, opt_off.dependency = "", 0, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_on, opt_off] result = service.get_features("copter", version_info.version_id, "CubeRed") @@ -789,7 +785,7 @@ def test_get_features_uses_fallback_defaults_when_no_artifacts_url( mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.assert_not_called() def test_get_features_uses_firmware_server_defaults_when_available( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that firmware-server defaults override build-options-py when present.""" version_info = VersionInfo( @@ -805,7 +801,7 @@ def test_get_features_uses_firmware_server_defaults_when_available( opt_b = Mock() opt_b.label, opt_b.define, opt_b.category = "FEATURE_B", "DEFINE_B", "Cat" opt_b.description, opt_b.default, opt_b.dependency = "", 1, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_a, opt_b] # firmware server says DEFINE_A is disabled, DEFINE_B is enabled mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = { @@ -822,7 +818,7 @@ def test_get_features_uses_firmware_server_defaults_when_available( assert by_id["FEATURE_B"].source == "firmware-server" def test_get_features_falls_back_to_defaults_when_firmware_server_returns_none( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that build-options-py fallback is used when firmware server fetch fails.""" version_info = VersionInfo( @@ -835,7 +831,7 @@ def test_get_features_falls_back_to_defaults_when_firmware_server_returns_none( opt = Mock() opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" opt.description, opt.default, opt.dependency = "", 1, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -845,7 +841,7 @@ def test_get_features_falls_back_to_defaults_when_firmware_server_returns_none( assert result[0].default.source == "build-options-py" def test_get_features_firmware_server_overrides_only_known_defines( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that a define absent from firmware-server data falls back to build-options-py.""" version_info = VersionInfo( @@ -861,7 +857,7 @@ def test_get_features_firmware_server_overrides_only_known_defines( opt_unknown = Mock() opt_unknown.label, opt_unknown.define, opt_unknown.category = "FEATURE_UNKNOWN", "DEFINE_UNKNOWN", "Cat" opt_unknown.description, opt_unknown.default, opt_unknown.dependency = "", 1, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_known, opt_unknown] # firmware server only knows about DEFINE_KNOWN mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = { @@ -877,7 +873,7 @@ def test_get_features_firmware_server_overrides_only_known_defines( assert by_id["FEATURE_UNKNOWN"].source == "build-options-py" def test_get_features_dependency_none( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that a feature with no dependency produces an empty dependencies list.""" version_info = VersionInfo( @@ -890,7 +886,7 @@ def test_get_features_dependency_none( opt = Mock() opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" opt.description, opt.default, opt.dependency = "", 1, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -899,7 +895,7 @@ def test_get_features_dependency_none( assert result[0].dependencies == [] def test_get_features_dependency_single( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that a single dependency string is parsed into a one-element list.""" version_info = VersionInfo( @@ -912,7 +908,7 @@ def test_get_features_dependency_single( opt = Mock() opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" opt.description, opt.default, opt.dependency = "", 1, "DEP_ONE" - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -921,7 +917,7 @@ def test_get_features_dependency_single( assert result[0].dependencies == ["DEP_ONE"] def test_get_features_dependency_multiple_comma_separated( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that a comma-separated dependency string is split into multiple entries.""" version_info = VersionInfo( @@ -934,7 +930,7 @@ def test_get_features_dependency_multiple_comma_separated( opt = Mock() opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" opt.description, opt.default, opt.dependency = "", 1, "DEP_ONE,DEP_TWO,DEP_THREE" - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -943,7 +939,7 @@ def test_get_features_dependency_multiple_comma_separated( assert result[0].dependencies == ["DEP_ONE", "DEP_TWO", "DEP_THREE"] def test_get_features_dependency_with_spaces( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that spaces around dependency labels are stripped.""" version_info = VersionInfo( @@ -956,7 +952,7 @@ def test_get_features_dependency_with_spaces( opt = Mock() opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" opt.description, opt.default, opt.dependency = "", 1, "DEP_ONE , DEP_TWO , DEP_THREE" - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -965,7 +961,7 @@ def test_get_features_dependency_with_spaces( assert result[0].dependencies == ["DEP_ONE", "DEP_TWO", "DEP_THREE"] def test_get_features_ids_filled_correctly( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that vehicle_id, version_id, and board_id are correctly set on each feature.""" version_info = VersionInfo( @@ -978,7 +974,7 @@ def test_get_features_ids_filled_correctly( opt = Mock() opt.label, opt.define, opt.category = "FEATURE_A", "DEFINE_A", "Cat" opt.description, opt.default, opt.dependency = "", 1, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -989,7 +985,7 @@ def test_get_features_ids_filled_correctly( assert result[0].board_id == "CubeRed" def test_get_features_category_filter_keeps_matching( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that category_id filter returns only features whose category matches.""" version_info = VersionInfo( @@ -1008,7 +1004,7 @@ def test_get_features_category_filter_keeps_matching( opt_sensors = Mock() opt_sensors.label, opt_sensors.define, opt_sensors.category = "HAL_BEACON_ENABLED", "HAL_BEACON_ENABLED", "Sensors" opt_sensors.description, opt_sensors.default, opt_sensors.dependency = "", 1, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_logging, opt_ekf, opt_sensors] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -1019,7 +1015,7 @@ def test_get_features_category_filter_keeps_matching( assert result[0].category.name == "Logging" def test_get_features_category_filter_excludes_non_matching( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that category_id filter excludes features whose category does not match.""" version_info = VersionInfo( @@ -1035,7 +1031,7 @@ def test_get_features_category_filter_excludes_non_matching( opt_ekf = Mock() opt_ekf.label, opt_ekf.define, opt_ekf.category = "HAL_NAVEKF3_AVAILABLE", "HAL_NAVEKF3_AVAILABLE", "EKF" opt_ekf.description, opt_ekf.default, opt_ekf.dependency = "", 1, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_logging, opt_ekf] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -1044,7 +1040,7 @@ def test_get_features_category_filter_excludes_non_matching( assert result == [] def test_get_features_category_filter_no_matches_returns_empty( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that a category_id with no matching features returns an empty list.""" version_info = VersionInfo( @@ -1060,7 +1056,7 @@ def test_get_features_category_filter_no_matches_returns_empty( opt_b = Mock() opt_b.label, opt_b.define, opt_b.category = "FEATURE_B", "DEFINE_B", "Logging" opt_b.description, opt_b.default, opt_b.dependency = "", 1, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_a, opt_b] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -1071,7 +1067,7 @@ def test_get_features_category_filter_no_matches_returns_empty( # Tests for get_feature def test_get_feature_found( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that the correct feature is returned when it exists.""" version_info = VersionInfo( @@ -1084,7 +1080,7 @@ def test_get_feature_found( opt = Mock() opt.label, opt.define, opt.category = "HAL_LOGGING_ENABLED", "HAL_LOGGING_ENABLED", "Logging" opt.description, opt.default, opt.dependency = "", 1, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -1095,7 +1091,7 @@ def test_get_feature_found( assert result.name == "HAL_LOGGING_ENABLED" def test_get_feature_not_found( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that None is returned when the feature does not exist.""" version_info = VersionInfo( @@ -1108,7 +1104,7 @@ def test_get_feature_not_found( opt = Mock() opt.label, opt.define, opt.category = "HAL_LOGGING_ENABLED", "HAL_LOGGING_ENABLED", "Logging" opt.description, opt.default, opt.dependency = "", 1, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None @@ -1117,7 +1113,7 @@ def test_get_feature_not_found( assert result is None def test_get_feature_returns_correct_match_among_many( - self, service, mock_versions_fetcher, mock_ap_src_metadata_fetcher + self, service, mock_versions_manager, mock_ap_src_metadata_fetcher ): """Test that only the matching feature is returned when multiple features exist.""" version_info = VersionInfo( @@ -1136,7 +1132,7 @@ def test_get_feature_returns_correct_match_among_many( opt_c = Mock() opt_c.label, opt_c.define, opt_c.category = "FEATURE_C", "DEFINE_C", "Cat" opt_c.description, opt_c.default, opt_c.dependency = "", 1, None - mock_versions_fetcher.get_version_info.return_value = version_info + mock_versions_manager.get_version_info.return_value = version_info mock_ap_src_metadata_fetcher.get_build_options_at_commit.return_value = [opt_a, opt_b, opt_c] mock_ap_src_metadata_fetcher.get_board_defaults_from_fw_server.return_value = None From b39cad9ccf453bf01432282f6af4d8f83234c1bd Mon Sep 17 00:00:00 2001 From: Shiv Tyagi Date: Sat, 11 Jul 2026 09:42:25 +0530 Subject: [PATCH 6/8] builder: add requests and packaging as requirements --- builder/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builder/requirements.txt b/builder/requirements.txt index 88a94029..3f2811b3 100644 --- a/builder/requirements.txt +++ b/builder/requirements.txt @@ -1,3 +1,5 @@ jsonschema==4.20.0 redis==5.2.1 dill==0.3.8 +requests==2.31.0 +packaging==26.2 From 684850591cfd781c270c0f15682c3138585a3af8 Mon Sep 17 00:00:00 2001 From: Shiv Tyagi Date: Sat, 11 Jul 2026 09:43:19 +0530 Subject: [PATCH 7/8] examples: use vehicle id in remotes.json instead of vehicle name --- examples/remotes.json.sample | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/remotes.json.sample b/examples/remotes.json.sample index 4af4a388..992ca008 100644 --- a/examples/remotes.json.sample +++ b/examples/remotes.json.sample @@ -4,7 +4,7 @@ "url": "https://github.com/ardupilot/ardupilot.git", "vehicles": [ { - "name": "Copter", + "id": "copter", "releases": [ { "release_type": "latest", @@ -21,7 +21,7 @@ "url": "https://github.com/cool-user-007/ardupilot.git", "vehicles": [ { - "name": "Copter", + "id": "copter", "releases": [ { "release_type": "Custom", @@ -37,7 +37,7 @@ "url": "https://github.com/ardupilot/ardupilot.git", "vehicles": [ { - "name": "Copter", + "id": "copter", "releases": [ { "release_type": "latest", @@ -55,4 +55,4 @@ } ] } -] \ No newline at end of file +] From 23eb45738fcf956a12235f075e02c9e1b8c7109c Mon Sep 17 00:00:00 2001 From: Shiv Tyagi Date: Sat, 11 Jul 2026 09:43:48 +0530 Subject: [PATCH 8/8] README: update docs to reflect remotes.json is now optional --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e91ff27f..b4302adf 100644 --- a/README.md +++ b/README.md @@ -137,9 +137,9 @@ The default directory structure is established as follows: ├── ardupilot (used by the web component) ├── artifacts ├── configs - | └── remotes.json (auto-generated, see examples/remotes.json.sample) + | └── remotes.json (optional, see examples/remotes.json.sample) ├── secrets - | └── reload_token (optional) + | └── admin_token (optional) ├── tmp └── ardupilot (used by the builder component) ```