From 61b23d57355cf586eb7ee0c346eea10157626c24 Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Tue, 10 Feb 2026 14:01:54 -0700 Subject: [PATCH 1/3] Add ignore_missing param to collection lookup --- python/lsst/daf/butler/_butler_collections.py | 1 + .../_direct_butler_collections.py | 2 ++ .../daf/butler/registry/collections/_base.py | 24 +++++++++++++------ .../registry/interfaces/_collections.py | 1 + 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/python/lsst/daf/butler/_butler_collections.py b/python/lsst/daf/butler/_butler_collections.py index a8005fe7d0..359cb207a2 100644 --- a/python/lsst/daf/butler/_butler_collections.py +++ b/python/lsst/daf/butler/_butler_collections.py @@ -301,6 +301,7 @@ def query_info( include_summary: bool = False, include_doc: bool = False, summary_datasets: Iterable[DatasetType] | Iterable[str] | None = None, + ignore_missing: bool = False, ) -> Sequence[CollectionInfo]: """Query the butler for collections matching an expression and return detailed information about those collections. diff --git a/python/lsst/daf/butler/direct_butler/_direct_butler_collections.py b/python/lsst/daf/butler/direct_butler/_direct_butler_collections.py index 2f5d9bb5de..9df46e9cf8 100644 --- a/python/lsst/daf/butler/direct_butler/_direct_butler_collections.py +++ b/python/lsst/daf/butler/direct_butler/_direct_butler_collections.py @@ -116,6 +116,7 @@ def query_info( include_summary: bool = False, include_doc: bool = False, summary_datasets: Iterable[DatasetType] | Iterable[str] | None = None, + ignore_missing: bool = False, ) -> Sequence[CollectionInfo]: info = [] if collection_types is None: @@ -128,6 +129,7 @@ def query_info( collection_types=collection_types, flatten_chains=flatten_chains, include_chains=include_chains, + ignore_missing=ignore_missing, ) summaries: Mapping[Any, CollectionSummary] = {} diff --git a/python/lsst/daf/butler/registry/collections/_base.py b/python/lsst/daf/butler/registry/collections/_base.py index 74daef1406..fd45d80cdd 100644 --- a/python/lsst/daf/butler/registry/collections/_base.py +++ b/python/lsst/daf/butler/registry/collections/_base.py @@ -287,7 +287,11 @@ def find(self, name: str) -> CollectionRecord[K]: return result def _find_many( - self, names: Iterable[str], flatten_chains: bool, collection_cache: CollectionRecordCache | None + self, + names: Iterable[str], + flatten_chains: bool, + collection_cache: CollectionRecordCache | None, + ignore_missing: bool = False, ) -> list[CollectionRecord[K]]: """Return multiple records given their names. @@ -373,11 +377,14 @@ def check_cache( records[record.name] = record self._addCachedRecord(record, collection_cache) - missing_names = [name for name in names if name not in records] - if len(missing_names) == 1: - raise MissingCollectionError(f"No collection with name '{missing_names[0]}' found.") - elif len(missing_names) > 1: - raise MissingCollectionError(f"No collections with names '{' '.join(missing_names)}' found.") + if ignore_missing: + names = [name for name in names if name in records] + else: + missing_names = [name for name in names if name not in records] + if len(missing_names) == 1: + raise MissingCollectionError(f"No collection with name '{missing_names[0]}' found.") + elif len(missing_names) > 1: + raise MissingCollectionError(f"No collections with names '{' '.join(missing_names)}' found.") def order(names: Iterable[str]) -> Iterator[CollectionRecord[K]]: for name in names: @@ -409,6 +416,7 @@ def resolve_wildcard( collection_types: Set[CollectionType] = CollectionType.all(), flatten_chains: bool = True, include_chains: bool | None = None, + ignore_missing: bool = False, ) -> list[CollectionRecord[K]]: # Docstring inherited include_chains = include_chains if include_chains is not None else not flatten_chains @@ -451,7 +459,9 @@ def filter_types(records: Iterable[CollectionRecord[K]]) -> Iterator[CollectionR if explicit_names: # _find_many() returns correctly ordered records, but there may be # duplicates. - for record in filter_types(self._find_many(explicit_names, flatten_chains, cache)): + for record in filter_types( + self._find_many(explicit_names, flatten_chains, cache, ignore_missing=ignore_missing) + ): if record.key not in done_keys: result.append(record) done_keys.add(record.key) diff --git a/python/lsst/daf/butler/registry/interfaces/_collections.py b/python/lsst/daf/butler/registry/interfaces/_collections.py index 8b1f531996..9daf696f65 100644 --- a/python/lsst/daf/butler/registry/interfaces/_collections.py +++ b/python/lsst/daf/butler/registry/interfaces/_collections.py @@ -533,6 +533,7 @@ def resolve_wildcard( collection_types: Set[CollectionType] = CollectionType.all(), flatten_chains: bool = True, include_chains: bool | None = None, + ignore_missing: bool = False, ) -> list[CollectionRecord[_Key]]: """Iterate over collection records that match a wildcard. From c52a0e563a8459776835cef1fef252b74ab81cad Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Tue, 10 Feb 2026 14:20:59 -0700 Subject: [PATCH 2/3] Add function to synchronize collection chain structure --- .../synchronize_collection_structure.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 python/lsst/daf/butler/_rubin/synchronize_collection_structure.py diff --git a/python/lsst/daf/butler/_rubin/synchronize_collection_structure.py b/python/lsst/daf/butler/_rubin/synchronize_collection_structure.py new file mode 100644 index 0000000000..aec980e8b4 --- /dev/null +++ b/python/lsst/daf/butler/_rubin/synchronize_collection_structure.py @@ -0,0 +1,34 @@ +from collections.abc import Iterable + +from .._butler import Butler +from .._butler_collections import CollectionInfo +from .._collection_type import CollectionType + + +def synchronize_collection_structure( + source_butler: Butler, target_butler: Butler, collections: Iterable[str] +) -> None: + source_info = source_butler.collections.query_info( + collections, include_doc=True, include_chains=True, flatten_chains=True + ) + target_info = target_butler.collections.query_info([c.name for c in source_info], ignore_missing=True) + target_map = {c.name: c for c in target_info} + + missing_collections: list[CollectionInfo] = [] + mismatched_children: dict[str, tuple[str, ...]] = {} + for source_collection in source_info: + target_collection = target_map.get(source_collection.name) + if not target_collection: + missing_collections.append(source_collection) + if source_collection.type == CollectionType.CHAINED: + mismatched_children[source_collection.name] = source_collection.children + elif source_collection.children != target_collection.children: + mismatched_children[source_collection.name] = source_collection.children + + for collection in missing_collections: + print(f"register {collection.name}") + target_butler.collections.register(collection.name, type=collection.type, doc=collection.doc) + + for parent, children in mismatched_children.items(): + print(f"define collection chain {parent}: {children}") + target_butler.collections.redefine_chain(parent, children) From 3af083627e19270459697a93e972ecd95fc32f8f Mon Sep 17 00:00:00 2001 From: "David H. Irving" Date: Tue, 10 Feb 2026 14:32:25 -0700 Subject: [PATCH 3/3] validate collection types --- .../daf/butler/_rubin/synchronize_collection_structure.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/lsst/daf/butler/_rubin/synchronize_collection_structure.py b/python/lsst/daf/butler/_rubin/synchronize_collection_structure.py index aec980e8b4..2b73403289 100644 --- a/python/lsst/daf/butler/_rubin/synchronize_collection_structure.py +++ b/python/lsst/daf/butler/_rubin/synchronize_collection_structure.py @@ -3,6 +3,7 @@ from .._butler import Butler from .._butler_collections import CollectionInfo from .._collection_type import CollectionType +from .._exceptions import CollectionTypeError def synchronize_collection_structure( @@ -22,6 +23,11 @@ def synchronize_collection_structure( missing_collections.append(source_collection) if source_collection.type == CollectionType.CHAINED: mismatched_children[source_collection.name] = source_collection.children + elif source_collection.type != target_collection.type: + raise CollectionTypeError( + f"Collection '{source_collection.name}' has different types in source and target repos." + f" Source: {source_collection.type.name} Target: {target_collection.type.name}" + ) elif source_collection.children != target_collection.children: mismatched_children[source_collection.name] = source_collection.children