From 7e0060a4c88307077371e8e190d53f5b73d98bf6 Mon Sep 17 00:00:00 2001 From: Stephen R Pietrowicz Date: Tue, 30 Jun 2026 09:24:51 -0500 Subject: [PATCH] Add "ignore_duplicate" to did client method call --- python/lsst/rucio/register/rucio_interface.py | 49 ++++--------------- tests/test_interface.py | 49 ++----------------- 2 files changed, 13 insertions(+), 85 deletions(-) diff --git a/python/lsst/rucio/register/rucio_interface.py b/python/lsst/rucio/register/rucio_interface.py index 50cb54f..da42416 100644 --- a/python/lsst/rucio/register/rucio_interface.py +++ b/python/lsst/rucio/register/rucio_interface.py @@ -236,30 +236,6 @@ def _add_replicas(self, bundles: list[ResourceBundle]) -> None: else: raise Exception(f"Tried {max_retries} times and couldn't add_replicas") - def _add_file_to_dataset_with_retries(self, dataset_id, did): - retries = 0 - max_retries = 5 - while True: - try: - self.did_client.add_files_to_dataset( - scope=self.scope, name=dataset_id, files=[did], rse=self.rse - ) - break - except rucio.common.exception.FileAlreadyExists: - if "pfn" in did: - logger.debug("file %s already registered in dataset %s", did["pfn"], dataset_id) - return # we can return, because it's already in the dataset - except rucio.common.exception.RucioException: - retries += 1 - if retries < max_retries: - seconds = random.randint(10, 20) - logger.debug("failed to register one did to %s; sleeping %d seconds", dataset_id, seconds) - time.sleep(seconds) - self.did_client = DIDClient() # XXX not sure we need to do this. - else: - # we tried max_retries times, and failed, so we'll bail out - raise Exception(f"Couldn't add {did['pfn']} to dataset {dataset_id}") - def _add_files_to_dataset(self, dataset_id: str, dids: list[dict]) -> None: """Attach a list of files specified by Rucio DIDs to a Rucio dataset. @@ -276,23 +252,18 @@ def _add_files_to_dataset(self, dataset_id: str, dids: list[dict]) -> None: max_retries = 5 while True: try: - self.did_client.add_files_to_dataset( - scope=self.scope, - name=dataset_id, - files=dids, - rse=self.rse, + self.did_client.add_files_to_datasets( + attachments=[ + { + "scope": self.scope, + "name": dataset_id, + "dids": dids, + "rse": self.rse, + } + ], + ignore_duplicate=True, ) return - except rucio.common.exception.FileAlreadyExists: - # At least one already is in the dataset. - # This shouldn't happen, but if it does, - # we have to retry each individually. - for did in dids: - self._add_file_to_dataset_with_retries( - dataset_id=dataset_id, - did=did, - ) - return except rucio.common.exception.DataIdentifierNotFound as e: raise e except rucio.common.exception.RucioException: diff --git a/tests/test_interface.py b/tests/test_interface.py index 5ee0a7d..03dd8ae 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -28,8 +28,6 @@ from rucio.client.didclient import DIDClient from rucio.client.replicaclient import ReplicaClient from rucio.common.exception import ( - DataIdentifierAlreadyExists, - DataIdentifierNotFound, FileAlreadyExists, RucioException, ) @@ -71,12 +69,14 @@ def setUp(self): self.dc_init = patch.object(DIDClient, "__init__", return_value=None) self.rc_add_replicas = patch.object(ReplicaClient, "add_replicas", return_value=None) self.dc_attach_dids = patch.object(DIDClient, "attach_dids", return_value=None) + self.dc_attach_dids_to_dids = patch.object(DIDClient, "attach_dids_to_dids", return_value=None) self.rand = patch("random.randint", return_value=1) self.mock_rc_init = self.rc_init.start() self.mock_dc_init = self.dc_init.start() self.mock_rc_add_replicas = self.rc_add_replicas.start() self.mock_dc_attach_dids = self.dc_attach_dids.start() + self.mock_dc_attach_dids_to_dids = self.dc_attach_dids_to_dids.start() self.mock_rand = self.rand.start() rucio_rse = "DRR1" @@ -154,53 +154,10 @@ def testException1TestCase(self, MC1): with self.assertRaises(Exception): self.common() - @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) - def testException2TestCase(self, MC1): - with self.assertRaises(Exception): - self.common() - @patch.object(DIDClient, "add_files_to_dataset", side_effect=FileAlreadyExists("failed")) - def testException3TestCase(self, MC1): + def testException2TestCase(self, MC1): self.common() - @patch.object(DIDClient, "add_dataset", return_value=None) - @patch.object(DIDClient, "add_files_to_dataset", side_effect=DataIdentifierNotFound("failed")) - def testException4TestCase(self, MC1, MC2): - with self.assertRaises(Exception): - self.common() - - @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) - def testException5TestCase(self, MC1): - with self.assertRaises(Exception): - self.common() - - @patch.object(DIDClient, "add_dataset", side_effect=DataIdentifierAlreadyExists("failed")) - @patch.object(DIDClient, "add_files_to_dataset", side_effect=DataIdentifierNotFound("failed")) - def testException6TestCase(self, MC1, MC2): - with self.assertRaises(Exception): - self.common() - - @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) - def testException7TestCase(self, MC1): - with self.assertRaises(Exception): - self.common() - - @patch.object(DIDClient, "add_dataset", side_effect=RucioException("failed")) - @patch.object(DIDClient, "add_files_to_dataset", side_effect=DataIdentifierNotFound("failed")) - def testException8TestCase(self, MC1, MC2): - with self.assertRaises(Exception): - self.common() - - @patch.object(DIDClient, "add_files_to_dataset", side_effect=RucioException("failed")) - def testException9Case(self, MC1): - rucio_rse = "DRR1" - scope = "test" - dtn_url = "root://xrd1:1094//rucio" - - ri = RucioInterface(self.butler, rucio_rse, scope, self.rse_root, dtn_url, DataType.DATA_PRODUCT) - with self.assertRaises(Exception): - ri._add_file_to_dataset_with_retries(None, None) - def tearDown(self): patch.stopall() shutil.rmtree(self.butler_repo, ignore_errors=True)