diff --git a/python/lsst/ctrl/bps/parsl/job.py b/python/lsst/ctrl/bps/parsl/job.py index 1798a0a..6541e08 100644 --- a/python/lsst/ctrl/bps/parsl/job.py +++ b/python/lsst/ctrl/bps/parsl/job.py @@ -1,13 +1,22 @@ import os import re import subprocess +import sys from functools import partial from textwrap import dedent from typing import Any, Dict, List, Optional, Sequence from lsst.ctrl.bps import BpsConfig, GenericWorkflow, GenericWorkflowJob -from parsl.app.bash import BashApp -from parsl.app.futures import Future + +try: + from parsl.app.bash import BashApp + from parsl.app.futures import Future +except ImportError: + if "pydoc" in sys.modules or "sphinx" in sys.modules: + BashApp = None + Future = None + else: + raise from .configuration import get_bps_config_value diff --git a/python/lsst/ctrl/bps/parsl/site.py b/python/lsst/ctrl/bps/parsl/site.py index c37de01..db1f07e 100644 --- a/python/lsst/ctrl/bps/parsl/site.py +++ b/python/lsst/ctrl/bps/parsl/site.py @@ -1,12 +1,22 @@ +import sys from abc import ABC, abstractmethod from types import ModuleType from typing import TYPE_CHECKING, List, Optional from lsst.ctrl.bps import BpsConfig from lsst.utils import doImport -from parsl.addresses import address_by_hostname -from parsl.executors.base import ParslExecutor -from parsl.monitoring import MonitoringHub + +try: + from parsl.addresses import address_by_hostname + from parsl.executors.base import ParslExecutor + from parsl.monitoring import MonitoringHub +except ImportError: + if "pydoc" in sys.modules or "sphinx" in sys.modules: + address_by_hostname = None + ParslExecutor = None + MonitoringHub = None + else: + raise from .configuration import get_bps_config_value, get_workflow_name from .environment import export_environment diff --git a/python/lsst/ctrl/bps/parsl/sites/local.py b/python/lsst/ctrl/bps/parsl/sites/local.py index 7558d6c..1c2ec44 100644 --- a/python/lsst/ctrl/bps/parsl/sites/local.py +++ b/python/lsst/ctrl/bps/parsl/sites/local.py @@ -1,8 +1,17 @@ +import sys from typing import TYPE_CHECKING, List -from parsl.executors import HighThroughputExecutor -from parsl.executors.base import ParslExecutor -from parsl.providers import LocalProvider +try: + from parsl.executors import HighThroughputExecutor + from parsl.executors.base import ParslExecutor + from parsl.providers import LocalProvider +except ImportError: + if "pydoc" in sys.modules or "sphinx" in sys.modules: + HighThroughputExecutor = None + ParslExecutor = None + LocalProvider = None + else: + raise from ..configuration import get_bps_config_value from ..site import SiteConfig diff --git a/python/lsst/ctrl/bps/parsl/sites/nersc.py b/python/lsst/ctrl/bps/parsl/sites/nersc.py index 64a8d39..a3fbb1f 100644 --- a/python/lsst/ctrl/bps/parsl/sites/nersc.py +++ b/python/lsst/ctrl/bps/parsl/sites/nersc.py @@ -1,7 +1,15 @@ +import sys from typing import Any, Dict, List, Optional -from parsl.executors.base import ParslExecutor -from parsl.launchers import SrunLauncher +try: + from parsl.executors.base import ParslExecutor + from parsl.launchers import SrunLauncher +except ImportError: + if "pydoc" in sys.modules or "sphinx" in sys.modules: + ParslExecutor = None + SrunLauncher = None + else: + raise from .slurm import TripleSlurm diff --git a/python/lsst/ctrl/bps/parsl/sites/princeton.py b/python/lsst/ctrl/bps/parsl/sites/princeton.py index a06d9b3..66e23c5 100644 --- a/python/lsst/ctrl/bps/parsl/sites/princeton.py +++ b/python/lsst/ctrl/bps/parsl/sites/princeton.py @@ -1,8 +1,17 @@ +import sys from typing import TYPE_CHECKING, List -from parsl.addresses import address_by_interface -from parsl.executors.base import ParslExecutor -from parsl.launchers import SrunLauncher +try: + from parsl.addresses import address_by_interface + from parsl.executors.base import ParslExecutor + from parsl.launchers import SrunLauncher +except ImportError: + if "pydoc" in sys.modules or "sphinx" in sys.modules: + address_by_interface = None + ParslExecutor = None + SrunLauncher = None + else: + raise from ..configuration import get_bps_config_value from ..environment import export_environment diff --git a/python/lsst/ctrl/bps/parsl/sites/slac.py b/python/lsst/ctrl/bps/parsl/sites/slac.py index bc448c9..b0041fa 100644 --- a/python/lsst/ctrl/bps/parsl/sites/slac.py +++ b/python/lsst/ctrl/bps/parsl/sites/slac.py @@ -1,7 +1,15 @@ +import sys from typing import TYPE_CHECKING, List -from parsl.executors.base import ParslExecutor -from parsl.launchers import SrunLauncher +try: + from parsl.executors.base import ParslExecutor + from parsl.launchers import SrunLauncher +except ImportError: + if "pydoc" in sys.modules or "sphinx" in sys.modules: + ParslExecutor = None + SrunLauncher = None + else: + raise from ..configuration import get_bps_config_value from ..environment import export_environment diff --git a/python/lsst/ctrl/bps/parsl/sites/slurm.py b/python/lsst/ctrl/bps/parsl/sites/slurm.py index c62e2ea..ef6b0b9 100644 --- a/python/lsst/ctrl/bps/parsl/sites/slurm.py +++ b/python/lsst/ctrl/bps/parsl/sites/slurm.py @@ -1,8 +1,17 @@ +import sys from typing import TYPE_CHECKING, Any, Dict, List, Optional -from parsl.executors import HighThroughputExecutor -from parsl.executors.base import ParslExecutor -from parsl.providers import SlurmProvider +try: + from parsl.executors import HighThroughputExecutor + from parsl.executors.base import ParslExecutor + from parsl.providers import SlurmProvider +except ImportError: + if "pydoc" in sys.modules or "sphinx" in sys.modules: + HighThroughputExecutor = None + ParslExecutor = None + SlurmProvider = None + else: + raise from ..configuration import get_bps_config_value, get_workflow_name from ..site import SiteConfig diff --git a/python/lsst/ctrl/bps/parsl/sites/work_queue.py b/python/lsst/ctrl/bps/parsl/sites/work_queue.py index a710241..40acc54 100644 --- a/python/lsst/ctrl/bps/parsl/sites/work_queue.py +++ b/python/lsst/ctrl/bps/parsl/sites/work_queue.py @@ -1,10 +1,21 @@ +import sys from typing import TYPE_CHECKING, Any, Dict, List -from parsl.executors import WorkQueueExecutor -from parsl.executors.base import ParslExecutor -from parsl.launchers import SrunLauncher -from parsl.providers import LocalProvider -from parsl.providers.provider_base import ExecutionProvider +try: + from parsl.executors import WorkQueueExecutor + from parsl.executors.base import ParslExecutor + from parsl.launchers import SrunLauncher + from parsl.providers import LocalProvider + from parsl.providers.provider_base import ExecutionProvider +except ImportError: + if "pydoc" in sys.modules or "sphinx" in sys.modules: + WorkQueueExecutor = None + ParslExecutor = None + SrunLauncher = None + LocalProvider = None + ExecutionProvider = None + else: + raise from ..configuration import get_bps_config_value from ..site import SiteConfig diff --git a/python/lsst/ctrl/bps/parsl/workflow.py b/python/lsst/ctrl/bps/parsl/workflow.py index 24ad1d7..c15a57d 100644 --- a/python/lsst/ctrl/bps/parsl/workflow.py +++ b/python/lsst/ctrl/bps/parsl/workflow.py @@ -1,13 +1,30 @@ import logging import os import pickle +import sys from typing import Dict, Iterable, List, Mapping, Optional -import parsl -import parsl.config +try: + import parsl +except ImportError: + if "pydoc" in sys.modules or "sphinx" in sys.modules: + parsl = None + else: + raise + from lsst.ctrl.bps import BaseWmsWorkflow, BpsConfig, GenericWorkflow, GenericWorkflowJob -from parsl.app.app import bash_app -from parsl.app.futures import Future + +try: + from parsl.app.app import bash_app + from parsl.app.futures import Future + from parsl.config import Config as ParslConfig +except ImportError: + if "pydoc" in sys.modules or "sphinx" in sys.modules: + bash_app = None + Future = None + ParslConfig = None + else: + raise from .configuration import get_bps_config_value, get_workflow_filename, set_parsl_logging from .job import ParslJob, get_file_paths @@ -18,7 +35,7 @@ _log = logging.getLogger("lsst.ctrl.bps.parsl") -def get_parsl_config(config: BpsConfig) -> parsl.config.Config: +def get_parsl_config(config: BpsConfig) -> ParslConfig: """Construct parsl configuration from BPS configuration For details on the site configuration, see `SiteConfig`. For details on the @@ -40,9 +57,7 @@ def get_parsl_config(config: BpsConfig) -> parsl.config.Config: executors = site.get_executors() retries = get_bps_config_value(site.site, "retries", int, 1) monitor = site.get_monitor() - return parsl.config.Config( - executors=executors, monitoring=monitor, retries=retries, checkpoint_mode="task_exit" - ) + return ParslConfig(executors=executors, monitoring=monitor, retries=retries, checkpoint_mode="task_exit") class ParslWorkflow(BaseWmsWorkflow):