Refactor verification of optional dependencies (#1443)

* Rather than checking keys in the `globals()` dictionary, check if the exception is not-None
* Remove `morpheus/utils/verify_dependencies.py`

Closes #1435

## By Submitting this PR I confirm:
- I am familiar with the [Contributing Guidelines](https://github.com/nv-morpheus/Morpheus/blob/main/docs/source/developer_guide/contributing.md).
- When the PR is ready for review, new or existing tests cover these changes.
- When the PR is ready for review, the documentation is up to date with these changes.

Authors:
  - David Gardner (https://github.com/dagardner-nv)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: #1443

morpheus/controllers/elasticsearch_controller.py

@@ -17,19 +17,17 @@
 
 import pandas as pd
 
-from morpheus.utils.verify_dependencies import _verify_deps
-
 logger = logging.getLogger(__name__)
 
-REQUIRED_DEPS = ('ESConnectionError', 'Elasticsearch', 'parallel_bulk')
+IMPORT_EXCEPTION = None
 IMPORT_ERROR_MESSAGE = "ElasticsearchController requires the elasticsearch package to be installed."
 
 try:
     from elasticsearch import ConnectionError as ESConnectionError
     from elasticsearch import Elasticsearch
     from elasticsearch.helpers import parallel_bulk
-except ImportError:
-    pass
+except ImportError as import_exc:
+    IMPORT_EXCEPTION = import_exc
 
 
 class ElasticsearchController:
@@ -47,7 +45,9 @@ class ElasticsearchController:
     """
 
     def __init__(self, connection_kwargs: dict, raise_on_exception: bool = False, refresh_period_secs: int = 2400):
-        _verify_deps(REQUIRED_DEPS, IMPORT_ERROR_MESSAGE, globals())
+        if IMPORT_EXCEPTION is not None:
+            raise ImportError(IMPORT_ERROR_MESSAGE) from IMPORT_EXCEPTION
+
         self._client = None
         self._last_refresh_time = None
         self._raise_on_exception = raise_on_exception

morpheus/controllers/rss_controller.py

@@ -23,18 +23,16 @@
 import requests
 import requests_cache
 
-from morpheus.utils.verify_dependencies import _verify_deps
-
 logger = logging.getLogger(__name__)
 
-REQUIRED_DEPS = ('BeautifulSoup', 'feedparser')
+IMPORT_EXCEPTION = None
 IMPORT_ERROR_MESSAGE = "RSSController requires the bs4 and feedparser packages to be installed"
 
 try:
     import feedparser
     from bs4 import BeautifulSoup
-except ImportError:
-    pass
+except ImportError as import_exc:
+    IMPORT_EXCEPTION = import_exc
 
 
 @dataclass
@@ -81,7 +79,9 @@ def __init__(self,
                  cache_dir: str = "./.cache/http",
                  cooldown_interval: int = 600,
                  request_timeout: float = 2.0):
-        _verify_deps(REQUIRED_DEPS, IMPORT_ERROR_MESSAGE, globals())
+        if IMPORT_EXCEPTION is not None:
+            raise ImportError(IMPORT_ERROR_MESSAGE) from IMPORT_EXCEPTION
+
         if (isinstance(feed_input, str)):
             feed_input = [feed_input]
 

morpheus/llm/services/nemo_llm_service.py

@@ -19,10 +19,10 @@
 
 from morpheus.llm.services.llm_service import LLMClient
 from morpheus.llm.services.llm_service import LLMService
-from morpheus.utils.verify_dependencies import _verify_deps
 
 logger = logging.getLogger(__name__)
 
+IMPORT_EXCEPTION = None
 IMPORT_ERROR_MESSAGE = (
     "NemoLLM not found. Install it and other additional dependencies by running the following command:\n"
     "`mamba install -n base -c conda-forge conda-merge`\n"
@@ -32,8 +32,8 @@
 
 try:
     import nemollm
-except ImportError:
-    pass
+except ImportError as import_exc:
+    IMPORT_EXCEPTION = import_exc
 
 
 class NeMoLLMClient(LLMClient):
@@ -53,9 +53,10 @@ class NeMoLLMClient(LLMClient):
     """
 
     def __init__(self, parent: "NeMoLLMService", model_name: str, **model_kwargs: dict[str, typing.Any]) -> None:
-        super().__init__()
-        _verify_deps(('nemollm', ), IMPORT_ERROR_MESSAGE, globals())
+        if IMPORT_EXCEPTION is not None:
+            raise ImportError(IMPORT_ERROR_MESSAGE) from IMPORT_EXCEPTION
 
+        super().__init__()
         self._parent = parent
         self._model_name = model_name
         self._model_kwargs = model_kwargs
@@ -149,9 +150,10 @@ class NeMoLLMService(LLMService):
     """
 
     def __init__(self, *, api_key: str = None, org_id: str = None) -> None:
-        super().__init__()
-        _verify_deps(('nemollm', ), IMPORT_ERROR_MESSAGE, globals())
+        if IMPORT_EXCEPTION is not None:
+            raise ImportError(IMPORT_ERROR_MESSAGE) from IMPORT_EXCEPTION
 
+        super().__init__()
         api_key = api_key if api_key is not None else os.environ.get("NGC_API_KEY", None)
         org_id = org_id if org_id is not None else os.environ.get("NGC_ORG_ID", None)
 

morpheus/llm/services/openai_chat_service.py

@@ -19,10 +19,10 @@
 
 from morpheus.llm.services.llm_service import LLMClient
 from morpheus.llm.services.llm_service import LLMService
-from morpheus.utils.verify_dependencies import _verify_deps
 
 logger = logging.getLogger(__name__)
 
+IMPORT_EXCEPTION = None
 IMPORT_ERROR_MESSAGE = (
     "OpenAIChatService & OpenAIChatClient require the openai package to be installed. "
     "Install it by running the following command:\n"
@@ -33,8 +33,8 @@
 
 try:
     import openai
-except ImportError:
-    pass
+except ImportError as import_exc:
+    IMPORT_EXCEPTION = import_exc
 
 
 class OpenAIChatClient(LLMClient):
@@ -55,9 +55,10 @@ class OpenAIChatClient(LLMClient):
     """
 
     def __init__(self, model_name: str, set_assistant: bool = False, **model_kwargs: dict[str, typing.Any]) -> None:
-        super().__init__()
-        _verify_deps(('openai', ), IMPORT_ERROR_MESSAGE, globals())
+        if IMPORT_EXCEPTION is not None:
+            raise ImportError(IMPORT_ERROR_MESSAGE) from IMPORT_EXCEPTION
 
+        super().__init__()
         self._model_name = model_name
         self._set_assistant = set_assistant
         self._prompt_key = "prompt"
@@ -189,8 +190,10 @@ class OpenAIChatService(LLMService):
     """
 
     def __init__(self) -> None:
+        if IMPORT_EXCEPTION is not None:
+            raise ImportError(IMPORT_ERROR_MESSAGE) from IMPORT_EXCEPTION
+
         super().__init__()
-        _verify_deps(('openai', ), IMPORT_ERROR_MESSAGE, globals())
 
     def get_client(self,
                    model_name: str,

morpheus/service/vdb/milvus_vector_db_service.py

@@ -26,20 +26,19 @@
 
 from morpheus.service.vdb.vector_db_service import VectorDBResourceService
 from morpheus.service.vdb.vector_db_service import VectorDBService
-from morpheus.utils.verify_dependencies import _verify_deps
 
 logger = logging.getLogger(__name__)
 
-REQUIRED_DEPS = ('pymilvus', 'MilvusClient', 'MutationResult')
+IMPORT_EXCEPTION = None
 IMPORT_ERROR_MESSAGE = "MilvusVectorDBResourceService requires the milvus and pymilvus packages to be installed."
 
 try:
     import pymilvus
     from pymilvus.orm.mutation import MutationResult
 
     from morpheus.service.vdb.milvus_client import MilvusClient  # pylint: disable=ungrouped-imports
-except ImportError:
-    pass
+except ImportError as import_exc:
+    IMPORT_EXCEPTION = import_exc
 
 
 class FieldSchemaEncoder(json.JSONEncoder):
@@ -226,7 +225,9 @@ class MilvusVectorDBResourceService(VectorDBResourceService):
     """
 
     def __init__(self, name: str, client: "MilvusClient") -> None:
-        _verify_deps(REQUIRED_DEPS, IMPORT_ERROR_MESSAGE, globals())
+        if IMPORT_EXCEPTION is not None:
+            raise ImportError(IMPORT_ERROR_MESSAGE) from IMPORT_EXCEPTION
+
         super().__init__()
 
         self._name = name

morpheus/stages/input/databricks_deltalake_source_stage.py

@@ -24,19 +24,18 @@
 from morpheus.pipeline.preallocator_mixin import PreallocatorMixin
 from morpheus.pipeline.single_output_source import SingleOutputSource
 from morpheus.pipeline.stage_schema import StageSchema
-from morpheus.utils.verify_dependencies import _verify_deps
 
 logger = logging.getLogger(__name__)
 
-REQUIRED_DEPS = ('DatabricksSession', 'sf', 'Window')
+IMPORT_EXCEPTION = None
 IMPORT_ERROR_MESSAGE = "DatabricksDeltaLakeSourceStage requires the databricks-connect package to be installed."
 
 try:
     from databricks.connect import DatabricksSession
     from pyspark.sql import functions as sf
     from pyspark.sql.window import Window
-except ImportError:
-    pass
+except ImportError as import_exc:
+    IMPORT_EXCEPTION = import_exc
 
 
 @register_stage("from-databricks-deltalake")
@@ -67,7 +66,9 @@ def __init__(self,
                  databricks_host: str = None,
                  databricks_token: str = None,
                  databricks_cluster_id: str = None):
-        _verify_deps(REQUIRED_DEPS, IMPORT_ERROR_MESSAGE, globals())
+        if IMPORT_EXCEPTION is not None:
+            raise ImportError(IMPORT_ERROR_MESSAGE) from IMPORT_EXCEPTION
+
         super().__init__(config)
         self.spark_query = spark_query
         self.spark = DatabricksSession.builder.remote(host=databricks_host,

morpheus/stages/output/write_to_databricks_deltalake_stage.py

@@ -26,18 +26,17 @@
 from morpheus.messages import MessageMeta
 from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin
 from morpheus.pipeline.single_port_stage import SinglePortStage
-from morpheus.utils.verify_dependencies import _verify_deps
 
 logger = logging.getLogger(__name__)
 
-REQUIRED_DEPS = ('DatabricksSession', 'sql_types')
+IMPORT_EXCEPTION = None
 IMPORT_ERROR_MESSAGE = "DataBricksDeltaLakeSinkStage requires the databricks-connect package to be installed."
 
 try:
     from databricks.connect import DatabricksSession
     from pyspark.sql import types as sql_types
-except ImportError:
-    pass
+except ImportError as import_exc:
+    IMPORT_EXCEPTION = import_exc
 
 
 @register_stage("to-databricks-deltalake")
@@ -68,7 +67,9 @@ def __init__(self,
                  databricks_token: str = None,
                  databricks_cluster_id: str = None,
                  delta_table_write_mode: str = "append"):
-        _verify_deps(REQUIRED_DEPS, IMPORT_ERROR_MESSAGE, globals())
+        if IMPORT_EXCEPTION is not None:
+            raise ImportError(IMPORT_ERROR_MESSAGE) from IMPORT_EXCEPTION
+
         super().__init__(config)
         self.delta_path = delta_path
         self.delta_table_write_mode = delta_table_write_mode