From b4f0fa1e46eaa1394ce7e2bb3b16134aec68df8b Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 3 Nov 2023 18:41:41 -0400 Subject: [PATCH] fix lint and reduce builtin process duplicated logic --- weaver/processes/builtin/file2string_array.py | 7 ++----- .../processes/builtin/file_index_selector.py | 7 ++----- weaver/processes/builtin/jsonarray2netcdf.py | 11 ++++++----- weaver/processes/builtin/metalink2netcdf.py | 8 +++----- weaver/processes/builtin/utils.py | 18 +++++++++++++++++- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/weaver/processes/builtin/file2string_array.py b/weaver/processes/builtin/file2string_array.py index e5977462c..6c0b90eff 100644 --- a/weaver/processes/builtin/file2string_array.py +++ b/weaver/processes/builtin/file2string_array.py @@ -15,12 +15,9 @@ # place weaver specific imports after sys path fixing to ensure they are found from external call # pylint: disable=C0413,wrong-import-order -from weaver import WEAVER_ROOT_DIR # isort:skip # noqa: E402 -from weaver.processes.builtin.utils import validate_reference # isort:skip # noqa: E402 +from weaver.processes.builtin.utils import get_package_details, validate_reference # isort:skip # noqa: E402 -PACKAGE_NAME = os.path.split(os.path.splitext(__file__)[0])[-1] -PACKAGE_BASE = __file__.rsplit(WEAVER_ROOT_DIR.rstrip("/") + "/", 1)[-1].rsplit(PACKAGE_NAME)[0] -PACKAGE_MODULE = f"{PACKAGE_BASE}{PACKAGE_NAME}".replace("/", ".") +PACKAGE_NAME, PACKAGE_BASE, PACKAGE_MODULE = get_package_details(__file__) # setup logger since it is not run from the main 'weaver' app LOGGER = logging.getLogger(PACKAGE_MODULE) diff --git a/weaver/processes/builtin/file_index_selector.py b/weaver/processes/builtin/file_index_selector.py index 6396a0814..947f3bdcf 100644 --- a/weaver/processes/builtin/file_index_selector.py +++ b/weaver/processes/builtin/file_index_selector.py @@ -19,13 +19,10 @@ # place weaver specific imports after sys path fixing to ensure they are found from external call # pylint: disable=C0413,wrong-import-order -from weaver import WEAVER_ROOT_DIR # isort:skip # noqa: E402 -from weaver.processes.builtin.utils import validate_reference # isort:skip # noqa: E402 +from weaver.processes.builtin.utils import get_package_details, validate_reference # isort:skip # noqa: E402 from weaver.utils import OutputMethod, fetch_file # isort:skip # noqa: E402 -PACKAGE_NAME = os.path.split(os.path.splitext(__file__)[0])[-1] -PACKAGE_BASE = __file__.rsplit(WEAVER_ROOT_DIR.rstrip("/") + "/", 1)[-1].rsplit(PACKAGE_NAME)[0] -PACKAGE_MODULE = f"{PACKAGE_BASE}{PACKAGE_NAME}".replace("/", ".") +PACKAGE_NAME, PACKAGE_BASE, PACKAGE_MODULE = get_package_details(__file__) # setup logger since it is not run from the main 'weaver' app LOGGER = logging.getLogger(PACKAGE_MODULE) diff --git a/weaver/processes/builtin/jsonarray2netcdf.py b/weaver/processes/builtin/jsonarray2netcdf.py index 1d97e951b..012dbb84f 100644 --- a/weaver/processes/builtin/jsonarray2netcdf.py +++ b/weaver/processes/builtin/jsonarray2netcdf.py @@ -17,14 +17,15 @@ # place weaver specific imports after sys path fixing to ensure they are found from external call # pylint: disable=C0413,wrong-import-order -from weaver import WEAVER_ROOT_DIR # isort:skip # noqa: E402 from weaver.formats import repr_json # isort:skip # noqa: E402 -from weaver.processes.builtin.utils import is_netcdf_url, validate_reference # isort:skip # noqa: E402 +from weaver.processes.builtin.utils import ( # isort:skip # noqa: E402 + get_package_details, + is_netcdf_url, + validate_reference +) from weaver.utils import fetch_file, get_secure_path # isort:skip # noqa: E402 -PACKAGE_NAME = os.path.split(os.path.splitext(__file__)[0])[-1] -PACKAGE_BASE = __file__.rsplit(WEAVER_ROOT_DIR.rstrip("/") + "/", 1)[-1].rsplit(PACKAGE_NAME)[0] -PACKAGE_MODULE = f"{PACKAGE_BASE}{PACKAGE_NAME}".replace("/", ".") +PACKAGE_NAME, PACKAGE_BASE, PACKAGE_MODULE = get_package_details(__file__) # setup logger since it is not run from the main 'weaver' app LOGGER = logging.getLogger(PACKAGE_MODULE) diff --git a/weaver/processes/builtin/metalink2netcdf.py b/weaver/processes/builtin/metalink2netcdf.py index 3173a7bbe..44819bdf4 100644 --- a/weaver/processes/builtin/metalink2netcdf.py +++ b/weaver/processes/builtin/metalink2netcdf.py @@ -16,14 +16,12 @@ # place weaver specific imports after sys path fixing to ensure they are found from external call # pylint: disable=C0413,wrong-import-order -from weaver import WEAVER_ROOT_DIR, xml_util # isort:skip # noqa: E402 -from weaver.processes.builtin.utils import validate_reference # isort:skip # noqa: E402 +from weaver import xml_util # isort:skip # noqa: E402 +from weaver.processes.builtin.utils import get_package_details, validate_reference # isort:skip # noqa: E402 from weaver.utils import fetch_file # isort:skip # noqa: E402 from weaver.processes.builtin.utils import is_netcdf_url # isort:skip # noqa: E402 -PACKAGE_NAME = os.path.split(os.path.splitext(__file__)[0])[-1] -PACKAGE_BASE = __file__.rsplit(WEAVER_ROOT_DIR.rstrip("/") + "/", 1)[-1].rsplit(PACKAGE_NAME)[0] -PACKAGE_MODULE = f"{PACKAGE_BASE}{PACKAGE_NAME}".replace("/", ".") +PACKAGE_NAME, PACKAGE_BASE, PACKAGE_MODULE = get_package_details(__file__) # setup logger since it is not run from the main 'weaver' app LOGGER = logging.getLogger(PACKAGE_MODULE) diff --git a/weaver/processes/builtin/utils.py b/weaver/processes/builtin/utils.py index 1db2a1e6e..c7c52ebf4 100644 --- a/weaver/processes/builtin/utils.py +++ b/weaver/processes/builtin/utils.py @@ -1,10 +1,14 @@ import os import tempfile -from typing import Any +from typing import TYPE_CHECKING from urllib.parse import urlparse +from weaver import WEAVER_ROOT_DIR from weaver.formats import ContentType, get_extension +if TYPE_CHECKING: + from typing import Any, Tuple + def is_netcdf_url(url): # type: (Any) -> bool @@ -49,3 +53,15 @@ def validate_reference(url, is_file): return if urlparse(url).scheme not in ["http", "https", "s3"]: raise ValueError(f"Not a valid file URL reference [{url}]. Scheme not supported.") + + +def get_package_details(file): + # type: (os.PathLike[str]) -> Tuple[str, str, str] + """ + Obtains the ``builtin`` process details from its file reference. + """ + name = os.path.split(os.path.splitext(file)[0])[-1] + root = WEAVER_ROOT_DIR.rstrip("/") # avoid double // + path = str(file).rsplit(f"{root}/", 1)[-1].rsplit(name)[0] + mod = f"{path}{name}".replace("/", ".") + return name, path, mod