From 2b99894dc3494a7702972339f269fc85dc307bc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sat, 28 Sep 2024 17:03:38 +0200 Subject: [PATCH 1/2] Refactor interface --- pysqa/{utils => base}/__init__.py | 0 pysqa/{utils/basic.py => base/config.py} | 85 +++++++++++++++++++----- pysqa/{utils => base}/core.py | 42 +++++++++++- pysqa/{utils => base}/validate.py | 36 +++++++++- pysqa/cmd.py | 2 +- pysqa/ext/modular.py | 4 +- pysqa/ext/remote.py | 4 +- pysqa/queueadapter.py | 5 +- pysqa/utils/config.py | 15 ----- pysqa/utils/execute.py | 43 ------------ pysqa/utils/queues.py | 47 ------------- tests/test_basic.py | 4 +- tests/test_execute_command.py | 2 +- tests/test_sge.py | 2 +- 14 files changed, 156 insertions(+), 135 deletions(-) rename pysqa/{utils => base}/__init__.py (100%) rename pysqa/{utils/basic.py => base/config.py} (83%) rename pysqa/{utils => base}/core.py (89%) rename pysqa/{utils => base}/validate.py (77%) delete mode 100644 pysqa/utils/config.py delete mode 100644 pysqa/utils/execute.py delete mode 100644 pysqa/utils/queues.py diff --git a/pysqa/utils/__init__.py b/pysqa/base/__init__.py similarity index 100% rename from pysqa/utils/__init__.py rename to pysqa/base/__init__.py diff --git a/pysqa/utils/basic.py b/pysqa/base/config.py similarity index 83% rename from pysqa/utils/basic.py rename to pysqa/base/config.py index 56b42f96..1358411a 100644 --- a/pysqa/utils/basic.py +++ b/pysqa/base/config.py @@ -1,14 +1,59 @@ import os from typing import List, Optional, Tuple, Union -import pandas from jinja2 import Template from jinja2.exceptions import TemplateSyntaxError +import pandas +import yaml + +from pysqa.base.core import QueueAdapterCore, execute_command +from pysqa.base.validate import value_error_if_none, check_queue_parameters + + +class Queues(object): + """ + Queues is an abstract class simply to make the list of queues available for auto completion. This is mainly used in + interactive environments like jupyter. + """ + + def __init__(self, list_of_queues: List[str]): + """ + Initialize the Queues object. + + Args: + list_of_queues (List[str]): A list of queue names. + + """ + self._list_of_queues = list_of_queues + + def __getattr__(self, item: str) -> str: + """ + Get the queue name. + + Args: + item (str): The name of the queue. + + Returns: + str: The name of the queue. + + Raises: + AttributeError: If the queue name is not in the list of queues. -from pysqa.utils.core import QueueAdapterCore -from pysqa.utils.execute import execute_command -from pysqa.utils.queues import Queues -from pysqa.utils.validate import value_error_if_none, value_in_range + """ + if item in self._list_of_queues: + return item + else: + raise AttributeError + + def __dir__(self) -> List[str]: + """ + Get the list of queues. + + Returns: + List[str]: The list of queues. + + """ + return self._list_of_queues class QueueAdapterWithConfig(QueueAdapterCore): @@ -162,18 +207,12 @@ def check_queue_parameters( """ if active_queue is None: active_queue = self._config["queues"][queue] - cores = value_in_range( - value=cores, - value_min=active_queue["cores_min"], - value_max=active_queue["cores_max"], - ) - run_time_max = value_in_range( - value=run_time_max, value_max=active_queue["run_time_max"] - ) - memory_max = value_in_range( - value=memory_max, value_max=active_queue["memory_max"] + return check_queue_parameters( + active_queue=active_queue, + cores=cores, + run_time_max=run_time_max, + memory_max=memory_max, ) - return cores, run_time_max, memory_max def _job_submission_template( self, @@ -270,3 +309,17 @@ def _load_templates(queue_lst_dict: dict, directory: str = ".") -> None: + error.message, lineno=error.lineno, ) + + +def read_config(file_name: str = "queue.yaml") -> dict: + """ + Read and parse a YAML configuration file. + + Args: + file_name (str): The name of the YAML file to read. + + Returns: + dict: The parsed configuration as a dictionary. + """ + with open(file_name, "r") as f: + return yaml.load(f, Loader=yaml.FullLoader) \ No newline at end of file diff --git a/pysqa/utils/core.py b/pysqa/base/core.py similarity index 89% rename from pysqa/utils/core.py rename to pysqa/base/core.py index 50b7f4fb..410a7bff 100644 --- a/pysqa/utils/core.py +++ b/pysqa/base/core.py @@ -1,12 +1,12 @@ import getpass import importlib import os +import subprocess from typing import List, Optional, Tuple, Union import pandas from jinja2 import Template -from pysqa.utils.execute import execute_command from pysqa.wrapper.abstract import SchedulerCommands queue_type_dict = { @@ -45,6 +45,46 @@ } +def execute_command( + commands: str, + working_directory: Optional[str] = None, + split_output: bool = True, + shell: bool = False, + error_filename: str = "pysqa.err", +) -> Union[str, List[str]]: + """ + A wrapper around the subprocess.check_output function. + + Args: + commands (str): The command(s) to be executed on the command line + working_directory (str, optional): The directory where the command is executed. Defaults to None. + split_output (bool, optional): Boolean flag to split newlines in the output. Defaults to True. + shell (bool, optional): Additional switch to convert commands to a single string. Defaults to False. + error_filename (str, optional): In case the execution fails, the output is written to this file. Defaults to "pysqa.err". + + Returns: + Union[str, List[str]]: Output of the shell command either as a string or as a list of strings + """ + if shell and isinstance(commands, list): + commands = " ".join(commands) + try: + out = subprocess.check_output( + commands, + cwd=working_directory, + stderr=subprocess.STDOUT, + universal_newlines=True, + shell=not isinstance(commands, list), + ) + except subprocess.CalledProcessError as e: + with open(os.path.join(working_directory, error_filename), "w") as f: + print(e.stdout, file=f) + out = None + if out is not None and split_output: + return out.split("\n") + else: + return out + + def get_queue_commands(queue_type: str) -> Union[SchedulerCommands, None]: """ Load queuing system commands class diff --git a/pysqa/utils/validate.py b/pysqa/base/validate.py similarity index 77% rename from pysqa/utils/validate.py rename to pysqa/base/validate.py index bc111767..239bd707 100644 --- a/pysqa/utils/validate.py +++ b/pysqa/base/validate.py @@ -1,7 +1,41 @@ import re -from typing import Union +from typing import Union, Tuple, Optional +def check_queue_parameters( + active_queue: Optional[dict] = None, + cores: int = 1, + run_time_max: Optional[int] = None, + memory_max: Optional[int] = None, +) -> Tuple[ + Union[float, int, None], Union[float, int, None], Union[float, int, None] +]: + """ + Check the parameters of a queue. + + Args: + + cores (int, optional): The number of cores. Defaults to 1. + run_time_max (int, optional): The maximum run time. Defaults to None. + memory_max (int, optional): The maximum memory. Defaults to None. + active_queue (dict, optional): The active queue. Defaults to None. + + Returns: + list: A list of queue parameters [cores, run_time_max, memory_max]. + """ + cores = value_in_range( + value=cores, + value_min=active_queue["cores_min"], + value_max=active_queue["cores_max"], + ) + run_time_max = value_in_range( + value=run_time_max, value_max=active_queue["run_time_max"] + ) + memory_max = value_in_range( + value=memory_max, value_max=active_queue["memory_max"] + ) + return cores, run_time_max, memory_max + def value_error_if_none(value: str) -> None: """ Raise a ValueError if the value is None or not a string. diff --git a/pysqa/cmd.py b/pysqa/cmd.py index 8c07dc90..2bc0a7e5 100644 --- a/pysqa/cmd.py +++ b/pysqa/cmd.py @@ -5,7 +5,7 @@ from typing import Optional from pysqa.queueadapter import QueueAdapter -from pysqa.utils.execute import execute_command +from pysqa.base.core import execute_command def command_line( diff --git a/pysqa/ext/modular.py b/pysqa/ext/modular.py index f908e9bb..382cf92b 100644 --- a/pysqa/ext/modular.py +++ b/pysqa/ext/modular.py @@ -2,8 +2,8 @@ import pandas -from pysqa.utils.basic import QueueAdapterWithConfig -from pysqa.utils.execute import execute_command +from pysqa.base.config import QueueAdapterWithConfig +from pysqa.base.core import execute_command class ModularQueueAdapter(QueueAdapterWithConfig): diff --git a/pysqa/ext/remote.py b/pysqa/ext/remote.py index d5714d31..7d60a355 100644 --- a/pysqa/ext/remote.py +++ b/pysqa/ext/remote.py @@ -8,8 +8,8 @@ import paramiko from tqdm import tqdm -from pysqa.utils.basic import QueueAdapterWithConfig -from pysqa.utils.execute import execute_command +from pysqa.base.config import QueueAdapterWithConfig +from pysqa.base.core import execute_command class RemoteQueueAdapter(QueueAdapterWithConfig): diff --git a/pysqa/queueadapter.py b/pysqa/queueadapter.py index 57ba017f..4f4d4edf 100644 --- a/pysqa/queueadapter.py +++ b/pysqa/queueadapter.py @@ -4,9 +4,8 @@ import pandas from pysqa.ext.modular import ModularQueueAdapter -from pysqa.utils.basic import QueueAdapterWithConfig -from pysqa.utils.config import read_config -from pysqa.utils.execute import execute_command +from pysqa.base.config import QueueAdapterWithConfig, read_config +from pysqa.base.core import execute_command class QueueAdapter(object): diff --git a/pysqa/utils/config.py b/pysqa/utils/config.py deleted file mode 100644 index 1883ea56..00000000 --- a/pysqa/utils/config.py +++ /dev/null @@ -1,15 +0,0 @@ -import yaml - - -def read_config(file_name: str = "queue.yaml") -> dict: - """ - Read and parse a YAML configuration file. - - Args: - file_name (str): The name of the YAML file to read. - - Returns: - dict: The parsed configuration as a dictionary. - """ - with open(file_name, "r") as f: - return yaml.load(f, Loader=yaml.FullLoader) diff --git a/pysqa/utils/execute.py b/pysqa/utils/execute.py deleted file mode 100644 index 9b4fdf0b..00000000 --- a/pysqa/utils/execute.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import subprocess -from typing import List, Optional, Union - - -def execute_command( - commands: str, - working_directory: Optional[str] = None, - split_output: bool = True, - shell: bool = False, - error_filename: str = "pysqa.err", -) -> Union[str, List[str]]: - """ - A wrapper around the subprocess.check_output function. - - Args: - commands (str): The command(s) to be executed on the command line - working_directory (str, optional): The directory where the command is executed. Defaults to None. - split_output (bool, optional): Boolean flag to split newlines in the output. Defaults to True. - shell (bool, optional): Additional switch to convert commands to a single string. Defaults to False. - error_filename (str, optional): In case the execution fails, the output is written to this file. Defaults to "pysqa.err". - - Returns: - Union[str, List[str]]: Output of the shell command either as a string or as a list of strings - """ - if shell and isinstance(commands, list): - commands = " ".join(commands) - try: - out = subprocess.check_output( - commands, - cwd=working_directory, - stderr=subprocess.STDOUT, - universal_newlines=True, - shell=not isinstance(commands, list), - ) - except subprocess.CalledProcessError as e: - with open(os.path.join(working_directory, error_filename), "w") as f: - print(e.stdout, file=f) - out = None - if out is not None and split_output: - return out.split("\n") - else: - return out diff --git a/pysqa/utils/queues.py b/pysqa/utils/queues.py deleted file mode 100644 index 3b348a23..00000000 --- a/pysqa/utils/queues.py +++ /dev/null @@ -1,47 +0,0 @@ -from typing import List - - -class Queues(object): - """ - Queues is an abstract class simply to make the list of queues available for auto completion. This is mainly used in - interactive environments like jupyter. - """ - - def __init__(self, list_of_queues: List[str]): - """ - Initialize the Queues object. - - Args: - list_of_queues (List[str]): A list of queue names. - - """ - self._list_of_queues = list_of_queues - - def __getattr__(self, item: str) -> str: - """ - Get the queue name. - - Args: - item (str): The name of the queue. - - Returns: - str: The name of the queue. - - Raises: - AttributeError: If the queue name is not in the list of queues. - - """ - if item in self._list_of_queues: - return item - else: - raise AttributeError - - def __dir__(self) -> List[str]: - """ - Get the list of queues. - - Returns: - List[str]: The list of queues. - - """ - return self._list_of_queues diff --git a/tests/test_basic.py b/tests/test_basic.py index 4eeeaf1b..592c440a 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -5,8 +5,8 @@ import unittest from jinja2.exceptions import TemplateSyntaxError from pysqa import QueueAdapter -from pysqa.utils.basic import QueueAdapterWithConfig -from pysqa.utils.validate import value_in_range +from pysqa.base.config import QueueAdapterWithConfig +from pysqa.base.validate import value_in_range __author__ = "Jan Janssen" __copyright__ = "Copyright 2019, Jan Janssen" diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py index 7bcb8d48..6611cf4a 100644 --- a/tests/test_execute_command.py +++ b/tests/test_execute_command.py @@ -1,6 +1,6 @@ import os import unittest -from pysqa.utils.execute import execute_command +from pysqa.base.core import execute_command class TestExecuteCommand(unittest.TestCase): diff --git a/tests/test_sge.py b/tests/test_sge.py index 06b6fb98..56eacc05 100644 --- a/tests/test_sge.py +++ b/tests/test_sge.py @@ -6,7 +6,7 @@ import unittest import getpass from pysqa import QueueAdapter -from pysqa.utils.validate import value_in_range +from pysqa.base.validate import value_in_range try: import defusedxml.ElementTree as ETree From ec4f38d64eac7e2a1e3812420479976da65eeb77 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 28 Sep 2024 15:04:33 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pysqa/base/config.py | 8 ++++---- pysqa/base/validate.py | 11 ++++------- pysqa/cmd.py | 2 +- pysqa/queueadapter.py | 2 +- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/pysqa/base/config.py b/pysqa/base/config.py index 1358411a..6b2d7508 100644 --- a/pysqa/base/config.py +++ b/pysqa/base/config.py @@ -1,13 +1,13 @@ import os from typing import List, Optional, Tuple, Union -from jinja2 import Template -from jinja2.exceptions import TemplateSyntaxError import pandas import yaml +from jinja2 import Template +from jinja2.exceptions import TemplateSyntaxError from pysqa.base.core import QueueAdapterCore, execute_command -from pysqa.base.validate import value_error_if_none, check_queue_parameters +from pysqa.base.validate import check_queue_parameters, value_error_if_none class Queues(object): @@ -322,4 +322,4 @@ def read_config(file_name: str = "queue.yaml") -> dict: dict: The parsed configuration as a dictionary. """ with open(file_name, "r") as f: - return yaml.load(f, Loader=yaml.FullLoader) \ No newline at end of file + return yaml.load(f, Loader=yaml.FullLoader) diff --git a/pysqa/base/validate.py b/pysqa/base/validate.py index 239bd707..77b7f43f 100644 --- a/pysqa/base/validate.py +++ b/pysqa/base/validate.py @@ -1,5 +1,5 @@ import re -from typing import Union, Tuple, Optional +from typing import Optional, Tuple, Union def check_queue_parameters( @@ -7,9 +7,7 @@ def check_queue_parameters( cores: int = 1, run_time_max: Optional[int] = None, memory_max: Optional[int] = None, -) -> Tuple[ - Union[float, int, None], Union[float, int, None], Union[float, int, None] -]: +) -> Tuple[Union[float, int, None], Union[float, int, None], Union[float, int, None]]: """ Check the parameters of a queue. @@ -31,11 +29,10 @@ def check_queue_parameters( run_time_max = value_in_range( value=run_time_max, value_max=active_queue["run_time_max"] ) - memory_max = value_in_range( - value=memory_max, value_max=active_queue["memory_max"] - ) + memory_max = value_in_range(value=memory_max, value_max=active_queue["memory_max"]) return cores, run_time_max, memory_max + def value_error_if_none(value: str) -> None: """ Raise a ValueError if the value is None or not a string. diff --git a/pysqa/cmd.py b/pysqa/cmd.py index 2bc0a7e5..0661a41f 100644 --- a/pysqa/cmd.py +++ b/pysqa/cmd.py @@ -4,8 +4,8 @@ import sys from typing import Optional -from pysqa.queueadapter import QueueAdapter from pysqa.base.core import execute_command +from pysqa.queueadapter import QueueAdapter def command_line( diff --git a/pysqa/queueadapter.py b/pysqa/queueadapter.py index 4f4d4edf..2e3ca760 100644 --- a/pysqa/queueadapter.py +++ b/pysqa/queueadapter.py @@ -3,9 +3,9 @@ import pandas -from pysqa.ext.modular import ModularQueueAdapter from pysqa.base.config import QueueAdapterWithConfig, read_config from pysqa.base.core import execute_command +from pysqa.ext.modular import ModularQueueAdapter class QueueAdapter(object):