diff --git a/pysqa/utils/__init__.py b/pysqa/base/__init__.py similarity index 100% rename from pysqa/utils/__init__.py rename to pysqa/base/__init__.py diff --git a/pysqa/utils/basic.py b/pysqa/base/config.py similarity index 83% rename from pysqa/utils/basic.py rename to pysqa/base/config.py index 56b42f96..6b2d7508 100644 --- a/pysqa/utils/basic.py +++ b/pysqa/base/config.py @@ -2,13 +2,58 @@ from typing import List, Optional, Tuple, Union import pandas +import yaml from jinja2 import Template from jinja2.exceptions import TemplateSyntaxError -from pysqa.utils.core import QueueAdapterCore -from pysqa.utils.execute import execute_command -from pysqa.utils.queues import Queues -from pysqa.utils.validate import value_error_if_none, value_in_range +from pysqa.base.core import QueueAdapterCore, execute_command +from pysqa.base.validate import check_queue_parameters, value_error_if_none + + +class Queues(object): + """ + Queues is an abstract class simply to make the list of queues available for auto completion. This is mainly used in + interactive environments like jupyter. + """ + + def __init__(self, list_of_queues: List[str]): + """ + Initialize the Queues object. + + Args: + list_of_queues (List[str]): A list of queue names. + + """ + self._list_of_queues = list_of_queues + + def __getattr__(self, item: str) -> str: + """ + Get the queue name. + + Args: + item (str): The name of the queue. + + Returns: + str: The name of the queue. + + Raises: + AttributeError: If the queue name is not in the list of queues. + + """ + if item in self._list_of_queues: + return item + else: + raise AttributeError + + def __dir__(self) -> List[str]: + """ + Get the list of queues. + + Returns: + List[str]: The list of queues. + + """ + return self._list_of_queues class QueueAdapterWithConfig(QueueAdapterCore): @@ -162,18 +207,12 @@ def check_queue_parameters( """ if active_queue is None: active_queue = self._config["queues"][queue] - cores = value_in_range( - value=cores, - value_min=active_queue["cores_min"], - value_max=active_queue["cores_max"], - ) - run_time_max = value_in_range( - value=run_time_max, value_max=active_queue["run_time_max"] - ) - memory_max = value_in_range( - value=memory_max, value_max=active_queue["memory_max"] + return check_queue_parameters( + active_queue=active_queue, + cores=cores, + run_time_max=run_time_max, + memory_max=memory_max, ) - return cores, run_time_max, memory_max def _job_submission_template( self, @@ -270,3 +309,17 @@ def _load_templates(queue_lst_dict: dict, directory: str = ".") -> None: + error.message, lineno=error.lineno, ) + + +def read_config(file_name: str = "queue.yaml") -> dict: + """ + Read and parse a YAML configuration file. + + Args: + file_name (str): The name of the YAML file to read. + + Returns: + dict: The parsed configuration as a dictionary. + """ + with open(file_name, "r") as f: + return yaml.load(f, Loader=yaml.FullLoader) diff --git a/pysqa/utils/core.py b/pysqa/base/core.py similarity index 89% rename from pysqa/utils/core.py rename to pysqa/base/core.py index 50b7f4fb..410a7bff 100644 --- a/pysqa/utils/core.py +++ b/pysqa/base/core.py @@ -1,12 +1,12 @@ import getpass import importlib import os +import subprocess from typing import List, Optional, Tuple, Union import pandas from jinja2 import Template -from pysqa.utils.execute import execute_command from pysqa.wrapper.abstract import SchedulerCommands queue_type_dict = { @@ -45,6 +45,46 @@ } +def execute_command( + commands: str, + working_directory: Optional[str] = None, + split_output: bool = True, + shell: bool = False, + error_filename: str = "pysqa.err", +) -> Union[str, List[str]]: + """ + A wrapper around the subprocess.check_output function. + + Args: + commands (str): The command(s) to be executed on the command line + working_directory (str, optional): The directory where the command is executed. Defaults to None. + split_output (bool, optional): Boolean flag to split newlines in the output. Defaults to True. + shell (bool, optional): Additional switch to convert commands to a single string. Defaults to False. + error_filename (str, optional): In case the execution fails, the output is written to this file. Defaults to "pysqa.err". + + Returns: + Union[str, List[str]]: Output of the shell command either as a string or as a list of strings + """ + if shell and isinstance(commands, list): + commands = " ".join(commands) + try: + out = subprocess.check_output( + commands, + cwd=working_directory, + stderr=subprocess.STDOUT, + universal_newlines=True, + shell=not isinstance(commands, list), + ) + except subprocess.CalledProcessError as e: + with open(os.path.join(working_directory, error_filename), "w") as f: + print(e.stdout, file=f) + out = None + if out is not None and split_output: + return out.split("\n") + else: + return out + + def get_queue_commands(queue_type: str) -> Union[SchedulerCommands, None]: """ Load queuing system commands class diff --git a/pysqa/utils/validate.py b/pysqa/base/validate.py similarity index 77% rename from pysqa/utils/validate.py rename to pysqa/base/validate.py index bc111767..77b7f43f 100644 --- a/pysqa/utils/validate.py +++ b/pysqa/base/validate.py @@ -1,5 +1,36 @@ import re -from typing import Union +from typing import Optional, Tuple, Union + + +def check_queue_parameters( + active_queue: Optional[dict] = None, + cores: int = 1, + run_time_max: Optional[int] = None, + memory_max: Optional[int] = None, +) -> Tuple[Union[float, int, None], Union[float, int, None], Union[float, int, None]]: + """ + Check the parameters of a queue. + + Args: + + cores (int, optional): The number of cores. Defaults to 1. + run_time_max (int, optional): The maximum run time. Defaults to None. + memory_max (int, optional): The maximum memory. Defaults to None. + active_queue (dict, optional): The active queue. Defaults to None. + + Returns: + list: A list of queue parameters [cores, run_time_max, memory_max]. + """ + cores = value_in_range( + value=cores, + value_min=active_queue["cores_min"], + value_max=active_queue["cores_max"], + ) + run_time_max = value_in_range( + value=run_time_max, value_max=active_queue["run_time_max"] + ) + memory_max = value_in_range(value=memory_max, value_max=active_queue["memory_max"]) + return cores, run_time_max, memory_max def value_error_if_none(value: str) -> None: diff --git a/pysqa/cmd.py b/pysqa/cmd.py index 8c07dc90..0661a41f 100644 --- a/pysqa/cmd.py +++ b/pysqa/cmd.py @@ -4,8 +4,8 @@ import sys from typing import Optional +from pysqa.base.core import execute_command from pysqa.queueadapter import QueueAdapter -from pysqa.utils.execute import execute_command def command_line( diff --git a/pysqa/ext/modular.py b/pysqa/ext/modular.py index f908e9bb..382cf92b 100644 --- a/pysqa/ext/modular.py +++ b/pysqa/ext/modular.py @@ -2,8 +2,8 @@ import pandas -from pysqa.utils.basic import QueueAdapterWithConfig -from pysqa.utils.execute import execute_command +from pysqa.base.config import QueueAdapterWithConfig +from pysqa.base.core import execute_command class ModularQueueAdapter(QueueAdapterWithConfig): diff --git a/pysqa/ext/remote.py b/pysqa/ext/remote.py index d5714d31..7d60a355 100644 --- a/pysqa/ext/remote.py +++ b/pysqa/ext/remote.py @@ -8,8 +8,8 @@ import paramiko from tqdm import tqdm -from pysqa.utils.basic import QueueAdapterWithConfig -from pysqa.utils.execute import execute_command +from pysqa.base.config import QueueAdapterWithConfig +from pysqa.base.core import execute_command class RemoteQueueAdapter(QueueAdapterWithConfig): diff --git a/pysqa/queueadapter.py b/pysqa/queueadapter.py index 57ba017f..2e3ca760 100644 --- a/pysqa/queueadapter.py +++ b/pysqa/queueadapter.py @@ -3,10 +3,9 @@ import pandas +from pysqa.base.config import QueueAdapterWithConfig, read_config +from pysqa.base.core import execute_command from pysqa.ext.modular import ModularQueueAdapter -from pysqa.utils.basic import QueueAdapterWithConfig -from pysqa.utils.config import read_config -from pysqa.utils.execute import execute_command class QueueAdapter(object): diff --git a/pysqa/utils/config.py b/pysqa/utils/config.py deleted file mode 100644 index 1883ea56..00000000 --- a/pysqa/utils/config.py +++ /dev/null @@ -1,15 +0,0 @@ -import yaml - - -def read_config(file_name: str = "queue.yaml") -> dict: - """ - Read and parse a YAML configuration file. - - Args: - file_name (str): The name of the YAML file to read. - - Returns: - dict: The parsed configuration as a dictionary. - """ - with open(file_name, "r") as f: - return yaml.load(f, Loader=yaml.FullLoader) diff --git a/pysqa/utils/execute.py b/pysqa/utils/execute.py deleted file mode 100644 index 9b4fdf0b..00000000 --- a/pysqa/utils/execute.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import subprocess -from typing import List, Optional, Union - - -def execute_command( - commands: str, - working_directory: Optional[str] = None, - split_output: bool = True, - shell: bool = False, - error_filename: str = "pysqa.err", -) -> Union[str, List[str]]: - """ - A wrapper around the subprocess.check_output function. - - Args: - commands (str): The command(s) to be executed on the command line - working_directory (str, optional): The directory where the command is executed. Defaults to None. - split_output (bool, optional): Boolean flag to split newlines in the output. Defaults to True. - shell (bool, optional): Additional switch to convert commands to a single string. Defaults to False. - error_filename (str, optional): In case the execution fails, the output is written to this file. Defaults to "pysqa.err". - - Returns: - Union[str, List[str]]: Output of the shell command either as a string or as a list of strings - """ - if shell and isinstance(commands, list): - commands = " ".join(commands) - try: - out = subprocess.check_output( - commands, - cwd=working_directory, - stderr=subprocess.STDOUT, - universal_newlines=True, - shell=not isinstance(commands, list), - ) - except subprocess.CalledProcessError as e: - with open(os.path.join(working_directory, error_filename), "w") as f: - print(e.stdout, file=f) - out = None - if out is not None and split_output: - return out.split("\n") - else: - return out diff --git a/pysqa/utils/queues.py b/pysqa/utils/queues.py deleted file mode 100644 index 3b348a23..00000000 --- a/pysqa/utils/queues.py +++ /dev/null @@ -1,47 +0,0 @@ -from typing import List - - -class Queues(object): - """ - Queues is an abstract class simply to make the list of queues available for auto completion. This is mainly used in - interactive environments like jupyter. - """ - - def __init__(self, list_of_queues: List[str]): - """ - Initialize the Queues object. - - Args: - list_of_queues (List[str]): A list of queue names. - - """ - self._list_of_queues = list_of_queues - - def __getattr__(self, item: str) -> str: - """ - Get the queue name. - - Args: - item (str): The name of the queue. - - Returns: - str: The name of the queue. - - Raises: - AttributeError: If the queue name is not in the list of queues. - - """ - if item in self._list_of_queues: - return item - else: - raise AttributeError - - def __dir__(self) -> List[str]: - """ - Get the list of queues. - - Returns: - List[str]: The list of queues. - - """ - return self._list_of_queues diff --git a/tests/test_basic.py b/tests/test_basic.py index 4eeeaf1b..592c440a 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -5,8 +5,8 @@ import unittest from jinja2.exceptions import TemplateSyntaxError from pysqa import QueueAdapter -from pysqa.utils.basic import QueueAdapterWithConfig -from pysqa.utils.validate import value_in_range +from pysqa.base.config import QueueAdapterWithConfig +from pysqa.base.validate import value_in_range __author__ = "Jan Janssen" __copyright__ = "Copyright 2019, Jan Janssen" diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py index 7bcb8d48..6611cf4a 100644 --- a/tests/test_execute_command.py +++ b/tests/test_execute_command.py @@ -1,6 +1,6 @@ import os import unittest -from pysqa.utils.execute import execute_command +from pysqa.base.core import execute_command class TestExecuteCommand(unittest.TestCase): diff --git a/tests/test_sge.py b/tests/test_sge.py index 06b6fb98..56eacc05 100644 --- a/tests/test_sge.py +++ b/tests/test_sge.py @@ -6,7 +6,7 @@ import unittest import getpass from pysqa import QueueAdapter -from pysqa.utils.validate import value_in_range +from pysqa.base.validate import value_in_range try: import defusedxml.ElementTree as ETree