From 83d253ceef6171d97c1ca493dd0dd40e2a56f96f Mon Sep 17 00:00:00 2001 From: Andrey Maslennikov Date: Fri, 10 May 2024 12:00:21 +0200 Subject: [PATCH 01/11] Add CI --- .github/workflows/ci.yml | 55 ++++++++++++++++++++++++ .gitignore | 91 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..06eda41fe --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,55 @@ +name: CI + +on: + pull_request: + +jobs: + + lint: + name: Linting + + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - name: Install dependencies + run: pip install -r requirements-dev.txt + + - name: Run ruff linter + run: ruff check . + + - name: Run ruff formatter + run: ruff format --check --diff . + + - name: Run pyright + run: pyright . + + - name: Run vulture check + run: vulture cloudai/ tests/ ci_tools/ --min-confidence 100 + + test: + name: Run pytest + + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - name: Install dependencies + run: pip install pytest -r requirements-dev.txt + + - name: Run pytest + run: pytest -vv diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..092d3741e --- /dev/null +++ b/.gitignore @@ -0,0 +1,91 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# pytype static type analyzer +.pytype/ + +# pycharm +.idea/ + +# VSCode +.vscode/ + +# Editors and IDEs +*.swp +*.bak +*.tmp +*~ +*.sublime-project +*.sublime-workspace + +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +*.log +install/ +results/ +.* From 0a49493c69864c859e279eea19650d1c8d38547c Mon Sep 17 00:00:00 2001 From: Andrey Maslennikov Date: Mon, 13 May 2024 16:05:50 +0200 Subject: [PATCH 02/11] Use src layout for packaging After installation it generates single `site-packages/cloudai` folder and put everything under it instead of spreading it out. --- pyproject.toml | 3 --- requirements-dev.txt | 1 + {cloudai => src/cloudai}/__init__.py | 0 {cloudai => src/cloudai}/grader/__init__.py | 0 {cloudai => src/cloudai}/grader/grader.py | 0 {cloudai => src/cloudai}/installer/__init__.py | 0 {cloudai => src/cloudai}/installer/base_installer.py | 0 {cloudai => src/cloudai}/installer/installer.py | 0 {cloudai => src/cloudai}/installer/slurm_installer.py | 0 {cloudai => src/cloudai}/installer/standalone_installer.py | 0 {cloudai => src/cloudai}/parser/__init__.py | 0 {cloudai => src/cloudai}/parser/core/__init__.py | 0 {cloudai => src/cloudai}/parser/core/base_multi_file_parser.py | 0 {cloudai => src/cloudai}/parser/core/base_system_parser.py | 0 {cloudai => src/cloudai}/parser/core/parser.py | 0 {cloudai => src/cloudai}/parser/core/system_parser.py | 0 {cloudai => src/cloudai}/parser/core/test_parser.py | 0 {cloudai => src/cloudai}/parser/core/test_scenario_parser.py | 0 {cloudai => src/cloudai}/parser/core/test_template_parser.py | 0 {cloudai => src/cloudai}/parser/system_parser/__init__.py | 0 .../cloudai}/parser/system_parser/slurm_system_parser.py | 0 .../cloudai}/parser/system_parser/standalone_system_parser.py | 0 {cloudai => src/cloudai}/report_generator/__init__.py | 0 {cloudai => src/cloudai}/report_generator/report_generator.py | 0 {cloudai => src/cloudai}/report_generator/tool/__init__.py | 0 .../cloudai}/report_generator/tool/bokeh_report_tool.py | 0 .../cloudai}/report_generator/tool/report_tool_interface.py | 0 .../cloudai}/report_generator/tool/tensorboard_data_reader.py | 0 {cloudai => src/cloudai}/report_generator/util.py | 0 {cloudai => src/cloudai}/runner/__init__.py | 0 {cloudai => src/cloudai}/runner/core/__init__.py | 0 {cloudai => src/cloudai}/runner/core/base_job.py | 0 {cloudai => src/cloudai}/runner/core/base_runner.py | 0 {cloudai => src/cloudai}/runner/core/runner.py | 0 {cloudai => src/cloudai}/runner/slurm/__init__.py | 0 {cloudai => src/cloudai}/runner/slurm/slurm_job.py | 0 {cloudai => src/cloudai}/runner/slurm/slurm_runner.py | 0 {cloudai => src/cloudai}/runner/standalone/__init__.py | 0 {cloudai => src/cloudai}/runner/standalone/standalone_job.py | 0 .../cloudai}/runner/standalone/standalone_runner.py | 0 {cloudai => src/cloudai}/schema/__init__.py | 0 {cloudai => src/cloudai}/schema/core/__init__.py | 0 {cloudai => src/cloudai}/schema/core/strategy/__init__.py | 0 .../cloudai}/schema/core/strategy/command_gen_strategy.py | 0 .../cloudai}/schema/core/strategy/grading_strategy.py | 0 .../cloudai}/schema/core/strategy/install_strategy.py | 0 .../cloudai}/schema/core/strategy/job_id_retrieval_strategy.py | 0 .../schema/core/strategy/report_generation_strategy.py | 0 .../cloudai}/schema/core/strategy/strategy_registry.py | 0 .../cloudai}/schema/core/strategy/test_template_strategy.py | 0 {cloudai => src/cloudai}/schema/core/system.py | 0 {cloudai => src/cloudai}/schema/core/test.py | 0 {cloudai => src/cloudai}/schema/core/test_scenario.py | 0 {cloudai => src/cloudai}/schema/core/test_template.py | 0 {cloudai => src/cloudai}/schema/system/__init__.py | 0 {cloudai => src/cloudai}/schema/system/slurm/__init__.py | 0 {cloudai => src/cloudai}/schema/system/slurm/slurm_node.py | 0 {cloudai => src/cloudai}/schema/system/slurm/slurm_system.py | 0 .../cloudai}/schema/system/slurm/strategy/__init__.py | 0 .../schema/system/slurm/strategy/slurm_command_gen_strategy.py | 0 .../schema/system/slurm/strategy/slurm_install_strategy.py | 0 {cloudai => src/cloudai}/schema/system/standalone_system.py | 0 {cloudai => src/cloudai}/schema/test_template/__init__.py | 0 .../cloudai}/schema/test_template/chakra_replay/__init__.py | 0 .../schema/test_template/chakra_replay/grading_strategy.py | 0 .../test_template/chakra_replay/report_generation_strategy.py | 0 .../test_template/chakra_replay/slurm_command_gen_strategy.py | 0 .../test_template/chakra_replay/slurm_install_strategy.py | 0 .../cloudai}/schema/test_template/chakra_replay/template.py | 0 .../cloudai}/schema/test_template/common/__init__.py | 0 .../test_template/common/slurm_job_id_retrieval_strategy.py | 0 .../common/standalone_job_id_retrieval_strategy.py | 0 .../cloudai}/schema/test_template/jax_toolbox/__init__.py | 0 .../schema/test_template/jax_toolbox/grading_strategy.py | 0 .../test_template/jax_toolbox/report_generation_strategy.py | 0 .../test_template/jax_toolbox/slurm_command_gen_strategy.py | 0 .../schema/test_template/jax_toolbox/slurm_install_strategy.py | 0 .../cloudai}/schema/test_template/jax_toolbox/template.py | 0 .../cloudai}/schema/test_template/nccl_miner/__init__.py | 0 .../schema/test_template/nccl_miner/grading_strategy.py | 0 .../test_template/nccl_miner/report_generation_strategy.py | 0 .../test_template/nccl_miner/slurm_command_gen_strategy.py | 0 .../schema/test_template/nccl_miner/slurm_install_strategy.py | 0 .../cloudai}/schema/test_template/nccl_miner/template.py | 0 .../cloudai}/schema/test_template/nccl_test/__init__.py | 0 .../schema/test_template/nccl_test/grading_strategy.py | 0 .../test_template/nccl_test/report_generation_strategy.py | 0 .../test_template/nccl_test/slurm_command_gen_strategy.py | 0 .../schema/test_template/nccl_test/slurm_install_strategy.py | 0 .../cloudai}/schema/test_template/nccl_test/template.py | 0 .../cloudai}/schema/test_template/nemo_launcher/__init__.py | 0 .../schema/test_template/nemo_launcher/grading_strategy.py | 0 .../test_template/nemo_launcher/report_generation_strategy.py | 0 .../test_template/nemo_launcher/slurm_command_gen_strategy.py | 0 .../test_template/nemo_launcher/slurm_install_strategy.py | 0 .../nemo_launcher/slurm_job_id_retrieval_strategy.py | 0 .../cloudai}/schema/test_template/nemo_launcher/template.py | 0 .../cloudai}/schema/test_template/sleep/__init__.py | 0 .../cloudai}/schema/test_template/sleep/grading_strategy.py | 0 .../schema/test_template/sleep/report_generation_strategy.py | 0 .../test_template/sleep/standalone_command_gen_strategy.py | 0 .../schema/test_template/sleep/standalone_install_strategy.py | 0 .../cloudai}/schema/test_template/sleep/template.py | 0 .../cloudai}/schema/test_template/ucc_test/__init__.py | 0 .../cloudai}/schema/test_template/ucc_test/grading_strategy.py | 0 .../test_template/ucc_test/report_generation_strategy.py | 0 .../test_template/ucc_test/slurm_command_gen_strategy.py | 0 .../schema/test_template/ucc_test/slurm_install_strategy.py | 0 .../cloudai}/schema/test_template/ucc_test/template.py | 0 {cloudai => src/cloudai}/system_object_updater/__init__.py | 0 .../system_object_updater/base_system_object_updater.py | 0 .../system_object_updater/slurm_system_object_updater.py | 0 .../system_object_updater/standalone_system_object_updater.py | 0 .../cloudai}/system_object_updater/system_object_updater.py | 0 {cloudai => src/cloudai}/util/__init__.py | 0 {cloudai => src/cloudai}/util/command_shell.py | 0 116 files changed, 1 insertion(+), 3 deletions(-) rename {cloudai => src/cloudai}/__init__.py (100%) rename {cloudai => src/cloudai}/grader/__init__.py (100%) rename {cloudai => src/cloudai}/grader/grader.py (100%) rename {cloudai => src/cloudai}/installer/__init__.py (100%) rename {cloudai => src/cloudai}/installer/base_installer.py (100%) rename {cloudai => src/cloudai}/installer/installer.py (100%) rename {cloudai => src/cloudai}/installer/slurm_installer.py (100%) rename {cloudai => src/cloudai}/installer/standalone_installer.py (100%) rename {cloudai => src/cloudai}/parser/__init__.py (100%) rename {cloudai => src/cloudai}/parser/core/__init__.py (100%) rename {cloudai => src/cloudai}/parser/core/base_multi_file_parser.py (100%) rename {cloudai => src/cloudai}/parser/core/base_system_parser.py (100%) rename {cloudai => src/cloudai}/parser/core/parser.py (100%) rename {cloudai => src/cloudai}/parser/core/system_parser.py (100%) rename {cloudai => src/cloudai}/parser/core/test_parser.py (100%) rename {cloudai => src/cloudai}/parser/core/test_scenario_parser.py (100%) rename {cloudai => src/cloudai}/parser/core/test_template_parser.py (100%) rename {cloudai => src/cloudai}/parser/system_parser/__init__.py (100%) rename {cloudai => src/cloudai}/parser/system_parser/slurm_system_parser.py (100%) rename {cloudai => src/cloudai}/parser/system_parser/standalone_system_parser.py (100%) rename {cloudai => src/cloudai}/report_generator/__init__.py (100%) rename {cloudai => src/cloudai}/report_generator/report_generator.py (100%) rename {cloudai => src/cloudai}/report_generator/tool/__init__.py (100%) rename {cloudai => src/cloudai}/report_generator/tool/bokeh_report_tool.py (100%) rename {cloudai => src/cloudai}/report_generator/tool/report_tool_interface.py (100%) rename {cloudai => src/cloudai}/report_generator/tool/tensorboard_data_reader.py (100%) rename {cloudai => src/cloudai}/report_generator/util.py (100%) rename {cloudai => src/cloudai}/runner/__init__.py (100%) rename {cloudai => src/cloudai}/runner/core/__init__.py (100%) rename {cloudai => src/cloudai}/runner/core/base_job.py (100%) rename {cloudai => src/cloudai}/runner/core/base_runner.py (100%) rename {cloudai => src/cloudai}/runner/core/runner.py (100%) rename {cloudai => src/cloudai}/runner/slurm/__init__.py (100%) rename {cloudai => src/cloudai}/runner/slurm/slurm_job.py (100%) rename {cloudai => src/cloudai}/runner/slurm/slurm_runner.py (100%) rename {cloudai => src/cloudai}/runner/standalone/__init__.py (100%) rename {cloudai => src/cloudai}/runner/standalone/standalone_job.py (100%) rename {cloudai => src/cloudai}/runner/standalone/standalone_runner.py (100%) rename {cloudai => src/cloudai}/schema/__init__.py (100%) rename {cloudai => src/cloudai}/schema/core/__init__.py (100%) rename {cloudai => src/cloudai}/schema/core/strategy/__init__.py (100%) rename {cloudai => src/cloudai}/schema/core/strategy/command_gen_strategy.py (100%) rename {cloudai => src/cloudai}/schema/core/strategy/grading_strategy.py (100%) rename {cloudai => src/cloudai}/schema/core/strategy/install_strategy.py (100%) rename {cloudai => src/cloudai}/schema/core/strategy/job_id_retrieval_strategy.py (100%) rename {cloudai => src/cloudai}/schema/core/strategy/report_generation_strategy.py (100%) rename {cloudai => src/cloudai}/schema/core/strategy/strategy_registry.py (100%) rename {cloudai => src/cloudai}/schema/core/strategy/test_template_strategy.py (100%) rename {cloudai => src/cloudai}/schema/core/system.py (100%) rename {cloudai => src/cloudai}/schema/core/test.py (100%) rename {cloudai => src/cloudai}/schema/core/test_scenario.py (100%) rename {cloudai => src/cloudai}/schema/core/test_template.py (100%) rename {cloudai => src/cloudai}/schema/system/__init__.py (100%) rename {cloudai => src/cloudai}/schema/system/slurm/__init__.py (100%) rename {cloudai => src/cloudai}/schema/system/slurm/slurm_node.py (100%) rename {cloudai => src/cloudai}/schema/system/slurm/slurm_system.py (100%) rename {cloudai => src/cloudai}/schema/system/slurm/strategy/__init__.py (100%) rename {cloudai => src/cloudai}/schema/system/slurm/strategy/slurm_command_gen_strategy.py (100%) rename {cloudai => src/cloudai}/schema/system/slurm/strategy/slurm_install_strategy.py (100%) rename {cloudai => src/cloudai}/schema/system/standalone_system.py (100%) rename {cloudai => src/cloudai}/schema/test_template/__init__.py (100%) rename {cloudai => src/cloudai}/schema/test_template/chakra_replay/__init__.py (100%) rename {cloudai => src/cloudai}/schema/test_template/chakra_replay/grading_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/chakra_replay/report_generation_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/chakra_replay/slurm_command_gen_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/chakra_replay/slurm_install_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/chakra_replay/template.py (100%) rename {cloudai => src/cloudai}/schema/test_template/common/__init__.py (100%) rename {cloudai => src/cloudai}/schema/test_template/common/slurm_job_id_retrieval_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/common/standalone_job_id_retrieval_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/jax_toolbox/__init__.py (100%) rename {cloudai => src/cloudai}/schema/test_template/jax_toolbox/grading_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/jax_toolbox/report_generation_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/jax_toolbox/slurm_command_gen_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/jax_toolbox/slurm_install_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/jax_toolbox/template.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nccl_miner/__init__.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nccl_miner/grading_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nccl_miner/report_generation_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nccl_miner/slurm_command_gen_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nccl_miner/slurm_install_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nccl_miner/template.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nccl_test/__init__.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nccl_test/grading_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nccl_test/report_generation_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nccl_test/slurm_command_gen_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nccl_test/slurm_install_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nccl_test/template.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nemo_launcher/__init__.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nemo_launcher/grading_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nemo_launcher/report_generation_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nemo_launcher/slurm_command_gen_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nemo_launcher/slurm_install_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nemo_launcher/slurm_job_id_retrieval_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/nemo_launcher/template.py (100%) rename {cloudai => src/cloudai}/schema/test_template/sleep/__init__.py (100%) rename {cloudai => src/cloudai}/schema/test_template/sleep/grading_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/sleep/report_generation_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/sleep/standalone_command_gen_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/sleep/standalone_install_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/sleep/template.py (100%) rename {cloudai => src/cloudai}/schema/test_template/ucc_test/__init__.py (100%) rename {cloudai => src/cloudai}/schema/test_template/ucc_test/grading_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/ucc_test/report_generation_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/ucc_test/slurm_command_gen_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/ucc_test/slurm_install_strategy.py (100%) rename {cloudai => src/cloudai}/schema/test_template/ucc_test/template.py (100%) rename {cloudai => src/cloudai}/system_object_updater/__init__.py (100%) rename {cloudai => src/cloudai}/system_object_updater/base_system_object_updater.py (100%) rename {cloudai => src/cloudai}/system_object_updater/slurm_system_object_updater.py (100%) rename {cloudai => src/cloudai}/system_object_updater/standalone_system_object_updater.py (100%) rename {cloudai => src/cloudai}/system_object_updater/system_object_updater.py (100%) rename {cloudai => src/cloudai}/util/__init__.py (100%) rename {cloudai => src/cloudai}/util/command_shell.py (100%) diff --git a/pyproject.toml b/pyproject.toml index 8389b3944..9e1342fe5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,9 +2,6 @@ name = "cloudai" version = "0.6" -[tool.setuptools.packages.find] -where = ["cloudai"] - [tool.ruff] target-version = "py39" line-length = 120 diff --git a/requirements-dev.txt b/requirements-dev.txt index b7d10377b..fe537ca2e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ pytest==8.1.1 ruff==0.3.7 pandas-stubs==2.2.* pyright==1.1.359 +build==1.2.* diff --git a/cloudai/__init__.py b/src/cloudai/__init__.py similarity index 100% rename from cloudai/__init__.py rename to src/cloudai/__init__.py diff --git a/cloudai/grader/__init__.py b/src/cloudai/grader/__init__.py similarity index 100% rename from cloudai/grader/__init__.py rename to src/cloudai/grader/__init__.py diff --git a/cloudai/grader/grader.py b/src/cloudai/grader/grader.py similarity index 100% rename from cloudai/grader/grader.py rename to src/cloudai/grader/grader.py diff --git a/cloudai/installer/__init__.py b/src/cloudai/installer/__init__.py similarity index 100% rename from cloudai/installer/__init__.py rename to src/cloudai/installer/__init__.py diff --git a/cloudai/installer/base_installer.py b/src/cloudai/installer/base_installer.py similarity index 100% rename from cloudai/installer/base_installer.py rename to src/cloudai/installer/base_installer.py diff --git a/cloudai/installer/installer.py b/src/cloudai/installer/installer.py similarity index 100% rename from cloudai/installer/installer.py rename to src/cloudai/installer/installer.py diff --git a/cloudai/installer/slurm_installer.py b/src/cloudai/installer/slurm_installer.py similarity index 100% rename from cloudai/installer/slurm_installer.py rename to src/cloudai/installer/slurm_installer.py diff --git a/cloudai/installer/standalone_installer.py b/src/cloudai/installer/standalone_installer.py similarity index 100% rename from cloudai/installer/standalone_installer.py rename to src/cloudai/installer/standalone_installer.py diff --git a/cloudai/parser/__init__.py b/src/cloudai/parser/__init__.py similarity index 100% rename from cloudai/parser/__init__.py rename to src/cloudai/parser/__init__.py diff --git a/cloudai/parser/core/__init__.py b/src/cloudai/parser/core/__init__.py similarity index 100% rename from cloudai/parser/core/__init__.py rename to src/cloudai/parser/core/__init__.py diff --git a/cloudai/parser/core/base_multi_file_parser.py b/src/cloudai/parser/core/base_multi_file_parser.py similarity index 100% rename from cloudai/parser/core/base_multi_file_parser.py rename to src/cloudai/parser/core/base_multi_file_parser.py diff --git a/cloudai/parser/core/base_system_parser.py b/src/cloudai/parser/core/base_system_parser.py similarity index 100% rename from cloudai/parser/core/base_system_parser.py rename to src/cloudai/parser/core/base_system_parser.py diff --git a/cloudai/parser/core/parser.py b/src/cloudai/parser/core/parser.py similarity index 100% rename from cloudai/parser/core/parser.py rename to src/cloudai/parser/core/parser.py diff --git a/cloudai/parser/core/system_parser.py b/src/cloudai/parser/core/system_parser.py similarity index 100% rename from cloudai/parser/core/system_parser.py rename to src/cloudai/parser/core/system_parser.py diff --git a/cloudai/parser/core/test_parser.py b/src/cloudai/parser/core/test_parser.py similarity index 100% rename from cloudai/parser/core/test_parser.py rename to src/cloudai/parser/core/test_parser.py diff --git a/cloudai/parser/core/test_scenario_parser.py b/src/cloudai/parser/core/test_scenario_parser.py similarity index 100% rename from cloudai/parser/core/test_scenario_parser.py rename to src/cloudai/parser/core/test_scenario_parser.py diff --git a/cloudai/parser/core/test_template_parser.py b/src/cloudai/parser/core/test_template_parser.py similarity index 100% rename from cloudai/parser/core/test_template_parser.py rename to src/cloudai/parser/core/test_template_parser.py diff --git a/cloudai/parser/system_parser/__init__.py b/src/cloudai/parser/system_parser/__init__.py similarity index 100% rename from cloudai/parser/system_parser/__init__.py rename to src/cloudai/parser/system_parser/__init__.py diff --git a/cloudai/parser/system_parser/slurm_system_parser.py b/src/cloudai/parser/system_parser/slurm_system_parser.py similarity index 100% rename from cloudai/parser/system_parser/slurm_system_parser.py rename to src/cloudai/parser/system_parser/slurm_system_parser.py diff --git a/cloudai/parser/system_parser/standalone_system_parser.py b/src/cloudai/parser/system_parser/standalone_system_parser.py similarity index 100% rename from cloudai/parser/system_parser/standalone_system_parser.py rename to src/cloudai/parser/system_parser/standalone_system_parser.py diff --git a/cloudai/report_generator/__init__.py b/src/cloudai/report_generator/__init__.py similarity index 100% rename from cloudai/report_generator/__init__.py rename to src/cloudai/report_generator/__init__.py diff --git a/cloudai/report_generator/report_generator.py b/src/cloudai/report_generator/report_generator.py similarity index 100% rename from cloudai/report_generator/report_generator.py rename to src/cloudai/report_generator/report_generator.py diff --git a/cloudai/report_generator/tool/__init__.py b/src/cloudai/report_generator/tool/__init__.py similarity index 100% rename from cloudai/report_generator/tool/__init__.py rename to src/cloudai/report_generator/tool/__init__.py diff --git a/cloudai/report_generator/tool/bokeh_report_tool.py b/src/cloudai/report_generator/tool/bokeh_report_tool.py similarity index 100% rename from cloudai/report_generator/tool/bokeh_report_tool.py rename to src/cloudai/report_generator/tool/bokeh_report_tool.py diff --git a/cloudai/report_generator/tool/report_tool_interface.py b/src/cloudai/report_generator/tool/report_tool_interface.py similarity index 100% rename from cloudai/report_generator/tool/report_tool_interface.py rename to src/cloudai/report_generator/tool/report_tool_interface.py diff --git a/cloudai/report_generator/tool/tensorboard_data_reader.py b/src/cloudai/report_generator/tool/tensorboard_data_reader.py similarity index 100% rename from cloudai/report_generator/tool/tensorboard_data_reader.py rename to src/cloudai/report_generator/tool/tensorboard_data_reader.py diff --git a/cloudai/report_generator/util.py b/src/cloudai/report_generator/util.py similarity index 100% rename from cloudai/report_generator/util.py rename to src/cloudai/report_generator/util.py diff --git a/cloudai/runner/__init__.py b/src/cloudai/runner/__init__.py similarity index 100% rename from cloudai/runner/__init__.py rename to src/cloudai/runner/__init__.py diff --git a/cloudai/runner/core/__init__.py b/src/cloudai/runner/core/__init__.py similarity index 100% rename from cloudai/runner/core/__init__.py rename to src/cloudai/runner/core/__init__.py diff --git a/cloudai/runner/core/base_job.py b/src/cloudai/runner/core/base_job.py similarity index 100% rename from cloudai/runner/core/base_job.py rename to src/cloudai/runner/core/base_job.py diff --git a/cloudai/runner/core/base_runner.py b/src/cloudai/runner/core/base_runner.py similarity index 100% rename from cloudai/runner/core/base_runner.py rename to src/cloudai/runner/core/base_runner.py diff --git a/cloudai/runner/core/runner.py b/src/cloudai/runner/core/runner.py similarity index 100% rename from cloudai/runner/core/runner.py rename to src/cloudai/runner/core/runner.py diff --git a/cloudai/runner/slurm/__init__.py b/src/cloudai/runner/slurm/__init__.py similarity index 100% rename from cloudai/runner/slurm/__init__.py rename to src/cloudai/runner/slurm/__init__.py diff --git a/cloudai/runner/slurm/slurm_job.py b/src/cloudai/runner/slurm/slurm_job.py similarity index 100% rename from cloudai/runner/slurm/slurm_job.py rename to src/cloudai/runner/slurm/slurm_job.py diff --git a/cloudai/runner/slurm/slurm_runner.py b/src/cloudai/runner/slurm/slurm_runner.py similarity index 100% rename from cloudai/runner/slurm/slurm_runner.py rename to src/cloudai/runner/slurm/slurm_runner.py diff --git a/cloudai/runner/standalone/__init__.py b/src/cloudai/runner/standalone/__init__.py similarity index 100% rename from cloudai/runner/standalone/__init__.py rename to src/cloudai/runner/standalone/__init__.py diff --git a/cloudai/runner/standalone/standalone_job.py b/src/cloudai/runner/standalone/standalone_job.py similarity index 100% rename from cloudai/runner/standalone/standalone_job.py rename to src/cloudai/runner/standalone/standalone_job.py diff --git a/cloudai/runner/standalone/standalone_runner.py b/src/cloudai/runner/standalone/standalone_runner.py similarity index 100% rename from cloudai/runner/standalone/standalone_runner.py rename to src/cloudai/runner/standalone/standalone_runner.py diff --git a/cloudai/schema/__init__.py b/src/cloudai/schema/__init__.py similarity index 100% rename from cloudai/schema/__init__.py rename to src/cloudai/schema/__init__.py diff --git a/cloudai/schema/core/__init__.py b/src/cloudai/schema/core/__init__.py similarity index 100% rename from cloudai/schema/core/__init__.py rename to src/cloudai/schema/core/__init__.py diff --git a/cloudai/schema/core/strategy/__init__.py b/src/cloudai/schema/core/strategy/__init__.py similarity index 100% rename from cloudai/schema/core/strategy/__init__.py rename to src/cloudai/schema/core/strategy/__init__.py diff --git a/cloudai/schema/core/strategy/command_gen_strategy.py b/src/cloudai/schema/core/strategy/command_gen_strategy.py similarity index 100% rename from cloudai/schema/core/strategy/command_gen_strategy.py rename to src/cloudai/schema/core/strategy/command_gen_strategy.py diff --git a/cloudai/schema/core/strategy/grading_strategy.py b/src/cloudai/schema/core/strategy/grading_strategy.py similarity index 100% rename from cloudai/schema/core/strategy/grading_strategy.py rename to src/cloudai/schema/core/strategy/grading_strategy.py diff --git a/cloudai/schema/core/strategy/install_strategy.py b/src/cloudai/schema/core/strategy/install_strategy.py similarity index 100% rename from cloudai/schema/core/strategy/install_strategy.py rename to src/cloudai/schema/core/strategy/install_strategy.py diff --git a/cloudai/schema/core/strategy/job_id_retrieval_strategy.py b/src/cloudai/schema/core/strategy/job_id_retrieval_strategy.py similarity index 100% rename from cloudai/schema/core/strategy/job_id_retrieval_strategy.py rename to src/cloudai/schema/core/strategy/job_id_retrieval_strategy.py diff --git a/cloudai/schema/core/strategy/report_generation_strategy.py b/src/cloudai/schema/core/strategy/report_generation_strategy.py similarity index 100% rename from cloudai/schema/core/strategy/report_generation_strategy.py rename to src/cloudai/schema/core/strategy/report_generation_strategy.py diff --git a/cloudai/schema/core/strategy/strategy_registry.py b/src/cloudai/schema/core/strategy/strategy_registry.py similarity index 100% rename from cloudai/schema/core/strategy/strategy_registry.py rename to src/cloudai/schema/core/strategy/strategy_registry.py diff --git a/cloudai/schema/core/strategy/test_template_strategy.py b/src/cloudai/schema/core/strategy/test_template_strategy.py similarity index 100% rename from cloudai/schema/core/strategy/test_template_strategy.py rename to src/cloudai/schema/core/strategy/test_template_strategy.py diff --git a/cloudai/schema/core/system.py b/src/cloudai/schema/core/system.py similarity index 100% rename from cloudai/schema/core/system.py rename to src/cloudai/schema/core/system.py diff --git a/cloudai/schema/core/test.py b/src/cloudai/schema/core/test.py similarity index 100% rename from cloudai/schema/core/test.py rename to src/cloudai/schema/core/test.py diff --git a/cloudai/schema/core/test_scenario.py b/src/cloudai/schema/core/test_scenario.py similarity index 100% rename from cloudai/schema/core/test_scenario.py rename to src/cloudai/schema/core/test_scenario.py diff --git a/cloudai/schema/core/test_template.py b/src/cloudai/schema/core/test_template.py similarity index 100% rename from cloudai/schema/core/test_template.py rename to src/cloudai/schema/core/test_template.py diff --git a/cloudai/schema/system/__init__.py b/src/cloudai/schema/system/__init__.py similarity index 100% rename from cloudai/schema/system/__init__.py rename to src/cloudai/schema/system/__init__.py diff --git a/cloudai/schema/system/slurm/__init__.py b/src/cloudai/schema/system/slurm/__init__.py similarity index 100% rename from cloudai/schema/system/slurm/__init__.py rename to src/cloudai/schema/system/slurm/__init__.py diff --git a/cloudai/schema/system/slurm/slurm_node.py b/src/cloudai/schema/system/slurm/slurm_node.py similarity index 100% rename from cloudai/schema/system/slurm/slurm_node.py rename to src/cloudai/schema/system/slurm/slurm_node.py diff --git a/cloudai/schema/system/slurm/slurm_system.py b/src/cloudai/schema/system/slurm/slurm_system.py similarity index 100% rename from cloudai/schema/system/slurm/slurm_system.py rename to src/cloudai/schema/system/slurm/slurm_system.py diff --git a/cloudai/schema/system/slurm/strategy/__init__.py b/src/cloudai/schema/system/slurm/strategy/__init__.py similarity index 100% rename from cloudai/schema/system/slurm/strategy/__init__.py rename to src/cloudai/schema/system/slurm/strategy/__init__.py diff --git a/cloudai/schema/system/slurm/strategy/slurm_command_gen_strategy.py b/src/cloudai/schema/system/slurm/strategy/slurm_command_gen_strategy.py similarity index 100% rename from cloudai/schema/system/slurm/strategy/slurm_command_gen_strategy.py rename to src/cloudai/schema/system/slurm/strategy/slurm_command_gen_strategy.py diff --git a/cloudai/schema/system/slurm/strategy/slurm_install_strategy.py b/src/cloudai/schema/system/slurm/strategy/slurm_install_strategy.py similarity index 100% rename from cloudai/schema/system/slurm/strategy/slurm_install_strategy.py rename to src/cloudai/schema/system/slurm/strategy/slurm_install_strategy.py diff --git a/cloudai/schema/system/standalone_system.py b/src/cloudai/schema/system/standalone_system.py similarity index 100% rename from cloudai/schema/system/standalone_system.py rename to src/cloudai/schema/system/standalone_system.py diff --git a/cloudai/schema/test_template/__init__.py b/src/cloudai/schema/test_template/__init__.py similarity index 100% rename from cloudai/schema/test_template/__init__.py rename to src/cloudai/schema/test_template/__init__.py diff --git a/cloudai/schema/test_template/chakra_replay/__init__.py b/src/cloudai/schema/test_template/chakra_replay/__init__.py similarity index 100% rename from cloudai/schema/test_template/chakra_replay/__init__.py rename to src/cloudai/schema/test_template/chakra_replay/__init__.py diff --git a/cloudai/schema/test_template/chakra_replay/grading_strategy.py b/src/cloudai/schema/test_template/chakra_replay/grading_strategy.py similarity index 100% rename from cloudai/schema/test_template/chakra_replay/grading_strategy.py rename to src/cloudai/schema/test_template/chakra_replay/grading_strategy.py diff --git a/cloudai/schema/test_template/chakra_replay/report_generation_strategy.py b/src/cloudai/schema/test_template/chakra_replay/report_generation_strategy.py similarity index 100% rename from cloudai/schema/test_template/chakra_replay/report_generation_strategy.py rename to src/cloudai/schema/test_template/chakra_replay/report_generation_strategy.py diff --git a/cloudai/schema/test_template/chakra_replay/slurm_command_gen_strategy.py b/src/cloudai/schema/test_template/chakra_replay/slurm_command_gen_strategy.py similarity index 100% rename from cloudai/schema/test_template/chakra_replay/slurm_command_gen_strategy.py rename to src/cloudai/schema/test_template/chakra_replay/slurm_command_gen_strategy.py diff --git a/cloudai/schema/test_template/chakra_replay/slurm_install_strategy.py b/src/cloudai/schema/test_template/chakra_replay/slurm_install_strategy.py similarity index 100% rename from cloudai/schema/test_template/chakra_replay/slurm_install_strategy.py rename to src/cloudai/schema/test_template/chakra_replay/slurm_install_strategy.py diff --git a/cloudai/schema/test_template/chakra_replay/template.py b/src/cloudai/schema/test_template/chakra_replay/template.py similarity index 100% rename from cloudai/schema/test_template/chakra_replay/template.py rename to src/cloudai/schema/test_template/chakra_replay/template.py diff --git a/cloudai/schema/test_template/common/__init__.py b/src/cloudai/schema/test_template/common/__init__.py similarity index 100% rename from cloudai/schema/test_template/common/__init__.py rename to src/cloudai/schema/test_template/common/__init__.py diff --git a/cloudai/schema/test_template/common/slurm_job_id_retrieval_strategy.py b/src/cloudai/schema/test_template/common/slurm_job_id_retrieval_strategy.py similarity index 100% rename from cloudai/schema/test_template/common/slurm_job_id_retrieval_strategy.py rename to src/cloudai/schema/test_template/common/slurm_job_id_retrieval_strategy.py diff --git a/cloudai/schema/test_template/common/standalone_job_id_retrieval_strategy.py b/src/cloudai/schema/test_template/common/standalone_job_id_retrieval_strategy.py similarity index 100% rename from cloudai/schema/test_template/common/standalone_job_id_retrieval_strategy.py rename to src/cloudai/schema/test_template/common/standalone_job_id_retrieval_strategy.py diff --git a/cloudai/schema/test_template/jax_toolbox/__init__.py b/src/cloudai/schema/test_template/jax_toolbox/__init__.py similarity index 100% rename from cloudai/schema/test_template/jax_toolbox/__init__.py rename to src/cloudai/schema/test_template/jax_toolbox/__init__.py diff --git a/cloudai/schema/test_template/jax_toolbox/grading_strategy.py b/src/cloudai/schema/test_template/jax_toolbox/grading_strategy.py similarity index 100% rename from cloudai/schema/test_template/jax_toolbox/grading_strategy.py rename to src/cloudai/schema/test_template/jax_toolbox/grading_strategy.py diff --git a/cloudai/schema/test_template/jax_toolbox/report_generation_strategy.py b/src/cloudai/schema/test_template/jax_toolbox/report_generation_strategy.py similarity index 100% rename from cloudai/schema/test_template/jax_toolbox/report_generation_strategy.py rename to src/cloudai/schema/test_template/jax_toolbox/report_generation_strategy.py diff --git a/cloudai/schema/test_template/jax_toolbox/slurm_command_gen_strategy.py b/src/cloudai/schema/test_template/jax_toolbox/slurm_command_gen_strategy.py similarity index 100% rename from cloudai/schema/test_template/jax_toolbox/slurm_command_gen_strategy.py rename to src/cloudai/schema/test_template/jax_toolbox/slurm_command_gen_strategy.py diff --git a/cloudai/schema/test_template/jax_toolbox/slurm_install_strategy.py b/src/cloudai/schema/test_template/jax_toolbox/slurm_install_strategy.py similarity index 100% rename from cloudai/schema/test_template/jax_toolbox/slurm_install_strategy.py rename to src/cloudai/schema/test_template/jax_toolbox/slurm_install_strategy.py diff --git a/cloudai/schema/test_template/jax_toolbox/template.py b/src/cloudai/schema/test_template/jax_toolbox/template.py similarity index 100% rename from cloudai/schema/test_template/jax_toolbox/template.py rename to src/cloudai/schema/test_template/jax_toolbox/template.py diff --git a/cloudai/schema/test_template/nccl_miner/__init__.py b/src/cloudai/schema/test_template/nccl_miner/__init__.py similarity index 100% rename from cloudai/schema/test_template/nccl_miner/__init__.py rename to src/cloudai/schema/test_template/nccl_miner/__init__.py diff --git a/cloudai/schema/test_template/nccl_miner/grading_strategy.py b/src/cloudai/schema/test_template/nccl_miner/grading_strategy.py similarity index 100% rename from cloudai/schema/test_template/nccl_miner/grading_strategy.py rename to src/cloudai/schema/test_template/nccl_miner/grading_strategy.py diff --git a/cloudai/schema/test_template/nccl_miner/report_generation_strategy.py b/src/cloudai/schema/test_template/nccl_miner/report_generation_strategy.py similarity index 100% rename from cloudai/schema/test_template/nccl_miner/report_generation_strategy.py rename to src/cloudai/schema/test_template/nccl_miner/report_generation_strategy.py diff --git a/cloudai/schema/test_template/nccl_miner/slurm_command_gen_strategy.py b/src/cloudai/schema/test_template/nccl_miner/slurm_command_gen_strategy.py similarity index 100% rename from cloudai/schema/test_template/nccl_miner/slurm_command_gen_strategy.py rename to src/cloudai/schema/test_template/nccl_miner/slurm_command_gen_strategy.py diff --git a/cloudai/schema/test_template/nccl_miner/slurm_install_strategy.py b/src/cloudai/schema/test_template/nccl_miner/slurm_install_strategy.py similarity index 100% rename from cloudai/schema/test_template/nccl_miner/slurm_install_strategy.py rename to src/cloudai/schema/test_template/nccl_miner/slurm_install_strategy.py diff --git a/cloudai/schema/test_template/nccl_miner/template.py b/src/cloudai/schema/test_template/nccl_miner/template.py similarity index 100% rename from cloudai/schema/test_template/nccl_miner/template.py rename to src/cloudai/schema/test_template/nccl_miner/template.py diff --git a/cloudai/schema/test_template/nccl_test/__init__.py b/src/cloudai/schema/test_template/nccl_test/__init__.py similarity index 100% rename from cloudai/schema/test_template/nccl_test/__init__.py rename to src/cloudai/schema/test_template/nccl_test/__init__.py diff --git a/cloudai/schema/test_template/nccl_test/grading_strategy.py b/src/cloudai/schema/test_template/nccl_test/grading_strategy.py similarity index 100% rename from cloudai/schema/test_template/nccl_test/grading_strategy.py rename to src/cloudai/schema/test_template/nccl_test/grading_strategy.py diff --git a/cloudai/schema/test_template/nccl_test/report_generation_strategy.py b/src/cloudai/schema/test_template/nccl_test/report_generation_strategy.py similarity index 100% rename from cloudai/schema/test_template/nccl_test/report_generation_strategy.py rename to src/cloudai/schema/test_template/nccl_test/report_generation_strategy.py diff --git a/cloudai/schema/test_template/nccl_test/slurm_command_gen_strategy.py b/src/cloudai/schema/test_template/nccl_test/slurm_command_gen_strategy.py similarity index 100% rename from cloudai/schema/test_template/nccl_test/slurm_command_gen_strategy.py rename to src/cloudai/schema/test_template/nccl_test/slurm_command_gen_strategy.py diff --git a/cloudai/schema/test_template/nccl_test/slurm_install_strategy.py b/src/cloudai/schema/test_template/nccl_test/slurm_install_strategy.py similarity index 100% rename from cloudai/schema/test_template/nccl_test/slurm_install_strategy.py rename to src/cloudai/schema/test_template/nccl_test/slurm_install_strategy.py diff --git a/cloudai/schema/test_template/nccl_test/template.py b/src/cloudai/schema/test_template/nccl_test/template.py similarity index 100% rename from cloudai/schema/test_template/nccl_test/template.py rename to src/cloudai/schema/test_template/nccl_test/template.py diff --git a/cloudai/schema/test_template/nemo_launcher/__init__.py b/src/cloudai/schema/test_template/nemo_launcher/__init__.py similarity index 100% rename from cloudai/schema/test_template/nemo_launcher/__init__.py rename to src/cloudai/schema/test_template/nemo_launcher/__init__.py diff --git a/cloudai/schema/test_template/nemo_launcher/grading_strategy.py b/src/cloudai/schema/test_template/nemo_launcher/grading_strategy.py similarity index 100% rename from cloudai/schema/test_template/nemo_launcher/grading_strategy.py rename to src/cloudai/schema/test_template/nemo_launcher/grading_strategy.py diff --git a/cloudai/schema/test_template/nemo_launcher/report_generation_strategy.py b/src/cloudai/schema/test_template/nemo_launcher/report_generation_strategy.py similarity index 100% rename from cloudai/schema/test_template/nemo_launcher/report_generation_strategy.py rename to src/cloudai/schema/test_template/nemo_launcher/report_generation_strategy.py diff --git a/cloudai/schema/test_template/nemo_launcher/slurm_command_gen_strategy.py b/src/cloudai/schema/test_template/nemo_launcher/slurm_command_gen_strategy.py similarity index 100% rename from cloudai/schema/test_template/nemo_launcher/slurm_command_gen_strategy.py rename to src/cloudai/schema/test_template/nemo_launcher/slurm_command_gen_strategy.py diff --git a/cloudai/schema/test_template/nemo_launcher/slurm_install_strategy.py b/src/cloudai/schema/test_template/nemo_launcher/slurm_install_strategy.py similarity index 100% rename from cloudai/schema/test_template/nemo_launcher/slurm_install_strategy.py rename to src/cloudai/schema/test_template/nemo_launcher/slurm_install_strategy.py diff --git a/cloudai/schema/test_template/nemo_launcher/slurm_job_id_retrieval_strategy.py b/src/cloudai/schema/test_template/nemo_launcher/slurm_job_id_retrieval_strategy.py similarity index 100% rename from cloudai/schema/test_template/nemo_launcher/slurm_job_id_retrieval_strategy.py rename to src/cloudai/schema/test_template/nemo_launcher/slurm_job_id_retrieval_strategy.py diff --git a/cloudai/schema/test_template/nemo_launcher/template.py b/src/cloudai/schema/test_template/nemo_launcher/template.py similarity index 100% rename from cloudai/schema/test_template/nemo_launcher/template.py rename to src/cloudai/schema/test_template/nemo_launcher/template.py diff --git a/cloudai/schema/test_template/sleep/__init__.py b/src/cloudai/schema/test_template/sleep/__init__.py similarity index 100% rename from cloudai/schema/test_template/sleep/__init__.py rename to src/cloudai/schema/test_template/sleep/__init__.py diff --git a/cloudai/schema/test_template/sleep/grading_strategy.py b/src/cloudai/schema/test_template/sleep/grading_strategy.py similarity index 100% rename from cloudai/schema/test_template/sleep/grading_strategy.py rename to src/cloudai/schema/test_template/sleep/grading_strategy.py diff --git a/cloudai/schema/test_template/sleep/report_generation_strategy.py b/src/cloudai/schema/test_template/sleep/report_generation_strategy.py similarity index 100% rename from cloudai/schema/test_template/sleep/report_generation_strategy.py rename to src/cloudai/schema/test_template/sleep/report_generation_strategy.py diff --git a/cloudai/schema/test_template/sleep/standalone_command_gen_strategy.py b/src/cloudai/schema/test_template/sleep/standalone_command_gen_strategy.py similarity index 100% rename from cloudai/schema/test_template/sleep/standalone_command_gen_strategy.py rename to src/cloudai/schema/test_template/sleep/standalone_command_gen_strategy.py diff --git a/cloudai/schema/test_template/sleep/standalone_install_strategy.py b/src/cloudai/schema/test_template/sleep/standalone_install_strategy.py similarity index 100% rename from cloudai/schema/test_template/sleep/standalone_install_strategy.py rename to src/cloudai/schema/test_template/sleep/standalone_install_strategy.py diff --git a/cloudai/schema/test_template/sleep/template.py b/src/cloudai/schema/test_template/sleep/template.py similarity index 100% rename from cloudai/schema/test_template/sleep/template.py rename to src/cloudai/schema/test_template/sleep/template.py diff --git a/cloudai/schema/test_template/ucc_test/__init__.py b/src/cloudai/schema/test_template/ucc_test/__init__.py similarity index 100% rename from cloudai/schema/test_template/ucc_test/__init__.py rename to src/cloudai/schema/test_template/ucc_test/__init__.py diff --git a/cloudai/schema/test_template/ucc_test/grading_strategy.py b/src/cloudai/schema/test_template/ucc_test/grading_strategy.py similarity index 100% rename from cloudai/schema/test_template/ucc_test/grading_strategy.py rename to src/cloudai/schema/test_template/ucc_test/grading_strategy.py diff --git a/cloudai/schema/test_template/ucc_test/report_generation_strategy.py b/src/cloudai/schema/test_template/ucc_test/report_generation_strategy.py similarity index 100% rename from cloudai/schema/test_template/ucc_test/report_generation_strategy.py rename to src/cloudai/schema/test_template/ucc_test/report_generation_strategy.py diff --git a/cloudai/schema/test_template/ucc_test/slurm_command_gen_strategy.py b/src/cloudai/schema/test_template/ucc_test/slurm_command_gen_strategy.py similarity index 100% rename from cloudai/schema/test_template/ucc_test/slurm_command_gen_strategy.py rename to src/cloudai/schema/test_template/ucc_test/slurm_command_gen_strategy.py diff --git a/cloudai/schema/test_template/ucc_test/slurm_install_strategy.py b/src/cloudai/schema/test_template/ucc_test/slurm_install_strategy.py similarity index 100% rename from cloudai/schema/test_template/ucc_test/slurm_install_strategy.py rename to src/cloudai/schema/test_template/ucc_test/slurm_install_strategy.py diff --git a/cloudai/schema/test_template/ucc_test/template.py b/src/cloudai/schema/test_template/ucc_test/template.py similarity index 100% rename from cloudai/schema/test_template/ucc_test/template.py rename to src/cloudai/schema/test_template/ucc_test/template.py diff --git a/cloudai/system_object_updater/__init__.py b/src/cloudai/system_object_updater/__init__.py similarity index 100% rename from cloudai/system_object_updater/__init__.py rename to src/cloudai/system_object_updater/__init__.py diff --git a/cloudai/system_object_updater/base_system_object_updater.py b/src/cloudai/system_object_updater/base_system_object_updater.py similarity index 100% rename from cloudai/system_object_updater/base_system_object_updater.py rename to src/cloudai/system_object_updater/base_system_object_updater.py diff --git a/cloudai/system_object_updater/slurm_system_object_updater.py b/src/cloudai/system_object_updater/slurm_system_object_updater.py similarity index 100% rename from cloudai/system_object_updater/slurm_system_object_updater.py rename to src/cloudai/system_object_updater/slurm_system_object_updater.py diff --git a/cloudai/system_object_updater/standalone_system_object_updater.py b/src/cloudai/system_object_updater/standalone_system_object_updater.py similarity index 100% rename from cloudai/system_object_updater/standalone_system_object_updater.py rename to src/cloudai/system_object_updater/standalone_system_object_updater.py diff --git a/cloudai/system_object_updater/system_object_updater.py b/src/cloudai/system_object_updater/system_object_updater.py similarity index 100% rename from cloudai/system_object_updater/system_object_updater.py rename to src/cloudai/system_object_updater/system_object_updater.py diff --git a/cloudai/util/__init__.py b/src/cloudai/util/__init__.py similarity index 100% rename from cloudai/util/__init__.py rename to src/cloudai/util/__init__.py diff --git a/cloudai/util/command_shell.py b/src/cloudai/util/command_shell.py similarity index 100% rename from cloudai/util/command_shell.py rename to src/cloudai/util/command_shell.py From f764172504f7d0a09d297e04c0aaf0813d30c658 Mon Sep 17 00:00:00 2001 From: Taekyung Heo <7621438+TaekyungHeo@users.noreply.github.com> Date: Tue, 14 May 2024 07:12:28 -0400 Subject: [PATCH 03/11] Add ISSUE_TEMPLATE and PULL_REQUEST_TEMPLATE --- .github/ISSUE_TEMPLATE/bug_report.md | 20 ++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 13 +++++++++++++ .github/ISSUE_TEMPLATE/general_question.md | 9 +++++++++ .github/PULL_REQUEST_TEMPLATE.md | 13 +++++++++++++ 4 files changed, 55 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/general_question.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..10ae23d50 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,20 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' +--- + +## Describe the Bug +> A clear and concise description of what the bug is. + +## Steps to Reproduce +> Steps to reproduce the behavior. +> Please include the version information where the bug was observed. + +## Expected Behavior +> A clear and concise description of what you expected to happen. + +## Screenshots +> If applicable, add screenshots to help explain your problem. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..a2a493d1d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,13 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' +--- + +## Problem Related to the Feature +> A clear and concise description of what the problem is. + +## Proposed Solution +> A clear and concise description of what you want to happen. diff --git a/.github/ISSUE_TEMPLATE/general_question.md b/.github/ISSUE_TEMPLATE/general_question.md new file mode 100644 index 000000000..0dc55f0f8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/general_question.md @@ -0,0 +1,9 @@ +--- +name: General question +about: Ask a question or seek clarification about the project +title: '' +labels: 'question' +assignees: '' +--- + +> Please provide a detailed description of your question or the information you seek. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..4b89184e2 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,13 @@ +## Summary +Provide a concise summary of the changes introduced by this pull request. Detail the purpose and scope of the changes, referencing any relevant issues or discussions. Explain how these changes address the problem or improve the project. + +## Test Plan +In this section, describe the testing you have performed to verify the changes. Include: +- A clear description of the testing environment. +- The steps you followed to test the new features or bug fixes. +- Any specific commands used during testing, along with their outputs. +- A description of the results and observations from your testing. +This information is crucial for reviewers to understand how the changes have been validated. + +## Additional Notes +Include any other notes or comments about the pull request here. This can include challenges faced, future considerations, or context that reviewers might find helpful. From b9b88ade5ae638b7e948f32388089e414f54cd42 Mon Sep 17 00:00:00 2001 From: Andrey Maslennikov Date: Tue, 14 May 2024 13:24:09 +0200 Subject: [PATCH 04/11] Run CLI via python -m cloudai --- README.md | 52 ++++++++++++++++++++++++------ pyproject.toml | 7 ++++ main.py => src/cloudai/__main__.py | 0 3 files changed, 50 insertions(+), 9 deletions(-) rename main.py => src/cloudai/__main__.py (100%) diff --git a/README.md b/README.md index 761bb66fd..7acfd7333 100644 --- a/README.md +++ b/README.md @@ -33,33 +33,33 @@ Cloud AI supports five modes: install, dry-run, run, generate-report, and uninst * Use the generate-report mode to generate reports under the test directories alongside the raw data. * Use the uninstall mode to remove installed test templates. -To install test templates, run main.py in install mode. +To install test templates, run Cloud AI CLI in install mode. Please make sure to use the correct system configuration file that corresponds to your current setup for installation and experiments. ```bash -python main.py\ +python -m cloudai\ --mode install\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml ``` To simulate running experiments without execution, use the dry-run mode: ```bash -python main.py\ +python -m cloudai\ --mode dry-run\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml\ --test_scenario_path conf/v0.6/general/test_scenario/sleep/test_scenario.toml ``` -To run experiments, execute main.py in run mode: +To run experiments, execute Cloud AI CLI in run mode: ```bash -python main.py\ +python -m cloudai\ --mode run\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml\ --test_scenario_path conf/v0.6/general/test_scenario/sleep/test_scenario.toml ``` -To generate reports, execute main.py in generate-report mode: +To generate reports, execute Cloud AI CLI in generate-report mode: ```bash -python main.py\ +python -m cloudai\ --mode generate-report\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml\ --output_path /path/to/output_directory @@ -67,13 +67,47 @@ python main.py\ In the generate-report mode, use the --output_path argument to specify a subdirectory under the result directory. This subdirectory is usually named with a timestamp for unique identification. -To uninstall test templates, run main.py in uninstall mode: +To uninstall test templates, run Cloud AI CLI in uninstall mode: ```bash -python main.py\ +python -m cloudai\ --mode uninstall\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml ``` +# Implementing a custom execution flow + +```py +# Create a System, TestTemplate and TestScenario objects +system, test_templates, test_scenario = System(), [TestTemplate()], TestScenario() + +# One way to do that is to use the Parser class +parser = Parser(, , , ) +system, test_templates, test_scenario = parser.parse() + +# Update the system object using relevant SystemObjectUpdater. This is necessary to update the system object +# with the correct system configuration. +system_object_updater = SystemObjectUpdater() +system_object_updater.update(system) + +# Check if test templates are installed +installer = Installer(system) +if not installer.is_installed(test_templates): + # raise an exception or + installer.install(test_templates) + +# Create a Runner object and run the test scenario. Use asyncio.run() to run the async function. +runner = Runner(args.mode, system, test_scenario) +asyncio.run(runner.run()) + +# Generate a report... +generator = ReportGenerator(runner.runner.output_path) +generator.generate_report(test_scenario) + +# ... and grade the test scenario +grader = Grader(runner.runner.output_path) +grader.grade(test_scenario) +``` + # Contributing Feel free to contribute to the CloudAI project. Your contributions are highly appreciated. diff --git a/pyproject.toml b/pyproject.toml index 9e1342fe5..c80cea9a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,13 @@ [project] name = "cloudai" version = "0.6" +dependencies = [ + "bokeh==3.4.1", + "pandas==2.2.1", + "requests==2.31.0", + "tbparse==0.0.8", + "toml==0.10.2", +] [tool.ruff] target-version = "py39" diff --git a/main.py b/src/cloudai/__main__.py similarity index 100% rename from main.py rename to src/cloudai/__main__.py From bff59dd9bf03889dded0364cc40bf97485844560 Mon Sep 17 00:00:00 2001 From: Andrey Maslennikov Date: Tue, 14 May 2024 13:56:45 +0200 Subject: [PATCH 05/11] Revert changes in README --- README.md | 52 +++++++++------------------------------------------- 1 file changed, 9 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 7acfd7333..761bb66fd 100644 --- a/README.md +++ b/README.md @@ -33,33 +33,33 @@ Cloud AI supports five modes: install, dry-run, run, generate-report, and uninst * Use the generate-report mode to generate reports under the test directories alongside the raw data. * Use the uninstall mode to remove installed test templates. -To install test templates, run Cloud AI CLI in install mode. +To install test templates, run main.py in install mode. Please make sure to use the correct system configuration file that corresponds to your current setup for installation and experiments. ```bash -python -m cloudai\ +python main.py\ --mode install\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml ``` To simulate running experiments without execution, use the dry-run mode: ```bash -python -m cloudai\ +python main.py\ --mode dry-run\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml\ --test_scenario_path conf/v0.6/general/test_scenario/sleep/test_scenario.toml ``` -To run experiments, execute Cloud AI CLI in run mode: +To run experiments, execute main.py in run mode: ```bash -python -m cloudai\ +python main.py\ --mode run\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml\ --test_scenario_path conf/v0.6/general/test_scenario/sleep/test_scenario.toml ``` -To generate reports, execute Cloud AI CLI in generate-report mode: +To generate reports, execute main.py in generate-report mode: ```bash -python -m cloudai\ +python main.py\ --mode generate-report\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml\ --output_path /path/to/output_directory @@ -67,47 +67,13 @@ python -m cloudai\ In the generate-report mode, use the --output_path argument to specify a subdirectory under the result directory. This subdirectory is usually named with a timestamp for unique identification. -To uninstall test templates, run Cloud AI CLI in uninstall mode: +To uninstall test templates, run main.py in uninstall mode: ```bash -python -m cloudai\ +python main.py\ --mode uninstall\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml ``` -# Implementing a custom execution flow - -```py -# Create a System, TestTemplate and TestScenario objects -system, test_templates, test_scenario = System(), [TestTemplate()], TestScenario() - -# One way to do that is to use the Parser class -parser = Parser(, , , ) -system, test_templates, test_scenario = parser.parse() - -# Update the system object using relevant SystemObjectUpdater. This is necessary to update the system object -# with the correct system configuration. -system_object_updater = SystemObjectUpdater() -system_object_updater.update(system) - -# Check if test templates are installed -installer = Installer(system) -if not installer.is_installed(test_templates): - # raise an exception or - installer.install(test_templates) - -# Create a Runner object and run the test scenario. Use asyncio.run() to run the async function. -runner = Runner(args.mode, system, test_scenario) -asyncio.run(runner.run()) - -# Generate a report... -generator = ReportGenerator(runner.runner.output_path) -generator.generate_report(test_scenario) - -# ... and grade the test scenario -grader = Grader(runner.runner.output_path) -grader.grade(test_scenario) -``` - # Contributing Feel free to contribute to the CloudAI project. Your contributions are highly appreciated. From 377d576899f47b17f804687475158524af9d066b Mon Sep 17 00:00:00 2001 From: Andrey Maslennikov Date: Tue, 14 May 2024 14:03:25 +0200 Subject: [PATCH 06/11] Run CLI as a binary after installation --- README.md | 18 +++++++++--------- pyproject.toml | 3 +++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 761bb66fd..f218e0230 100644 --- a/README.md +++ b/README.md @@ -33,33 +33,33 @@ Cloud AI supports five modes: install, dry-run, run, generate-report, and uninst * Use the generate-report mode to generate reports under the test directories alongside the raw data. * Use the uninstall mode to remove installed test templates. -To install test templates, run main.py in install mode. +To install test templates, run Cloud AI CLI in install mode. Please make sure to use the correct system configuration file that corresponds to your current setup for installation and experiments. ```bash -python main.py\ +cloudai\ --mode install\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml ``` To simulate running experiments without execution, use the dry-run mode: ```bash -python main.py\ +cloudai\ --mode dry-run\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml\ --test_scenario_path conf/v0.6/general/test_scenario/sleep/test_scenario.toml ``` -To run experiments, execute main.py in run mode: +To run experiments, execute Cloud AI CLI in run mode: ```bash -python main.py\ +cloudai\ --mode run\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml\ --test_scenario_path conf/v0.6/general/test_scenario/sleep/test_scenario.toml ``` -To generate reports, execute main.py in generate-report mode: +To generate reports, execute Cloud AI CLI in generate-report mode: ```bash -python main.py\ +cloudai\ --mode generate-report\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml\ --output_path /path/to/output_directory @@ -67,9 +67,9 @@ python main.py\ In the generate-report mode, use the --output_path argument to specify a subdirectory under the result directory. This subdirectory is usually named with a timestamp for unique identification. -To uninstall test templates, run main.py in uninstall mode: +To uninstall test templates, run Cloud AI CLI in uninstall mode: ```bash -python main.py\ +cloudai\ --mode uninstall\ --system_config_path conf/v0.6/general/system/example_slurm_cluster.toml ``` diff --git a/pyproject.toml b/pyproject.toml index c80cea9a8..2dedebf46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,9 @@ dependencies = [ "toml==0.10.2", ] +[project.scripts] +cloudai = "cloudai.__main__:main" + [tool.ruff] target-version = "py39" line-length = 120 From 1f5de6057c5341a784e4c9d351c0406dddaf21bb Mon Sep 17 00:00:00 2001 From: Andrey Maslennikov Date: Tue, 14 May 2024 17:36:40 +0200 Subject: [PATCH 07/11] Add vulture into requirements-dev.txt --- requirements-dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index fe537ca2e..33241661a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,3 +4,5 @@ ruff==0.3.7 pandas-stubs==2.2.* pyright==1.1.359 build==1.2.* +vulture==2.11 + From 9b4ed836c5f457d8e3a0f247da2e0e97c311450f Mon Sep 17 00:00:00 2001 From: Andrey Maslennikov Date: Tue, 14 May 2024 17:38:34 +0200 Subject: [PATCH 08/11] Fix paths for vulture checks --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06eda41fe..d518cbe51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,7 @@ jobs: run: pyright . - name: Run vulture check - run: vulture cloudai/ tests/ ci_tools/ --min-confidence 100 + run: vulture src/ tests/ ci_tools/ --min-confidence 100 test: name: Run pytest From 74d8e8d7f49ce7e563d107bada6a46e09db763f3 Mon Sep 17 00:00:00 2001 From: Taekyung Heo <7621438+TaekyungHeo@users.noreply.github.com> Date: Tue, 14 May 2024 07:18:39 -0400 Subject: [PATCH 09/11] Update stats collection to drop initial 10 epochs --- .../jax_toolbox/report_generation_strategy.py | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/cloudai/schema/test_template/jax_toolbox/report_generation_strategy.py b/src/cloudai/schema/test_template/jax_toolbox/report_generation_strategy.py index ce17fb6a4..438cac96d 100644 --- a/src/cloudai/schema/test_template/jax_toolbox/report_generation_strategy.py +++ b/src/cloudai/schema/test_template/jax_toolbox/report_generation_strategy.py @@ -55,32 +55,34 @@ def generate_report(self, directory_path: str, sol: Optional[float] = None) -> N def _extract_times(self, directory_path: str) -> List[float]: """ Extracts elapsed times from all error files matching the pattern in the directory, - excluding the first time value recorded in each file. + starting after the 10th occurrence of a line matching the "[PAX STATUS]: train_step() took" pattern. Args: directory_path (str): Directory containing error files. Returns: - List[float]: List of extracted times as floats, after excluding the first time from each file. + List[float]: List of extracted times as floats, starting from the epoch after the 10th occurrence. """ times = [] error_files = glob.glob(os.path.join(directory_path, "error-*.txt")) for stderr_path in error_files: file_times = [] + epoch_count = 0 with open(stderr_path, "r") as file: for line in file: - if "Elapsed time for" in line and "run" in line and ":434" in line: - parts = line.split() - time_str = parts[parts.index(":") + 1] - try: - time_value = float(time_str.split("seconds")[0]) - file_times.append(time_value) - except ValueError: - continue # Skip any lines where conversion fails + if "[PAX STATUS]: train_step() took" in line: + epoch_count += 1 + if epoch_count > 10: # Start recording times after 10 epochs + # Extract the time value right after the keyword + parts = line.split("took") + time_str = parts[1].strip().split("seconds")[0].strip() + try: + time_value = float(time_str) + file_times.append(time_value) + except ValueError: + continue # Skip any lines where conversion fails - # Exclude the first time record from each file if it exists - if file_times: - times.extend(file_times[1:]) + times.extend(file_times) return times From 5c55a25cc7402a7ce9708a897ff360a4d723f03e Mon Sep 17 00:00:00 2001 From: Taekyung Heo <7621438+TaekyungHeo@users.noreply.github.com> Date: Tue, 14 May 2024 07:19:03 -0400 Subject: [PATCH 10/11] Collect stdev in JaxToolbox report generation strategy --- .../test_template/jax_toolbox/report_generation_strategy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cloudai/schema/test_template/jax_toolbox/report_generation_strategy.py b/src/cloudai/schema/test_template/jax_toolbox/report_generation_strategy.py index 438cac96d..d3bb87c52 100644 --- a/src/cloudai/schema/test_template/jax_toolbox/report_generation_strategy.py +++ b/src/cloudai/schema/test_template/jax_toolbox/report_generation_strategy.py @@ -49,6 +49,7 @@ def generate_report(self, directory_path: str, sol: Optional[float] = None) -> N "max": max(times), "average": sum(times) / len(times), "median": statistics.median(times), + "stdev": statistics.stdev(times) if len(times) > 1 else 0, } self._write_report(directory_path, stats) From c5b37f4de3a90b34c5d35fe37c0233b9928ed07f Mon Sep 17 00:00:00 2001 From: Taekyung Heo <7621438+TaekyungHeo@users.noreply.github.com> Date: Tue, 14 May 2024 09:37:57 -0400 Subject: [PATCH 11/11] Add unit tests for JaxToolboxReportGenerationStrategy Co-authored-by: Andrey Maslennikov --- .../test_report_generation_strategy.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/schema/test_template/jax_toolbox/test_report_generation_strategy.py diff --git a/tests/schema/test_template/jax_toolbox/test_report_generation_strategy.py b/tests/schema/test_template/jax_toolbox/test_report_generation_strategy.py new file mode 100644 index 000000000..f725f3081 --- /dev/null +++ b/tests/schema/test_template/jax_toolbox/test_report_generation_strategy.py @@ -0,0 +1,34 @@ +from pathlib import Path + +from cloudai.schema.test_template.jax_toolbox.report_generation_strategy import ( + JaxToolboxReportGenerationStrategy, +) + + +class TestJaxExtractTime: + """Tests for the JaxToolboxReportGenerationStrategy class.""" + + def setup_method(self) -> None: + """Setup method for initializing JaxToolboxReportGenerationStrategy.""" + self.js = JaxToolboxReportGenerationStrategy() + + def test_no_files(self, tmp_path: Path) -> None: + """Test that no times are extracted when no files are present.""" + assert self.js._extract_times(str(tmp_path)) == [] + + def test_no_matches(self, tmp_path: Path) -> None: + """Test that no times are extracted when no matching lines are present.""" + (tmp_path / "error-1.txt").write_text("fake line") + assert self.js._extract_times(str(tmp_path)) == [] + + def test_one_match(self, tmp_path: Path) -> None: + """Test that the correct time is extracted when one matching line is present.""" + err_file = tmp_path / "error-1.txt" + sample_line = ( + "I0508 15:25:28.482553 140737334253888 programs.py:379] " + "[PAX STATUS]: train_step() took 38.727223 seconds.\n" + ) + with err_file.open("w") as f: + for _ in range(11): + f.write(sample_line) + assert self.js._extract_times(str(err_file.parent)) == [38.727223]