From 21ca3e32b4d1598569c5f9cac820a44d45430307 Mon Sep 17 00:00:00 2001 From: SorooshMani-NOAA Date: Mon, 19 Aug 2024 11:38:25 -0400 Subject: [PATCH 1/8] Note versions for workflow, coupldemodeldriver and pyschism in output. --- stormworkflow/scripts/workflow.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stormworkflow/scripts/workflow.sh b/stormworkflow/scripts/workflow.sh index 3b14816..f1bbf4f 100755 --- a/stormworkflow/scripts/workflow.sh +++ b/stormworkflow/scripts/workflow.sh @@ -48,8 +48,11 @@ function init { done logfile=$run_dir/versions.info + version $logfile stormworkflow version $logfile stormevents version $logfile ensembleperturbation + version $logfile coupledmodeldriver + version $logfile pyschism version $logfile ocsmesh echo "SCHISM: see solver.version each outputs dir" >> $logfile From c2252e04a91bbdb89680da52cf9bc9b79e916e15 Mon Sep 17 00:00:00 2001 From: SorooshMani-NOAA Date: Mon, 19 Aug 2024 12:04:28 -0400 Subject: [PATCH 2/8] Add input version check and enforcement --- pyproject.toml | 1 + stormworkflow/main.py | 55 +++++++++++++++++++++++++++++-- stormworkflow/scripts/workflow.sh | 2 +- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4dd25e5..446076b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ dependencies = [ "numpy", "numba", "ocsmesh==1.5.3", + "packaging", "pandas", "pyarrow", "pygeos", diff --git a/stormworkflow/main.py b/stormworkflow/main.py index e649f6c..1aa5265 100644 --- a/stormworkflow/main.py +++ b/stormworkflow/main.py @@ -6,16 +6,62 @@ from argparse import ArgumentParser from pathlib import Path -import stormworkflow import yaml +from packaging.version import Version try: from yaml import CLoader as Loader, CDumper as Dumper except ImportError: from yaml import Loader, Dumper +import stormworkflow _logger = logging.getLogger(__file__) +CUR_INPUT_VER = Version('0.0.2') + + +def _handle_input_v0_0_1_to_v0_0_2(inout_conf): + + # Only update conf if specified version matches the assumed one + if ver != Version('0.0.1'): + return ver + + + _logger.info( + "Adding perturbation variables for persistent RMW perturbation" + ) + inout_conf['perturb_vars'] = [ + 'cross_track', + 'along_track', + 'radius_of_maximum_winds_persistent', + 'max_sustained_wind_speed', + ] + + return Version('0.0.2') + + +def _handle_input_version(inout_conf): + + if 'input_version' not in inout_conf: + ver = CUR_INPUT_VER + _logger.warning( + f"`input_version` is NOT specified in `input.yaml`; assuming {ver}" + ) + + ver = Version(inout_conf['input_version']) + + if ver > CUR_INPUT_VER: + raise ValueError( + f"Input version not supported! Max version supported is {CUR_INPUT_VER}" + ) + + ver = _handle_input_v0_0_1_to_v0_0_2(inout_conf) + + if ver != CUR_INPUT_VER + raise ValueError( + f"Could NOT update input to the latest version! Updated to {ver}" + + def main(): parser = ArgumentParser() @@ -28,12 +74,17 @@ def main(): infile = args.configuration if infile is None: - _logger.warn('No input configuration provided, using reference file!') + _logger.warning( + 'No input configuration provided, using reference file!' + ) infile = refs.joinpath('input.yaml') with open(infile, 'r') as yfile: conf = yaml.load(yfile, Loader=Loader) + _handle_input_version(conf) + # TODO: Write out the updated conf as a yaml file + wf = scripts.joinpath('workflow.sh') run_env = os.environ.copy() diff --git a/stormworkflow/scripts/workflow.sh b/stormworkflow/scripts/workflow.sh index f1bbf4f..723f4b0 100755 --- a/stormworkflow/scripts/workflow.sh +++ b/stormworkflow/scripts/workflow.sh @@ -56,7 +56,7 @@ function init { version $logfile ocsmesh echo "SCHISM: see solver.version each outputs dir" >> $logfile - cp $input_file $run_dir/input.yaml + cp $input_file $run_dir/input_asis.yaml echo $run_dir } From bc90ec1d7d5d0a96d40b90a36454785aee1c67fd Mon Sep 17 00:00:00 2001 From: SorooshMani-NOAA Date: Mon, 19 Aug 2024 12:06:33 -0400 Subject: [PATCH 3/8] Syntax error fix --- stormworkflow/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stormworkflow/main.py b/stormworkflow/main.py index 1aa5265..4bdc056 100644 --- a/stormworkflow/main.py +++ b/stormworkflow/main.py @@ -57,9 +57,10 @@ def _handle_input_version(inout_conf): ver = _handle_input_v0_0_1_to_v0_0_2(inout_conf) - if ver != CUR_INPUT_VER + if ver != CUR_INPUT_VER: raise ValueError( f"Could NOT update input to the latest version! Updated to {ver}" + ) def main(): From 6382e34bce056e27a48774d4dbb071890ada37ed Mon Sep 17 00:00:00 2001 From: SorooshMani-NOAA Date: Mon, 19 Aug 2024 12:10:18 -0400 Subject: [PATCH 4/8] Update config object version after handling updates --- stormworkflow/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/stormworkflow/main.py b/stormworkflow/main.py index 4bdc056..f228960 100644 --- a/stormworkflow/main.py +++ b/stormworkflow/main.py @@ -22,7 +22,7 @@ def _handle_input_v0_0_1_to_v0_0_2(inout_conf): - # Only update conf if specified version matches the assumed one + # Only update config if specified version matches the assumed one if ver != Version('0.0.1'): return ver @@ -62,6 +62,8 @@ def _handle_input_version(inout_conf): f"Could NOT update input to the latest version! Updated to {ver}" ) + inout_conf['input_version'] = str(v2) + def main(): @@ -84,7 +86,7 @@ def main(): conf = yaml.load(yfile, Loader=Loader) _handle_input_version(conf) - # TODO: Write out the updated conf as a yaml file + # TODO: Write out the updated config as a yaml file wf = scripts.joinpath('workflow.sh') From e0e95d4b5017769692c8ac7a8f810a9ff8688e0e Mon Sep 17 00:00:00 2001 From: SorooshMani-NOAA Date: Mon, 19 Aug 2024 14:43:48 -0400 Subject: [PATCH 5/8] First iteration of CI tests --- .github/workflows/tests.yml | 36 ++++++++++++++++++++++++ pyproject.toml | 7 ++++- stormworkflow/main.py | 11 +++++--- tests/data/refs/input_v0.0.1.yaml | 40 +++++++++++++++++++++++++++ tests/data/refs/input_v0.0.2.yaml | 46 +++++++++++++++++++++++++++++++ tests/test_input_version.py | 43 +++++++++++++++++++++++++++++ 6 files changed, 178 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/tests.yml create mode 100644 tests/data/refs/input_v0.0.1.yaml create mode 100644 tests/data/refs/input_v0.0.2.yaml create mode 100644 tests/test_input_version.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..026d464 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,36 @@ +name: tests + +on: + push: + branches: + - main + paths: + - '**.py' + - '.github/workflows/tests.yml' + - 'pyproject.toml' + pull_request: + branches: + - main + +jobs: + test: + name: test + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-latest, macos-latest ] + python-version: [ '3.9', '3.10', '3.11' ] + steps: + - name: clone repository + uses: actions/checkout@v4 + - name: conda virtual environment + uses: mamba-org/setup-micromamba@v1 + with: + init-shell: bash + environment-file: environment.yml + - name: install the package + run: pip install ".[dev]" + shell: micromamba-shell {0} + - name: run tests + run: pytest --numprocesses auto + shell: micromamba-shell {0} diff --git a/pyproject.toml b/pyproject.toml index 446076b..2f7cedc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ description = "A set of scripts to generate probabilistic storm surge results!" license = {file = "LICENSE"} -requires-python = ">= 3.8, < 3.12" +requires-python = ">= 3.9, < 3.12" dependencies = [ "cartopy", @@ -66,6 +66,11 @@ dependencies = [ "xarray", ] +[project.optional-dependencies] +dev = [ + "pytest" +] + [tool.setuptools_scm] version_file = "stormworkflow/_version.py" diff --git a/stormworkflow/main.py b/stormworkflow/main.py index f228960..80fa06f 100644 --- a/stormworkflow/main.py +++ b/stormworkflow/main.py @@ -2,6 +2,7 @@ import logging import os import shlex +import warnings from importlib.resources import files from argparse import ArgumentParser from pathlib import Path @@ -40,13 +41,15 @@ def _handle_input_v0_0_1_to_v0_0_2(inout_conf): return Version('0.0.2') -def _handle_input_version(inout_conf): +def handle_input_version(inout_conf): if 'input_version' not in inout_conf: ver = CUR_INPUT_VER - _logger.warning( + warnings.warn( f"`input_version` is NOT specified in `input.yaml`; assuming {ver}" ) + inout_conf['input_version'] = ver + return ver = Version(inout_conf['input_version']) @@ -77,7 +80,7 @@ def main(): infile = args.configuration if infile is None: - _logger.warning( + warnings.warn( 'No input configuration provided, using reference file!' ) infile = refs.joinpath('input.yaml') @@ -85,7 +88,7 @@ def main(): with open(infile, 'r') as yfile: conf = yaml.load(yfile, Loader=Loader) - _handle_input_version(conf) + handle_input_version(conf) # TODO: Write out the updated config as a yaml file wf = scripts.joinpath('workflow.sh') diff --git a/tests/data/refs/input_v0.0.1.yaml b/tests/data/refs/input_v0.0.1.yaml new file mode 100644 index 0000000..1e8141e --- /dev/null +++ b/tests/data/refs/input_v0.0.1.yaml @@ -0,0 +1,40 @@ +--- +input_version: 0.0.1 + +storm: "florence" +year: 2018 +suffix: "" +subset_mesh: 1 +hr_prelandfall: -1 +past_forecast: 1 +hydrology: 0 +use_wwm: 0 +pahm_model: "gahm" +num_perturb: 2 +sample_rule: "korobov" +spinup_exec: "pschism_PAHM_TVD-VL" +hotstart_exec: "pschism_PAHM_TVD-VL" + +hpc_solver_nnodes: 3 +hpc_solver_ntasks: 108 +hpc_account: "" +hpc_partition: "" + +RUN_OUT: "" +L_NWM_DATASET: "" +L_TPXO_DATASET: "" +L_LEADTIMES_DATASET: "" +L_TRACK_DIR: "" +L_DEM_HI: "" +L_DEM_LO: "" +L_MESH_HI: "" +L_MESH_LO: "" +L_SHP_DIR: "" + +TMPDIR: "/tmp" +PATH_APPEND: "" + +L_SOLVE_MODULES: + - "intel/2022.1.2" + - "impi/2022.1.2" + - "netcdf" diff --git a/tests/data/refs/input_v0.0.2.yaml b/tests/data/refs/input_v0.0.2.yaml new file mode 100644 index 0000000..f438167 --- /dev/null +++ b/tests/data/refs/input_v0.0.2.yaml @@ -0,0 +1,46 @@ +--- +input_version: 0.0.2 + +storm: "florence" +year: 2018 +suffix: "" +subset_mesh: 1 +hr_prelandfall: -1 +past_forecast: 1 +hydrology: 0 +use_wwm: 0 +pahm_model: "gahm" +num_perturb: 2 +sample_rule: "korobov" +spinup_exec: "pschism_PAHM_TVD-VL" +hotstart_exec: "pschism_PAHM_TVD-VL" +perturb_vars: + - 'cross_track' + - 'along_track' +# - 'radius_of_maximum_winds' + - 'radius_of_maximum_winds_persistent' + - 'max_sustained_wind_speed' + +hpc_solver_nnodes: 3 +hpc_solver_ntasks: 108 +hpc_account: "" +hpc_partition: "" + +RUN_OUT: "" +L_NWM_DATASET: "" +L_TPXO_DATASET: "" +L_LEADTIMES_DATASET: "" +L_TRACK_DIR: "" +L_DEM_HI: "" +L_DEM_LO: "" +L_MESH_HI: "" +L_MESH_LO: "" +L_SHP_DIR: "" + +TMPDIR: "/tmp" +PATH_APPEND: "" + +L_SOLVE_MODULES: + - "intel/2022.1.2" + - "impi/2022.1.2" + - "netcdf" diff --git a/tests/test_input_version.py b/tests/test_input_version.py new file mode 100644 index 0000000..fb5a0a6 --- /dev/null +++ b/tests/test_input_version.py @@ -0,0 +1,43 @@ +from importlib.resources import files + +import pytest +import yaml +from packaging.version import Version +from yaml import Loader, Dumper + +from stormworkflow.main import handle_input_version, CUR_INPUT_VER + + +refs = files('tests.data.refs') +input_v0_0_1 = refs.joinpath('input_v0.0.1.yaml') +input_v0_0_2 = refs.joinpath('input_v0.0.2.yaml') + + +def read_conf(infile): + with open(infile, 'r') as yfile: + conf = yaml.load(yfile, Loader=Loader) + return conf + + +@pytest.fixture +def conf_v0_0_1(): + return read_conf(input_v0_0_1) + + +@pytest.fixture +def conf_v0_0_2(): + return read_conf(input_v0_0_2) + + +@pytest.fixture +def conf_latest(conf_v0_0_2): + return conf_v0_0_2 + + +def test_no_version_specified(conf_latest): + conf_latest.pop('input_version') + with pytest.warns(UserWarning): + handle_input_version(conf_latest) + + assert conf_latest['input_version'] == CUR_INPUT_VER + From ef073fa807f79eca76cd5fa2703d7ed814155ce6 Mon Sep 17 00:00:00 2001 From: SorooshMani-NOAA Date: Mon, 19 Aug 2024 16:25:37 -0400 Subject: [PATCH 6/8] Add more test and bugfix for version check --- stormworkflow/main.py | 6 ++++-- tests/test_input_version.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/stormworkflow/main.py b/stormworkflow/main.py index 80fa06f..665dbff 100644 --- a/stormworkflow/main.py +++ b/stormworkflow/main.py @@ -23,6 +23,8 @@ def _handle_input_v0_0_1_to_v0_0_2(inout_conf): + ver = Version(inout_conf['input_version']) + # Only update config if specified version matches the assumed one if ver != Version('0.0.1'): return ver @@ -48,7 +50,7 @@ def handle_input_version(inout_conf): warnings.warn( f"`input_version` is NOT specified in `input.yaml`; assuming {ver}" ) - inout_conf['input_version'] = ver + inout_conf['input_version'] = str(ver) return ver = Version(inout_conf['input_version']) @@ -65,7 +67,7 @@ def handle_input_version(inout_conf): f"Could NOT update input to the latest version! Updated to {ver}" ) - inout_conf['input_version'] = str(v2) + inout_conf['input_version'] = str(ver) def main(): diff --git a/tests/test_input_version.py b/tests/test_input_version.py index fb5a0a6..6fa47b3 100644 --- a/tests/test_input_version.py +++ b/tests/test_input_version.py @@ -1,3 +1,4 @@ +from copy import deepcopy from importlib.resources import files import pytest @@ -38,6 +39,29 @@ def test_no_version_specified(conf_latest): conf_latest.pop('input_version') with pytest.warns(UserWarning): handle_input_version(conf_latest) - - assert conf_latest['input_version'] == CUR_INPUT_VER + + assert conf_latest['input_version'] == str(CUR_INPUT_VER) + +def test_invalid_version_specified(conf_latest): + + invalid_1 = deepcopy(conf_latest) + invalid_1['input_version'] = ( + f'{CUR_INPUT_VER.major}.{CUR_INPUT_VER.minor}.{CUR_INPUT_VER.micro + 1}' + ) + with pytest.raises(ValueError) as e: + handle_input_version(invalid_1) + + assert "max" in str(e.value).lower() + + + invalid_2 = deepcopy(conf_latest) + invalid_2['input_version'] = 'a.b.c' + with pytest.raises(ValueError) as e: + handle_input_version(invalid_2) + assert "invalid version" in str(e.value).lower() + + +def test_v0_0_1_to_v0_0_2(conf_v0_0_1, conf_v0_0_2): + handle_input_version(conf_v0_0_1) + assert conf_v0_0_2 == conf_v0_0_1 From f1d82a11aa5e052248678fb3d6d1377f4532d69a Mon Sep 17 00:00:00 2001 From: SorooshMani-NOAA Date: Mon, 19 Aug 2024 16:38:04 -0400 Subject: [PATCH 7/8] Remove macos from tests --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 026d464..d03fa52 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-latest, macos-latest ] + os: [ ubuntu-latest ] python-version: [ '3.9', '3.10', '3.11' ] steps: - name: clone repository From edb27e509be6813046303e6b16a927553d61f221 Mon Sep 17 00:00:00 2001 From: SorooshMani-NOAA Date: Mon, 19 Aug 2024 16:46:27 -0400 Subject: [PATCH 8/8] Update pytest arguments --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d03fa52..bcf9649 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,5 +32,5 @@ jobs: run: pip install ".[dev]" shell: micromamba-shell {0} - name: run tests - run: pytest --numprocesses auto + run: pytest shell: micromamba-shell {0}