Merge pull request #112 from eWaterCycle/allow-apptainer-version

Allow apptainer version

.github/workflows/ci.yml

@@ -5,7 +5,9 @@ name: CI
 
 on:
   push:
+    branches: [main]
   pull_request:
+    types: [opened, synchronize, reopened]
 
 jobs:
   python:
@@ -15,14 +17,14 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.6
+        python-version: '3.10'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        python setup.py install
-    - name: Setup Singularity
-      uses: eWaterCycle/setup-singularity@v6
+        pip install -e .[R]
+    - name: Setup Apptainer
+      uses: eWaterCycle/setup-apptainer@v2
       with:
         singularity-version: 3.8.3
     - name: Pull Docker image
@@ -36,8 +38,16 @@ jobs:
     - name: Pull Singularity image
       if: steps.cache-singularity-image.outputs.cache-hit != 'true'
       run: singularity pull docker://ewatercycle/walrus-grpc4bmi:v0.2.0
+    - uses: r-lib/actions/setup-r@v2
+      with:
+        install-r: false
+    - name: Install R dependencies
+      run: |
+        Rscript -e "install.packages('remotes')"
+        Rscript -e "install.packages('R6')"
     - name: Test with pytest
-      run: pytest --cov=grpc4bmi --cov-report xml
+      run: |
+        pytest -vv --cov=grpc4bmi --cov-report xml
     - name: Correct coverage paths
       run: sed -i "s+$PWD/++g" coverage.xml
     - name: SonarCloud analysis

grpc4bmi/bmi_client_singularity.py

@@ -6,25 +6,34 @@
 from tempfile import SpooledTemporaryFile
 from typing import Iterable
 
-import semver
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
 from typeguard import check_argument_types, qualified_name
 
 from grpc4bmi.bmi_grpc_client import BmiClient
-from grpc4bmi.exceptions import DeadContainerException, SingularityVersionException
-
-REQUIRED_SINGULARITY_VERSION = '3.6.0'
-
+from grpc4bmi.exceptions import ApptainerVersionException, DeadContainerException, SingularityVersionException
+
+SUPPORTED_SINGULARITY_VERSIONS = '>=3.6.0'
+SUPPORTED_APPTAINER_VERSIONS = '>=1.0.0-rc.2'  # First apptainer release with binaries
+
+def check_singularity_version_string(version_output: str) -> bool:
+    (app, _, version) = version_output.split(' ')
+    local_version = Version(version)
+    if app == 'singularity' and local_version not in SpecifierSet(SUPPORTED_SINGULARITY_VERSIONS):
+        raise SingularityVersionException(f'Unsupported version ({version_output}) of singularity found, '
+                                          f'supported versions {SUPPORTED_SINGULARITY_VERSIONS}')
+    # Apptainer creates /usr/bin/singularity symlink, so if installed will report the apptainer version.
+    if app == 'apptainer' and local_version not in SpecifierSet(SUPPORTED_APPTAINER_VERSIONS):
+        raise ApptainerVersionException(f'Unsupported version ({version_output}) of apptainer found, '
+                                          f'supported versions {SUPPORTED_APPTAINER_VERSIONS}')
+    return True
 
 def check_singularity_version():
-    p = subprocess.Popen(['singularity', 'version'], stdout=subprocess.PIPE)
-    (stdout, _stderr) = p.communicate()
+    p = subprocess.Popen(['singularity', '--version'], stdout=subprocess.PIPE)
+    (stdout, _) = p.communicate()
     if p.returncode != 0:
         raise SingularityVersionException('Unable to determine singularity version')
-    local_version = semver.VersionInfo.parse(stdout.decode('utf-8').replace('_', '-'))
-    if local_version < REQUIRED_SINGULARITY_VERSION:
-        raise SingularityVersionException(f'Wrong version ({local_version}) of singularity found, '
-                                          f'require version {REQUIRED_SINGULARITY_VERSION}')
-    return True
+    check_singularity_version_string(stdout.decode('utf-8'))
 
 
 class BmiClientSingularity(BmiClient):

grpc4bmi/bmi_r_model.py

@@ -31,7 +31,7 @@ def update_until(self, time):
         self.model['updateUntil'](time)
 
     def update_fraq(self, time_frac):
-        self.model['updateFrac '](time_frac)
+        self.model['updateFrac'](time_frac)
 
     def finalize(self):
         self.model['bmi_finalize']()

grpc4bmi/exceptions.py

@@ -19,3 +19,6 @@ def __init__(self, message, exitcode, logs, *args):
 
 class SingularityVersionException(ValueError):
     pass
+
+class ApptainerVersionException(ValueError):
+    pass

requirements.txt

@@ -5,15 +5,15 @@ grpcio-tools
 grpcio-reflection
 numpy
 docutils<0.18
-protobuf
+# Pin protobuf, see https://github.com/eWaterCycle/grpc4bmi/issues/115
+protobuf<=3.20.3
 PyYAML
 numpydoc
 cfunits
 scipy
 docker
 pytest
 pytest-cov
-futures
 https://github.com/eWaterCycle/bmi-python/archive/master.zip#egg=bmi-heat
 sphinx
 sphinxcontrib-apidoc

setup.py

@@ -27,11 +27,12 @@ def read(fname):
       install_requires=[
           "grpcio",
           "grpcio-reflection",
-          "protobuf",
+          # Pin protobuf, see https://github.com/eWaterCycle/grpc4bmi/issues/115
+          "protobuf<=3.20.3",
           "numpy",
           "docker",
           "basic-modeling-interface",
-          "semver>=2.10.0",
+          "packaging",
           "typeguard",
       ],
       extras_require={
@@ -44,6 +45,9 @@ def read(fname):
                    "Programming Language :: Python :: 3.5",
                    "Programming Language :: Python :: 3.6",
                    "Programming Language :: Python :: 3.7",
+                   "Programming Language :: Python :: 3.8",
+                   "Programming Language :: Python :: 3.9",
+                   "Programming Language :: Python :: 3.10",
                    "Topic :: Utilities",
                    "Topic :: Scientific/Engineering",
                    "License :: OSI Approved :: Apache Software License"

sonar-project.properties

@@ -2,6 +2,8 @@ sonar.projectKey=grpc4bmi
 sonar.host.url=https://sonarcloud.io
 sonar.organization=ewatercycle
 sonar.sources=grpc4bmi/
+# Generated by the protocol buffer compiler
+sonar.exclusions=grpc4bmi/bmi_pb2.py,grpc4bmi/bmi_pb2_grpc.py
 sonar.python.coverage.reportPaths=coverage.xml
 # Disable cpp analysis as it is a paid feature, not available for open source projects on sonarcloud.io
 sonar.c.file.suffixes=-

test/fake.r

@@ -0,0 +1,48 @@
+library(R6)
+
+FakeFailingRModel <- R6Class(
+  public = list(
+    # R6 constructor is also called initialize so rename bmi initialize
+    bmi_initialize = function(config_file) stop('Always fails'),
+    update = function() stop('Always fails'),
+    updateUntil = function(until) stop('Always fails'),
+    updateFrac = function(frac) stop('Always fails'),
+    # R6 destructor is also called finalize so rename bmi finalize
+    bmi_finalize = function() stop('Always fails'),
+    runModel = function() stop('Always fails'),
+
+    getComponentName = function() stop('Always fails'),
+    getInputVarNames = function() stop('Always fails'),
+    getOutputVarNames = function() stop('Always fails'),
+
+    getTimeUnits = function() stop('Always fails'),
+    getTimeStep = function() stop('Always fails'),
+    getCurrentTime = function() stop('Always fails'),
+    getStartTime = function() stop('Always fails'),
+    getEndTime = function() stop('Always fails'),
+
+    getVarGrid = function(name) stop('Always fails'),
+    getVarType = function(name) stop('Always fails'),
+    getVarItemSize = function(name) stop('Always fails'),
+    getVarUnits = function(name) stop('Always fails'),
+    getVarNBytes = function(name) stop('Always fails'),
+
+    getValue = function(name) stop('Always fails'),
+    getValueAtIndices = function(name, indices) stop('Always fails'),
+
+    setValue = function(name, values) stop('Always fails'),
+    setValueAtIndices = function(name, indices, values) stop('Always fails'),
+
+    getGridSize = function(grid_id) stop('Always fails'),
+    getGridType = function(grid_id) stop('Always fails'),
+    getGridRank = function(grid_id) stop('Always fails'),
+    getGridShape = function(grid_id) stop('Always fails'),
+    getGridSpacing = function(grid_id) stop('Always fails'),
+    getGridOrigin = function(grid_id) stop('Always fails'),
+    getGridX = function(grid_id) stop('Always fails'),
+    getGridY = function(grid_id) stop('Always fails'),
+    getGridZ = function(grid_id) stop('Always fails'),
+    getGridConnectivity = function(grid_id) stop('Always fails'),
+    getGridOffset = function(grid_id) stop('Always fails')
+  )
+)

test/test_r.py

@@ -0,0 +1,58 @@
+from pathlib import Path
+import numpy as np
+import pytest
+from rpy2.rinterface_lib.embedded import RRuntimeError
+
+from grpc4bmi.run_server import BmiR, build_r
+
+@pytest.fixture
+def model():
+    return build_r('FakeFailingRModel', 'test/fake.r')
+
+@pytest.mark.skipif(not BmiR, reason='R and its dependencies are not installed')
+class TestFakeFailingRModel:
+    @pytest.mark.parametrize(
+    'fn_name,fn_args',
+    [
+        ('get_component_name', tuple()),
+        ('get_input_var_names', tuple()),
+        ('get_output_var_names', tuple()),
+        ('get_start_time', tuple()),
+        ('get_end_time', tuple()),
+        ('get_time_step', tuple()),
+        ('get_time_units', tuple()),
+        ('get_var_type', ['plate_surface__temperature']),
+        ('get_var_units', ['plate_surface__temperature']),
+        ('get_var_itemsize', ['plate_surface__temperature']),
+        ('get_var_nbytes', ['plate_surface__temperature']),
+        ('get_var_grid', ['plate_surface__temperature']),
+        ('get_grid_shape', [0]),
+        ('get_grid_x', [0]),
+        ('get_grid_y', [0]),
+        ('get_grid_z', [0]),
+        ('get_grid_spacing', [0]),
+        ('get_grid_origin', [0]),
+        ('get_grid_connectivity', [0]),
+        ('get_grid_offset', [0]),
+        ('get_grid_rank', [0]),
+        ('get_grid_size', [0]),
+        ('get_grid_type', [0]),
+        ('update', tuple()),
+        ('update_until', [2]),
+        ('finalize', tuple()),
+        ('get_current_time', tuple()),
+        ('get_value_at_indices', ['plate_surface__temperature', [1, 2, 3]]),
+        ('set_value', ['plate_surface__temperature', np.ones((10, 20))]),
+        ('set_value_at_indices', ['plate_surface__temperature', [1, 2, 3], [4, 5, 6]]),        
+        # TODO figure out functions below do not raise error
+        # ('update_frac', [0.5]),
+        # ('get_value', ['plate_surface__temperature']),
+        # ('get_value_ref', ['plate_surface__temperature']),
+    ]
+)
+    def test_r_function_is_called(self, model: BmiR, fn_name, fn_args):
+        # Every method in 'test/fake.r' executes the stop error action
+        # So if no stop then no r function is called
+        with pytest.raises(RRuntimeError, match="Always fails"):
+            fn = getattr(model, fn_name)
+            fn(*fn_args)

test/test_singularity.py

@@ -2,14 +2,15 @@
 import subprocess
 from tempfile import TemporaryDirectory
 from textwrap import dedent
+from typing import Type, Union
 
 import pytest
 from grpc import RpcError
 from nbconvert.preprocessors import ExecutePreprocessor
 from nbformat.v4 import new_notebook, new_code_cell
 
-from grpc4bmi.bmi_client_singularity import BmiClientSingularity
-from grpc4bmi.exceptions import DeadContainerException
+from grpc4bmi.bmi_client_singularity import SUPPORTED_APPTAINER_VERSIONS, SUPPORTED_SINGULARITY_VERSIONS, BmiClientSingularity, check_singularity_version_string
+from grpc4bmi.exceptions import ApptainerVersionException, DeadContainerException, SingularityVersionException
 from test.conftest import write_config, write_datafile
 
 IMAGE_NAME = "docker://ewatercycle/walrus-grpc4bmi:v0.2.0"
@@ -219,3 +220,27 @@ def notebook(tmp_path):
 def test_from_notebook(notebook, tmp_path):
     ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
     ep.preprocess(notebook, {'metadata': {'path': tmp_path}})
+
+class Test_check_singularity_version_string:
+    @pytest.mark.parametrize("test_input", [
+        ('singularity version 3.6.0'),
+        ('singularity version 3.8.7'),  # Last OSS version before fork
+        ('apptainer version 1.0.0-rc.2'),  
+        ('apptainer version 1.0.0'),
+        ('apptainer version 1.0.3'),
+        ('apptainer version 1.1.0-rc.3'),
+        ('apptainer version 1.1.2'),
+    ])
+    def test_ok(self, test_input: str):
+        result = check_singularity_version_string(test_input)
+        assert result
+
+
+    @pytest.mark.parametrize("test_input,error_class,expected", [
+        ('singularity version 3.5.0', SingularityVersionException, SUPPORTED_SINGULARITY_VERSIONS),
+        ('apptainer version 1.0.0-rc.1', ApptainerVersionException, SUPPORTED_APPTAINER_VERSIONS),
+        ('apptainer version 0.1.0', ApptainerVersionException, SUPPORTED_APPTAINER_VERSIONS),
+    ])
+    def test_too_old(self, test_input: str, error_class: Union[Type[ApptainerVersionException], Type[SingularityVersionException]], expected: str):
+        with pytest.raises(error_class, match=expected):
+            check_singularity_version_string(test_input)