Merge pull request #20 from KNMI/ndarray-str

Add integer and string support for NdArray values

README.md

@@ -39,19 +39,19 @@ from datetime import datetime, timezone
 from pydantic import AwareDatetime
 from covjson_pydantic.coverage import Coverage
 from covjson_pydantic.domain import Domain, Axes, ValuesAxis, DomainType
-from covjson_pydantic.ndarray import NdArray
+from covjson_pydantic.ndarray import NdArrayFloat
 
 c = Coverage(
     domain=Domain(
         domainType=DomainType.point_series,
         axes=Axes(
             x=ValuesAxis[float](values=[1.23]),
             y=ValuesAxis[float](values=[4.56]),
-            t=ValuesAxis[AwareDatetime](values=[datetime.now(tz=timezone.utc)])
-        )
+            t=ValuesAxis[AwareDatetime](values=[datetime(2024, 8, 1, tzinfo=timezone.utc)]),
+        ),
     ),
     ranges={
-        "temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])
+        "temperature": NdArrayFloat(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])
     }
 )
 
@@ -77,7 +77,7 @@ Will print
             },
             "t": {
                 "values": [
-                    "2023-09-14T11:54:02.151493Z"
+                    "2024-08-01T00:00:00Z"
                 ]
             }
         }
@@ -140,8 +140,7 @@ This library is used to build an OGC Environmental Data Retrieval (EDR) API, ser
 ## TODOs
 Help is wanted in the following areas to fully implement the CovJSON spec:
 * The polygon based domain types are not supported.
-* The `Trajectory` and `Section` domain type are not supported.
-* The `NdArray` only supports `float` data.
+* The `Section` domain type is not supported.
 * Not all requirements in the spec relating different fields are implemented.
 
 ## License

example.py

@@ -6,7 +6,7 @@
 from covjson_pydantic.domain import Domain
 from covjson_pydantic.domain import DomainType
 from covjson_pydantic.domain import ValuesAxis
-from covjson_pydantic.ndarray import NdArray
+from covjson_pydantic.ndarray import NdArrayFloat
 from pydantic import AwareDatetime
 
 c = Coverage(
@@ -15,10 +15,10 @@
         axes=Axes(
             x=ValuesAxis[float](values=[1.23]),
             y=ValuesAxis[float](values=[4.56]),
-            t=ValuesAxis[AwareDatetime](values=[datetime.now(tz=timezone.utc)]),
+            t=ValuesAxis[AwareDatetime](values=[datetime(2024, 8, 1, tzinfo=timezone.utc)]),
         ),
     ),
-    ranges={"temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])},
+    ranges={"temperature": NdArrayFloat(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])},
 )
 
 print(c.model_dump_json(exclude_none=True, indent=4))

performance.py

@@ -0,0 +1,22 @@
+import timeit
+from pathlib import Path
+
+filename = Path(__file__).parent.resolve() / "tests" / "test_data" / "coverage-json.json"
+
+setup = f"""
+import json
+from covjson_pydantic.coverage import Coverage
+
+file = "{filename}"
+# Put JSON in default unindented format
+with open(file, "r") as f:
+    data = json.load(f)
+json_string = json.dumps(data, separators=(",", ":"))
+cj = Coverage.model_validate_json(json_string)
+"""
+
+# This can be used to quickly check performance. The first call checks JSON to Python conversion
+# The second call checks Python to JSON conversion
+# Consider generating a larger CoverageJSON file
+print(timeit.timeit("Coverage.model_validate_json(json_string)", setup, number=1000))
+print(timeit.timeit("cj.model_dump_json(exclude_none=True)", setup, number=1000))

pyproject.toml

@@ -20,7 +20,7 @@ classifiers = [
     "Topic :: Scientific/Engineering :: GIS",
     "Typing :: Typed",
 ]
-version = "0.4.0"
+version = "0.5.0"
 dependencies = ["pydantic>=2.3,<3"]
 
 [project.optional-dependencies]

src/covjson_pydantic/coverage.py

@@ -1,28 +1,40 @@
+import sys
+
+if sys.version_info < (3, 9):
+    from typing_extensions import Annotated
+else:
+    from typing import Annotated
+
 from typing import Dict
 from typing import List
 from typing import Literal
 from typing import Optional
 from typing import Union
 
 from pydantic import AnyUrl
+from pydantic import Field
 
 from .base_models import CovJsonBaseModel
 from .domain import Domain
 from .domain import DomainType
-from .ndarray import NdArray
-from .ndarray import TiledNdArray
+from .ndarray import NdArrayFloat
+from .ndarray import NdArrayInt
+from .ndarray import NdArrayStr
+from .ndarray import TiledNdArrayFloat
 from .parameter import Parameter
 from .parameter import ParameterGroup
 from .reference_system import ReferenceSystemConnectionObject
 
+NdArrayTypes = Annotated[Union[NdArrayFloat, NdArrayInt, NdArrayStr], Field(discriminator="dataType")]
+
 
 class Coverage(CovJsonBaseModel, extra="allow"):
     id: Optional[str] = None
     type: Literal["Coverage"] = "Coverage"
     domain: Domain
     parameters: Optional[Dict[str, Parameter]] = None
     parameterGroups: Optional[List[ParameterGroup]] = None  # noqa: N815
-    ranges: Dict[str, Union[NdArray, TiledNdArray, AnyUrl]]
+    ranges: Dict[str, Union[NdArrayTypes, TiledNdArrayFloat, AnyUrl]]
 
 
 class CoverageCollection(CovJsonBaseModel, extra="allow"):

src/covjson_pydantic/ndarray.py

@@ -1,5 +1,4 @@
 import math
-from enum import Enum
 from typing import List
 from typing import Literal
 from typing import Optional
@@ -9,17 +8,20 @@
 from .base_models import CovJsonBaseModel
 
 
-# TODO: Support for integers and strings
-class DataType(str, Enum):
-    float = "float"
-
-
 class NdArray(CovJsonBaseModel, extra="allow"):
     type: Literal["NdArray"] = "NdArray"
-    dataType: DataType = DataType.float  # noqa: N815
+    dataType: str  # Kept here to ensure order of output in JSON  # noqa: N815
     axisNames: Optional[List[str]] = None  # noqa: N815
     shape: Optional[List[int]] = None
-    values: List[Optional[float]]
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_is_sub_class(cls, values):
+        if cls is NdArray:
+            raise TypeError(
+                "NdArray cannot be instantiated directly, please use a NdArrayFloat, NdArrayInt or NdArrayStr"
+            )
+        return values
 
     @model_validator(mode="after")
     def check_field_dependencies(self):
@@ -43,15 +45,31 @@ def check_field_dependencies(self):
         return self
 
 
+class NdArrayFloat(NdArray):
+    dataType: Literal["float"] = "float"  # noqa: N815
+    values: List[Optional[float]]
+
+
+class NdArrayInt(NdArray):
+    dataType: Literal["integer"] = "integer"  # noqa: N815
+    values: List[Optional[int]]
+
+
+class NdArrayStr(NdArray):
+    dataType: Literal["string"] = "string"  # noqa: N815
+    values: List[Optional[str]]
+
+
 class TileSet(CovJsonBaseModel):
     tileShape: List[Optional[int]]  # noqa: N815
     urlTemplate: str  # noqa: N815
 
 
 # TODO: Validation of field dependencies
-class TiledNdArray(CovJsonBaseModel, extra="allow"):
+# TODO: Support string and integer type TiledNdArray
+class TiledNdArrayFloat(CovJsonBaseModel, extra="allow"):
     type: Literal["TiledNdArray"] = "TiledNdArray"
-    dataType: DataType = DataType.float  # noqa: N815
+    dataType: Literal["float"] = "float"  # noqa: N815
     axisNames: List[str]  # noqa: N815
     shape: List[int]
     tileSets: List[TileSet]  # noqa: N815

tests/test_coverage.py

@@ -1,4 +1,6 @@
 import json
+import sys
+from io import StringIO
 from pathlib import Path
 
 import pytest
@@ -7,7 +9,10 @@
 from covjson_pydantic.domain import Axes
 from covjson_pydantic.domain import Domain
 from covjson_pydantic.ndarray import NdArray
-from covjson_pydantic.ndarray import TiledNdArray
+from covjson_pydantic.ndarray import NdArrayFloat
+from covjson_pydantic.ndarray import NdArrayInt
+from covjson_pydantic.ndarray import NdArrayStr
+from covjson_pydantic.ndarray import TiledNdArrayFloat
 from covjson_pydantic.parameter import Parameter
 from covjson_pydantic.parameter import ParameterGroup
 from covjson_pydantic.reference_system import ReferenceSystem
@@ -18,7 +23,9 @@
     ("spec-axes.json", Axes),
     ("str-axes.json", Axes),
     ("coverage-json.json", Coverage),
+    ("coverage-mixed-type-ndarray.json", Coverage),
     ("doc-example-coverage.json", Coverage),
+    ("example_py.json", Coverage),
     ("spec-vertical-profile-coverage.json", Coverage),
     ("spec-trajectory-coverage.json", Coverage),
     ("doc-example-coverage-collection.json", CoverageCollection),
@@ -32,9 +39,11 @@
     ("spec-domain-multipoint-series.json", Domain),
     ("spec-domain-multipoint.json", Domain),
     ("spec-domain-trajectory.json", Domain),
-    ("ndarray-float.json", NdArray),
-    ("spec-ndarray.json", NdArray),
-    ("spec-tiled-ndarray.json", TiledNdArray),
+    ("ndarray-float.json", NdArrayFloat),
+    ("ndarray-string.json", NdArrayStr),
+    ("ndarray-integer.json", NdArrayInt),
+    ("spec-ndarray.json", NdArrayFloat),
+    ("spec-tiled-ndarray.json", TiledNdArrayFloat),
     ("continuous-data-parameter.json", Parameter),
     ("categorical-data-parameter.json", Parameter),
     ("spec-parametergroup.json", ParameterGroup),
@@ -65,6 +74,12 @@ def test_happy_cases(file_name, object_type):
     ("point-series-domain-no-t.json", Domain, r"A 'PointSeries' must have a 't'-axis."),
     ("mixed-type-axes.json", Axes, r"Input should be a valid number"),
     ("mixed-type-axes-2.json", Axes, r"Input should be a valid string"),
+    ("mixed-type-ndarray-1.json", NdArrayFloat, r"Input should be a valid number"),
+    ("mixed-type-ndarray-1.json", NdArrayStr, r"Input should be 'string'"),
+    ("mixed-type-ndarray-2.json", NdArrayFloat, r"Input should be a valid number"),
+    ("mixed-type-ndarray-2.json", NdArrayStr, r"Input should be 'string'"),
+    ("mixed-type-ndarray-3.json", NdArrayInt, r"Input should be a valid integer"),
+    ("mixed-type-ndarray-3.json", NdArrayFloat, r"Input should be 'float'"),
 ]
 
 
@@ -78,3 +93,24 @@ def test_error_cases(file_name, object_type, error_message):
 
     with pytest.raises(ValidationError, match=error_message):
         object_type.model_validate_json(json_string)
+
+
+def test_ndarray_directly():
+    with pytest.raises(TypeError, match="NdArray cannot be instantiated directly"):
+        NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])
+
+
+def test_example_py():
+    file = Path(__file__).parent.parent.resolve() / "example.py"
+
+    with open(file, "r") as f:
+        code = f.read()
+
+    old_stdout = sys.stdout
+    sys.stdout = my_stdout = StringIO()
+    exec(code)
+    sys.stdout = old_stdout
+
+    file = Path(__file__).parent.resolve() / "test_data" / "example_py.json"
+    with open(file, "r") as f:
+        assert my_stdout.getvalue() == f.read()