diff --git a/README.md b/README.md index 0d95722..1c7f5a7 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ from datetime import datetime, timezone from pydantic import AwareDatetime from covjson_pydantic.coverage import Coverage from covjson_pydantic.domain import Domain, Axes, ValuesAxis, DomainType -from covjson_pydantic.ndarray import NdArray +from covjson_pydantic.ndarray import NdArrayFloat c = Coverage( domain=Domain( @@ -47,11 +47,11 @@ c = Coverage( axes=Axes( x=ValuesAxis[float](values=[1.23]), y=ValuesAxis[float](values=[4.56]), - t=ValuesAxis[AwareDatetime](values=[datetime.now(tz=timezone.utc)]) - ) + t=ValuesAxis[AwareDatetime](values=[datetime(2024, 8, 1, tzinfo=timezone.utc)]), + ), ), ranges={ - "temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0]) + "temperature": NdArrayFloat(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0]) } ) @@ -77,7 +77,7 @@ Will print }, "t": { "values": [ - "2023-09-14T11:54:02.151493Z" + "2024-08-01T00:00:00Z" ] } } @@ -140,8 +140,7 @@ This library is used to build an OGC Environmental Data Retrieval (EDR) API, ser ## TODOs Help is wanted in the following areas to fully implement the CovJSON spec: * The polygon based domain types are not supported. -* The `Trajectory` and `Section` domain type are not supported. -* The `NdArray` only supports `float` data. +* The `Section` domain type is not supported. * Not all requirements in the spec relating different fields are implemented. ## License diff --git a/example.py b/example.py index a494f56..1458ac2 100644 --- a/example.py +++ b/example.py @@ -6,7 +6,7 @@ from covjson_pydantic.domain import Domain from covjson_pydantic.domain import DomainType from covjson_pydantic.domain import ValuesAxis -from covjson_pydantic.ndarray import NdArray +from covjson_pydantic.ndarray import NdArrayFloat from pydantic import AwareDatetime c = Coverage( @@ -15,10 +15,10 @@ axes=Axes( x=ValuesAxis[float](values=[1.23]), y=ValuesAxis[float](values=[4.56]), - t=ValuesAxis[AwareDatetime](values=[datetime.now(tz=timezone.utc)]), + t=ValuesAxis[AwareDatetime](values=[datetime(2024, 8, 1, tzinfo=timezone.utc)]), ), ), - ranges={"temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])}, + ranges={"temperature": NdArrayFloat(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])}, ) print(c.model_dump_json(exclude_none=True, indent=4)) diff --git a/performance.py b/performance.py new file mode 100644 index 0000000..9dfa615 --- /dev/null +++ b/performance.py @@ -0,0 +1,22 @@ +import timeit +from pathlib import Path + +filename = Path(__file__).parent.resolve() / "tests" / "test_data" / "coverage-json.json" + +setup = f""" +import json +from covjson_pydantic.coverage import Coverage + +file = "{filename}" +# Put JSON in default unindented format +with open(file, "r") as f: + data = json.load(f) +json_string = json.dumps(data, separators=(",", ":")) +cj = Coverage.model_validate_json(json_string) +""" + +# This can be used to quickly check performance. The first call checks JSON to Python conversion +# The second call checks Python to JSON conversion +# Consider generating a larger CoverageJSON file +print(timeit.timeit("Coverage.model_validate_json(json_string)", setup, number=1000)) +print(timeit.timeit("cj.model_dump_json(exclude_none=True)", setup, number=1000)) diff --git a/pyproject.toml b/pyproject.toml index 8c8bd72..b9ab1cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: GIS", "Typing :: Typed", ] -version = "0.4.0" +version = "0.5.0" dependencies = ["pydantic>=2.3,<3"] [project.optional-dependencies] diff --git a/src/covjson_pydantic/coverage.py b/src/covjson_pydantic/coverage.py index 279240b..c43baaa 100644 --- a/src/covjson_pydantic/coverage.py +++ b/src/covjson_pydantic/coverage.py @@ -1,3 +1,10 @@ +import sys + +if sys.version_info < (3, 9): + from typing_extensions import Annotated +else: + from typing import Annotated + from typing import Dict from typing import List from typing import Literal @@ -5,16 +12,21 @@ from typing import Union from pydantic import AnyUrl +from pydantic import Field from .base_models import CovJsonBaseModel from .domain import Domain from .domain import DomainType -from .ndarray import NdArray -from .ndarray import TiledNdArray +from .ndarray import NdArrayFloat +from .ndarray import NdArrayInt +from .ndarray import NdArrayStr +from .ndarray import TiledNdArrayFloat from .parameter import Parameter from .parameter import ParameterGroup from .reference_system import ReferenceSystemConnectionObject +NdArrayTypes = Annotated[Union[NdArrayFloat, NdArrayInt, NdArrayStr], Field(discriminator="dataType")] + class Coverage(CovJsonBaseModel, extra="allow"): id: Optional[str] = None @@ -22,7 +34,7 @@ class Coverage(CovJsonBaseModel, extra="allow"): domain: Domain parameters: Optional[Dict[str, Parameter]] = None parameterGroups: Optional[List[ParameterGroup]] = None # noqa: N815 - ranges: Dict[str, Union[NdArray, TiledNdArray, AnyUrl]] + ranges: Dict[str, Union[NdArrayTypes, TiledNdArrayFloat, AnyUrl]] class CoverageCollection(CovJsonBaseModel, extra="allow"): diff --git a/src/covjson_pydantic/ndarray.py b/src/covjson_pydantic/ndarray.py index abda788..754d0d8 100644 --- a/src/covjson_pydantic/ndarray.py +++ b/src/covjson_pydantic/ndarray.py @@ -1,5 +1,4 @@ import math -from enum import Enum from typing import List from typing import Literal from typing import Optional @@ -9,17 +8,20 @@ from .base_models import CovJsonBaseModel -# TODO: Support for integers and strings -class DataType(str, Enum): - float = "float" - - class NdArray(CovJsonBaseModel, extra="allow"): type: Literal["NdArray"] = "NdArray" - dataType: DataType = DataType.float # noqa: N815 + dataType: str # Kept here to ensure order of output in JSON # noqa: N815 axisNames: Optional[List[str]] = None # noqa: N815 shape: Optional[List[int]] = None - values: List[Optional[float]] + + @model_validator(mode="before") + @classmethod + def validate_is_sub_class(cls, values): + if cls is NdArray: + raise TypeError( + "NdArray cannot be instantiated directly, please use a NdArrayFloat, NdArrayInt or NdArrayStr" + ) + return values @model_validator(mode="after") def check_field_dependencies(self): @@ -43,15 +45,31 @@ def check_field_dependencies(self): return self +class NdArrayFloat(NdArray): + dataType: Literal["float"] = "float" # noqa: N815 + values: List[Optional[float]] + + +class NdArrayInt(NdArray): + dataType: Literal["integer"] = "integer" # noqa: N815 + values: List[Optional[int]] + + +class NdArrayStr(NdArray): + dataType: Literal["string"] = "string" # noqa: N815 + values: List[Optional[str]] + + class TileSet(CovJsonBaseModel): tileShape: List[Optional[int]] # noqa: N815 urlTemplate: str # noqa: N815 # TODO: Validation of field dependencies -class TiledNdArray(CovJsonBaseModel, extra="allow"): +# TODO: Support string and integer type TiledNdArray +class TiledNdArrayFloat(CovJsonBaseModel, extra="allow"): type: Literal["TiledNdArray"] = "TiledNdArray" - dataType: DataType = DataType.float # noqa: N815 + dataType: Literal["float"] = "float" # noqa: N815 axisNames: List[str] # noqa: N815 shape: List[int] tileSets: List[TileSet] # noqa: N815 diff --git a/tests/test_coverage.py b/tests/test_coverage.py index c862a74..cb36f4c 100644 --- a/tests/test_coverage.py +++ b/tests/test_coverage.py @@ -1,4 +1,6 @@ import json +import sys +from io import StringIO from pathlib import Path import pytest @@ -7,7 +9,10 @@ from covjson_pydantic.domain import Axes from covjson_pydantic.domain import Domain from covjson_pydantic.ndarray import NdArray -from covjson_pydantic.ndarray import TiledNdArray +from covjson_pydantic.ndarray import NdArrayFloat +from covjson_pydantic.ndarray import NdArrayInt +from covjson_pydantic.ndarray import NdArrayStr +from covjson_pydantic.ndarray import TiledNdArrayFloat from covjson_pydantic.parameter import Parameter from covjson_pydantic.parameter import ParameterGroup from covjson_pydantic.reference_system import ReferenceSystem @@ -18,7 +23,9 @@ ("spec-axes.json", Axes), ("str-axes.json", Axes), ("coverage-json.json", Coverage), + ("coverage-mixed-type-ndarray.json", Coverage), ("doc-example-coverage.json", Coverage), + ("example_py.json", Coverage), ("spec-vertical-profile-coverage.json", Coverage), ("spec-trajectory-coverage.json", Coverage), ("doc-example-coverage-collection.json", CoverageCollection), @@ -32,9 +39,11 @@ ("spec-domain-multipoint-series.json", Domain), ("spec-domain-multipoint.json", Domain), ("spec-domain-trajectory.json", Domain), - ("ndarray-float.json", NdArray), - ("spec-ndarray.json", NdArray), - ("spec-tiled-ndarray.json", TiledNdArray), + ("ndarray-float.json", NdArrayFloat), + ("ndarray-string.json", NdArrayStr), + ("ndarray-integer.json", NdArrayInt), + ("spec-ndarray.json", NdArrayFloat), + ("spec-tiled-ndarray.json", TiledNdArrayFloat), ("continuous-data-parameter.json", Parameter), ("categorical-data-parameter.json", Parameter), ("spec-parametergroup.json", ParameterGroup), @@ -65,6 +74,12 @@ def test_happy_cases(file_name, object_type): ("point-series-domain-no-t.json", Domain, r"A 'PointSeries' must have a 't'-axis."), ("mixed-type-axes.json", Axes, r"Input should be a valid number"), ("mixed-type-axes-2.json", Axes, r"Input should be a valid string"), + ("mixed-type-ndarray-1.json", NdArrayFloat, r"Input should be a valid number"), + ("mixed-type-ndarray-1.json", NdArrayStr, r"Input should be 'string'"), + ("mixed-type-ndarray-2.json", NdArrayFloat, r"Input should be a valid number"), + ("mixed-type-ndarray-2.json", NdArrayStr, r"Input should be 'string'"), + ("mixed-type-ndarray-3.json", NdArrayInt, r"Input should be a valid integer"), + ("mixed-type-ndarray-3.json", NdArrayFloat, r"Input should be 'float'"), ] @@ -78,3 +93,24 @@ def test_error_cases(file_name, object_type, error_message): with pytest.raises(ValidationError, match=error_message): object_type.model_validate_json(json_string) + + +def test_ndarray_directly(): + with pytest.raises(TypeError, match="NdArray cannot be instantiated directly"): + NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0]) + + +def test_example_py(): + file = Path(__file__).parent.parent.resolve() / "example.py" + + with open(file, "r") as f: + code = f.read() + + old_stdout = sys.stdout + sys.stdout = my_stdout = StringIO() + exec(code) + sys.stdout = old_stdout + + file = Path(__file__).parent.resolve() / "test_data" / "example_py.json" + with open(file, "r") as f: + assert my_stdout.getvalue() == f.read() diff --git a/tests/test_data/coverage-mixed-type-ndarray.json b/tests/test_data/coverage-mixed-type-ndarray.json new file mode 100644 index 0000000..e8c1132 --- /dev/null +++ b/tests/test_data/coverage-mixed-type-ndarray.json @@ -0,0 +1,139 @@ +{ + "type": "Coverage", + "domain": { + "type": "Domain", + "domainType": "PointSeries", + "axes": { + "x": { + "values": [ + 5.3 + ] + }, + "y": { + "values": [ + 53.2 + ] + }, + "t": { + "values": [ + "2022-01-01T04:10:00Z", + "2022-01-01T04:20:00Z", + "2022-01-01T04:30:00Z" + ] + } + } + }, + "parameters": { + "float-parameter": { + "type": "Parameter", + "observedProperty": { + "label": { + "en": "float" + } + } + }, + "string-parameter": { + "type": "Parameter", + "observedProperty": { + "label": { + "en": "string" + } + } + }, + "integer-parameter": { + "type": "Parameter", + "observedProperty": { + "label": { + "en": "integer" + } + } + }, + "null-parameter": { + "type": "Parameter", + "observedProperty": { + "label": { + "en": "null" + } + } + } + }, + "ranges": { + "string-parameter": { + "type": "NdArray", + "dataType": "string", + "axisNames": [ + "x", + "y", + "t" + ], + "shape": [ + 1, + 1, + 3 + ], + "values": [ + null, + "foo", + "bar" + ] + }, + "float-parameter": { + "type": "NdArray", + "dataType": "float", + "axisNames": [ + "x", + "y", + "t" + ], + "shape": [ + 1, + 1, + 3 + ], + "values": [ + 62.0, + null, + 63.136801411019825 + ] + }, + "integer-parameter": { + "type": "NdArray", + "dataType": "integer", + "axisNames": [ + "x", + "y", + "t" + ], + "shape": [ + 1, + 1, + 3 + ], + "values": [ + 1, + null, + 3 + ] + }, + "null-parameter": { + "type": "NdArray", + "dataType": "integer", + "axisNames": [ + "x", + "y", + "t" + ], + "shape": [ + 1, + 1, + 3 + ], + "values": [ + null, + null, + null + ] + } + }, + "extra:extra": "extra fields allowed" +} diff --git a/tests/test_data/example_py.json b/tests/test_data/example_py.json new file mode 100644 index 0000000..4483a79 --- /dev/null +++ b/tests/test_data/example_py.json @@ -0,0 +1,43 @@ +{ + "type": "Coverage", + "domain": { + "type": "Domain", + "domainType": "PointSeries", + "axes": { + "x": { + "values": [ + 1.23 + ] + }, + "y": { + "values": [ + 4.56 + ] + }, + "t": { + "values": [ + "2024-08-01T00:00:00Z" + ] + } + } + }, + "ranges": { + "temperature": { + "type": "NdArray", + "dataType": "float", + "axisNames": [ + "x", + "y", + "t" + ], + "shape": [ + 1, + 1, + 1 + ], + "values": [ + 42.0 + ] + } + } +} diff --git a/tests/test_data/mixed-type-ndarray-1.json b/tests/test_data/mixed-type-ndarray-1.json new file mode 100644 index 0000000..3a48cf6 --- /dev/null +++ b/tests/test_data/mixed-type-ndarray-1.json @@ -0,0 +1,15 @@ +{ + "type": "NdArray", + "dataType": "float", + "axisNames": [ + "y", + "x" + ], + "shape": [ + 2 + ], + "values": [ + "42.0", + 123 + ] +} diff --git a/tests/test_data/mixed-type-ndarray-2.json b/tests/test_data/mixed-type-ndarray-2.json new file mode 100644 index 0000000..b35e50b --- /dev/null +++ b/tests/test_data/mixed-type-ndarray-2.json @@ -0,0 +1,15 @@ +{ + "type": "NdArray", + "dataType": "float", + "axisNames": [ + "y", + "x" + ], + "shape": [ + 2 + ], + "values": [ + "foo", + "bar" + ] +} diff --git a/tests/test_data/mixed-type-ndarray-3.json b/tests/test_data/mixed-type-ndarray-3.json new file mode 100644 index 0000000..2d30d0a --- /dev/null +++ b/tests/test_data/mixed-type-ndarray-3.json @@ -0,0 +1,15 @@ +{ + "type": "NdArray", + "dataType": "integer", + "axisNames": [ + "y", + "x" + ], + "shape": [ + 2 + ], + "values": [ + 1, + 1.42 + ] +} diff --git a/tests/test_data/ndarray-integer.json b/tests/test_data/ndarray-integer.json new file mode 100644 index 0000000..006cf5e --- /dev/null +++ b/tests/test_data/ndarray-integer.json @@ -0,0 +1,19 @@ +{ + "type": "NdArray", + "dataType": "integer", + "axisNames": [ + "t", + "y", + "x" + ], + "shape": [ + 1, + 1, + 3 + ], + "values": [ + 1, + 2, + 42 + ] +} diff --git a/tests/test_data/ndarray-string.json b/tests/test_data/ndarray-string.json new file mode 100644 index 0000000..b021fe0 --- /dev/null +++ b/tests/test_data/ndarray-string.json @@ -0,0 +1,22 @@ +{ + "type": "NdArray", + "dataType": "string", + "axisNames": [ + "t", + "y", + "x" + ], + "shape": [ + 1, + 2, + 3 + ], + "values": [ + "ABC", + "DEF", + null, + "XYZ", + "a123", + "qwerty" + ] +}