From b66c0c3bc807497aed05e4ba1bd0d04ad5587897 Mon Sep 17 00:00:00 2001 From: Lukas Phaf Date: Thu, 18 Jan 2024 11:19:46 +0100 Subject: [PATCH 1/4] Fix (hack?) so that the JSON schema/swagger spec does not contain oneOf null types for optional fields. --- src/covjson_pydantic/base_models.py | 11 +++++++++++ src/covjson_pydantic/coverage.py | 16 ++++++++-------- src/covjson_pydantic/domain.py | 22 +++++++++++----------- src/covjson_pydantic/ndarray.py | 5 +++-- src/covjson_pydantic/observed_property.py | 10 +++++----- src/covjson_pydantic/parameter.py | 20 ++++++++++---------- src/covjson_pydantic/reference_system.py | 20 ++++++++++---------- src/covjson_pydantic/unit.py | 8 ++++---- 8 files changed, 62 insertions(+), 50 deletions(-) diff --git a/src/covjson_pydantic/base_models.py b/src/covjson_pydantic/base_models.py index 4c1c3eb..26b4768 100644 --- a/src/covjson_pydantic/base_models.py +++ b/src/covjson_pydantic/base_models.py @@ -1,5 +1,16 @@ +from typing import Annotated +from typing import TypeVar +from typing import Union + from pydantic import BaseModel as PydanticBaseModel from pydantic import ConfigDict +from pydantic import Field +from pydantic.json_schema import SkipJsonSchema + + +# Define an alternative Optional type that doesn't show the None/null value and the default in the schema +T = TypeVar("T") +OptionalS = Annotated[Union[T, SkipJsonSchema[None]], Field(json_schema_extra=lambda x: x.pop("default"))] class CovJsonBaseModel(PydanticBaseModel): diff --git a/src/covjson_pydantic/coverage.py b/src/covjson_pydantic/coverage.py index 279240b..f25bcab 100644 --- a/src/covjson_pydantic/coverage.py +++ b/src/covjson_pydantic/coverage.py @@ -1,12 +1,12 @@ from typing import Dict from typing import List from typing import Literal -from typing import Optional from typing import Union from pydantic import AnyUrl from .base_models import CovJsonBaseModel +from .base_models import OptionalS from .domain import Domain from .domain import DomainType from .ndarray import NdArray @@ -17,18 +17,18 @@ class Coverage(CovJsonBaseModel, extra="allow"): - id: Optional[str] = None + id: OptionalS[str] = None type: Literal["Coverage"] = "Coverage" domain: Domain - parameters: Optional[Dict[str, Parameter]] = None - parameterGroups: Optional[List[ParameterGroup]] = None # noqa: N815 + parameters: OptionalS[Dict[str, Parameter]] = None + parameterGroups: OptionalS[List[ParameterGroup]] = None # noqa: N815 ranges: Dict[str, Union[NdArray, TiledNdArray, AnyUrl]] class CoverageCollection(CovJsonBaseModel, extra="allow"): type: Literal["CoverageCollection"] = "CoverageCollection" - domainType: Optional[DomainType] = None # noqa: N815 + domainType: OptionalS[DomainType] = None # noqa: N815 coverages: List[Coverage] - parameters: Optional[Dict[str, Parameter]] = None - parameterGroups: Optional[List[ParameterGroup]] = None # noqa: N815 - referencing: Optional[List[ReferenceSystemConnectionObject]] = None + parameters: OptionalS[Dict[str, Parameter]] = None + parameterGroups: OptionalS[List[ParameterGroup]] = None # noqa: N815 + referencing: OptionalS[List[ReferenceSystemConnectionObject]] = None diff --git a/src/covjson_pydantic/domain.py b/src/covjson_pydantic/domain.py index 61dfdfb..b5fc3c3 100644 --- a/src/covjson_pydantic/domain.py +++ b/src/covjson_pydantic/domain.py @@ -2,7 +2,6 @@ from typing import Generic from typing import List from typing import Literal -from typing import Optional from typing import Tuple from typing import TypeVar from typing import Union @@ -13,6 +12,7 @@ from pydantic import PositiveInt from .base_models import CovJsonBaseModel +from .base_models import OptionalS from .reference_system import ReferenceSystemConnectionObject @@ -34,10 +34,10 @@ def single_value_case(self): # Combination between Generics (ValuesT) and datetime and strict mode causes issues between JSON <-> Pydantic # conversions. Strict mode has been disabled. Issue: https://github.com/KNMI/covjson-pydantic/issues/4 class ValuesAxis(CovJsonBaseModel, Generic[ValuesT], extra="allow", strict=False): - dataType: Optional[str] = None # noqa: N815 - coordinates: Optional[List[str]] = None + dataType: OptionalS[str] = None # noqa: N815 + coordinates: OptionalS[List[str]] = None values: List[ValuesT] - bounds: Optional[List[ValuesT]] = None + bounds: OptionalS[List[ValuesT]] = None @model_validator(mode="after") def bounds_length(self): @@ -56,11 +56,11 @@ class DomainType(str, Enum): class Axes(CovJsonBaseModel): - x: Optional[Union[ValuesAxis[float], CompactAxis]] = None - y: Optional[Union[ValuesAxis[float], CompactAxis]] = None - z: Optional[Union[ValuesAxis[float], CompactAxis]] = None - t: Optional[ValuesAxis[AwareDatetime]] = None - composite: Optional[ValuesAxis[Tuple]] = None + x: OptionalS[Union[ValuesAxis[float], CompactAxis]] = None + y: OptionalS[Union[ValuesAxis[float], CompactAxis]] = None + z: OptionalS[Union[ValuesAxis[float], CompactAxis]] = None + t: OptionalS[ValuesAxis[AwareDatetime]] = None + composite: OptionalS[ValuesAxis[Tuple]] = None @model_validator(mode="after") def at_least_one_axes(self): @@ -71,9 +71,9 @@ def at_least_one_axes(self): class Domain(CovJsonBaseModel, extra="allow"): type: Literal["Domain"] = "Domain" - domainType: Optional[DomainType] = None # noqa: N815 + domainType: OptionalS[DomainType] = None # noqa: N815 axes: Axes - referencing: Optional[List[ReferenceSystemConnectionObject]] = None + referencing: OptionalS[List[ReferenceSystemConnectionObject]] = None # TODO: This is a workaround to allow domainType to work in strict mode, in combination with FastAPI. # See: https://github.com/tiangolo/fastapi/discussions/9868 diff --git a/src/covjson_pydantic/ndarray.py b/src/covjson_pydantic/ndarray.py index abda788..5321563 100644 --- a/src/covjson_pydantic/ndarray.py +++ b/src/covjson_pydantic/ndarray.py @@ -7,6 +7,7 @@ from pydantic import model_validator from .base_models import CovJsonBaseModel +from .base_models import OptionalS # TODO: Support for integers and strings @@ -17,8 +18,8 @@ class DataType(str, Enum): class NdArray(CovJsonBaseModel, extra="allow"): type: Literal["NdArray"] = "NdArray" dataType: DataType = DataType.float # noqa: N815 - axisNames: Optional[List[str]] = None # noqa: N815 - shape: Optional[List[int]] = None + axisNames: OptionalS[List[str]] = None # noqa: N815 + shape: OptionalS[List[int]] = None values: List[Optional[float]] @model_validator(mode="after") diff --git a/src/covjson_pydantic/observed_property.py b/src/covjson_pydantic/observed_property.py index 976a067..461326e 100644 --- a/src/covjson_pydantic/observed_property.py +++ b/src/covjson_pydantic/observed_property.py @@ -1,18 +1,18 @@ from typing import List -from typing import Optional from .base_models import CovJsonBaseModel +from .base_models import OptionalS from .i18n import i18n class Category(CovJsonBaseModel): id: str label: i18n - description: Optional[i18n] = None + description: OptionalS[i18n] = None class ObservedProperty(CovJsonBaseModel): - id: Optional[str] = None + id: OptionalS[str] = None label: i18n - description: Optional[i18n] = None - categories: Optional[List[Category]] = None + description: OptionalS[i18n] = None + categories: OptionalS[List[Category]] = None diff --git a/src/covjson_pydantic/parameter.py b/src/covjson_pydantic/parameter.py index 9f684cc..69df921 100644 --- a/src/covjson_pydantic/parameter.py +++ b/src/covjson_pydantic/parameter.py @@ -1,12 +1,12 @@ from typing import Dict from typing import List from typing import Literal -from typing import Optional from typing import Union from pydantic import model_validator from .base_models import CovJsonBaseModel +from .base_models import OptionalS from .i18n import i18n from .observed_property import ObservedProperty from .unit import Unit @@ -14,12 +14,12 @@ class Parameter(CovJsonBaseModel, extra="allow"): type: Literal["Parameter"] = "Parameter" - id: Optional[str] = None - label: Optional[i18n] = None - description: Optional[i18n] = None + id: OptionalS[str] = None + label: OptionalS[i18n] = None + description: OptionalS[i18n] = None observedProperty: ObservedProperty # noqa: N815 - categoryEncoding: Optional[Dict[str, Union[int, List[int]]]] = None # noqa: N815 - unit: Optional[Unit] = None + categoryEncoding: OptionalS[Dict[str, Union[int, List[int]]]] = None # noqa: N815 + unit: OptionalS[Unit] = None @model_validator(mode="after") def must_not_have_unit_if_observed_property_has_categories(self): @@ -34,10 +34,10 @@ def must_not_have_unit_if_observed_property_has_categories(self): class ParameterGroup(CovJsonBaseModel, extra="allow"): type: Literal["ParameterGroup"] = "ParameterGroup" - id: Optional[str] = None - label: Optional[i18n] = None - description: Optional[i18n] = None - observedProperty: Optional[ObservedProperty] = None # noqa: N815 + id: OptionalS[str] = None + label: OptionalS[i18n] = None + description: OptionalS[i18n] = None + observedProperty: OptionalS[ObservedProperty] = None # noqa: N815 members: List[str] @model_validator(mode="after") diff --git a/src/covjson_pydantic/reference_system.py b/src/covjson_pydantic/reference_system.py index 7222330..1ca7adc 100644 --- a/src/covjson_pydantic/reference_system.py +++ b/src/covjson_pydantic/reference_system.py @@ -1,35 +1,35 @@ from typing import Dict from typing import List from typing import Literal -from typing import Optional from typing import Union from pydantic import AnyUrl from pydantic import model_validator from .base_models import CovJsonBaseModel +from .base_models import OptionalS from .i18n import i18n class TargetConcept(CovJsonBaseModel): - id: Optional[str] = None # Not in spec, but needed for example in spec for 'Identifier-based Reference Systems' + id: OptionalS[str] = None # Not in spec, but needed for example in spec for 'Identifier-based Reference Systems' label: i18n - description: Optional[i18n] = None + description: OptionalS[i18n] = None class ReferenceSystem(CovJsonBaseModel, extra="allow"): type: Literal["GeographicCRS", "ProjectedCRS", "VerticalCRS", "TemporalRS", "IdentifierRS"] - id: Optional[str] = None - description: Optional[i18n] = None + id: OptionalS[str] = None + description: OptionalS[i18n] = None # Only for TemporalRS - calendar: Optional[Union[Literal["Gregorian"], AnyUrl]] = None - timeScale: Optional[AnyUrl] = None # noqa: N815 + calendar: OptionalS[Union[Literal["Gregorian"], AnyUrl]] = None + timeScale: OptionalS[AnyUrl] = None # noqa: N815 # Only for IdentifierRS - label: Optional[i18n] = None - targetConcept: Optional[TargetConcept] = None # noqa: N815 - identifiers: Optional[Dict[str, TargetConcept]] = None + label: OptionalS[i18n] = None + targetConcept: OptionalS[TargetConcept] = None # noqa: N815 + identifiers: OptionalS[Dict[str, TargetConcept]] = None @model_validator(mode="after") def check_type_specific_fields(self): diff --git a/src/covjson_pydantic/unit.py b/src/covjson_pydantic/unit.py index f1783dd..d2edda4 100644 --- a/src/covjson_pydantic/unit.py +++ b/src/covjson_pydantic/unit.py @@ -1,9 +1,9 @@ -from typing import Optional from typing import Union from pydantic import model_validator from .base_models import CovJsonBaseModel +from .base_models import OptionalS from .i18n import i18n @@ -13,9 +13,9 @@ class Symbol(CovJsonBaseModel): class Unit(CovJsonBaseModel): - id: Optional[str] = None - label: Optional[i18n] = None - symbol: Optional[Union[str, Symbol]] = None + id: OptionalS[str] = None + label: OptionalS[i18n] = None + symbol: OptionalS[Union[str, Symbol]] = None @model_validator(mode="after") def check_either_label_or_symbol(self): From 2c733f3169cde0a330021eb4ad225b0bde31f70b Mon Sep 17 00:00:00 2001 From: Lukas Phaf Date: Thu, 18 Jan 2024 11:23:02 +0100 Subject: [PATCH 2/4] Drop Python version 3.8 --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb550ca..195609d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] +# python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v3 From e72f16b33db8c26e3af8331956eb145350427ddf Mon Sep 17 00:00:00 2001 From: Lukas Phaf Date: Thu, 18 Jan 2024 11:41:29 +0100 Subject: [PATCH 3/4] Add partial functionality for Python 3.8. --- .github/workflows/ci.yml | 3 +-- src/covjson_pydantic/base_models.py | 8 ++++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 195609d..dfbfd3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,8 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: -# python-version: ['3.8', '3.9', '3.10', '3.11'] - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v3 diff --git a/src/covjson_pydantic/base_models.py b/src/covjson_pydantic/base_models.py index 26b4768..959e105 100644 --- a/src/covjson_pydantic/base_models.py +++ b/src/covjson_pydantic/base_models.py @@ -1,4 +1,5 @@ -from typing import Annotated +# from typing import Annotated +import typing from typing import TypeVar from typing import Union @@ -10,7 +11,10 @@ # Define an alternative Optional type that doesn't show the None/null value and the default in the schema T = TypeVar("T") -OptionalS = Annotated[Union[T, SkipJsonSchema[None]], Field(json_schema_extra=lambda x: x.pop("default"))] +if hasattr(typing, "Annotated"): # Check if Annotated exists (Python >=3.9) + OptionalS = typing.Annotated[Union[T, SkipJsonSchema[None]], Field(json_schema_extra=lambda x: x.pop("default"))] +else: + OptionalS = Union[T, SkipJsonSchema[None]] # For Python 3.8 we don't support dropping the default value class CovJsonBaseModel(PydanticBaseModel): From eefe8cff769d2906a06a1d48c9d84eee75975272 Mon Sep 17 00:00:00 2001 From: Lukas Phaf Date: Thu, 18 Jan 2024 11:57:35 +0100 Subject: [PATCH 4/4] Fix mypy. --- src/covjson_pydantic/base_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/covjson_pydantic/base_models.py b/src/covjson_pydantic/base_models.py index 959e105..b1d2a32 100644 --- a/src/covjson_pydantic/base_models.py +++ b/src/covjson_pydantic/base_models.py @@ -13,8 +13,8 @@ T = TypeVar("T") if hasattr(typing, "Annotated"): # Check if Annotated exists (Python >=3.9) OptionalS = typing.Annotated[Union[T, SkipJsonSchema[None]], Field(json_schema_extra=lambda x: x.pop("default"))] -else: - OptionalS = Union[T, SkipJsonSchema[None]] # For Python 3.8 we don't support dropping the default value +else: # For Python 3.8 we don't support dropping the default value + OptionalS = Union[T, SkipJsonSchema[None]] # type: ignore class CovJsonBaseModel(PydanticBaseModel):