From 88f4359fbb91e156adf3fd9afb1d9953411ea305 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Mon, 7 Aug 2023 17:31:38 +0200 Subject: [PATCH] Issue #424 initial support for load_url (vector cubes) --- CHANGELOG.md | 1 + openeo/rest/connection.py | 25 ++++++++++++++++++++++++- openeo/rest/vectorcube.py | 25 ++++++++++++++++++++++++- tests/rest/datacube/test_vectorcube.py | 13 +++++++++++++ tests/rest/test_connection.py | 19 +++++++++++++++++++ 5 files changed, 81 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec642f337..a54650d5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Initial `load_geojson` support with `Connection.load_geojson()` ([#424](https://github.com/Open-EO/openeo-python-client/issues/424)) +- Initial `load_url` (for vector cubes) support with `Connection.load_url()` ([#424](https://github.com/Open-EO/openeo-python-client/issues/424)) ### Changed diff --git a/openeo/rest/connection.py b/openeo/rest/connection.py index bcdceb569..535212764 100644 --- a/openeo/rest/connection.py +++ b/openeo/rest/connection.py @@ -802,6 +802,9 @@ def capabilities(self) -> RESTCapabilities: load=lambda: RESTCapabilities(data=self.get('/', expected_status=200).json(), url=self._orig_url) ) + def list_input_formats(self) -> dict: + return self.list_file_formats().get("input", {}) + def list_output_formats(self) -> dict: return self.list_file_formats().get("output", {}) @@ -1319,7 +1322,6 @@ def load_geojson( """ Converts GeoJSON data as defined by RFC 7946 into a vector data cube. - :param connection: the connection to use to connect with the openEO back-end. :param data: the geometry to load. One of: - GeoJSON-style data structure: e.g. a dictionary with ``"type": "Polygon"`` and ``"coordinates"`` fields @@ -1337,6 +1339,27 @@ def load_geojson( return VectorCube.load_geojson(connection=self, data=data, properties=properties) + @openeo_process + def load_url(self, url: str, format: str, options: Optional[dict] = None): + """ + Loads a file from a URL + + :param url: The URL to read from. Authentication details such as API keys or tokens may need to be included in the URL. + :param format: The file format to use when loading the data. + :param options: The file format parameters to use when reading the data. + Must correspond to the parameters that the server reports as supported parameters for the chosen ``format`` + :return: new VectorCube instance + + .. warning:: EXPERIMENTAL: this process is experimental with the potential for major things to change. + + .. versionadded:: 0.22.0 + """ + if format not in self.list_input_formats(): + # TODO: make this an error? + _log.warning(f"Format {format!r} not listed in back-end input formats") + # TODO: Inspect format's gis_data_type to see if we need to load a VectorCube or classic raster DataCube + return VectorCube.load_url(connection=self, url=url, format=format, options=options) + def create_service(self, graph: dict, type: str, **kwargs) -> Service: # TODO: type hint for graph: is it a nested or a flat one? req = self._build_request_with_process_graph(process_graph=graph, type=type, **kwargs) diff --git a/openeo/rest/vectorcube.py b/openeo/rest/vectorcube.py index 32fa5c5f2..1d9d3f919 100644 --- a/openeo/rest/vectorcube.py +++ b/openeo/rest/vectorcube.py @@ -58,7 +58,7 @@ def load_geojson( connection: "openeo.Connection", data: Union[dict, str, pathlib.Path, shapely.geometry.base.BaseGeometry, Parameter], properties: Optional[List[str]] = None, - ): + ) -> "VectorCube": """ Converts GeoJSON data as defined by RFC 7946 into a vector data cube. @@ -78,6 +78,7 @@ def load_geojson( .. versionadded:: 0.22.0 """ # TODO: unify with `DataCube._get_geometry_argument` + # TODO: also support client side fetching of GeoJSON from URL? if isinstance(data, str) and data.strip().startswith("{"): # Assume JSON dump geometry = json.loads(data) @@ -98,6 +99,28 @@ def load_geojson( pg = PGNode(process_id="load_geojson", data=geometry, properties=properties or []) return cls(graph=pg, connection=connection) + @classmethod + @openeo_process + def load_url( + cls, connection: "openeo.Connection", url: str, format: str, options: Optional[dict] = None + ) -> "VectorCube": + """ + Loads a file from a URL + + :param connection: the connection to use to connect with the openEO back-end. + :param url: The URL to read from. Authentication details such as API keys or tokens may need to be included in the URL. + :param format: The file format to use when loading the data. + :param options: The file format parameters to use when reading the data. + Must correspond to the parameters that the server reports as supported parameters for the chosen ``format`` + :return: new VectorCube instance + + .. warning:: EXPERIMENTAL: this process is experimental with the potential for major things to change. + + .. versionadded:: 0.22.0 + """ + pg = PGNode(process_id="load_url", arguments=dict_no_none(url=url, format=format, options=options)) + return cls(graph=pg, connection=connection) + @openeo_process def run_udf( self, diff --git a/tests/rest/datacube/test_vectorcube.py b/tests/rest/datacube/test_vectorcube.py index 995f66af0..5f5813dc6 100644 --- a/tests/rest/datacube/test_vectorcube.py +++ b/tests/rest/datacube/test_vectorcube.py @@ -240,3 +240,16 @@ def test_load_geojson_parameter(con100, dummy_backend): "result": True, } } + + +def test_load_url(con100, dummy_backend): + vc = VectorCube.load_url(connection=con100, url="https://example.com/geometry.json", format="GeoJSON") + assert isinstance(vc, VectorCube) + vc.execute() + assert dummy_backend.get_pg() == { + "loadurl1": { + "process_id": "load_url", + "arguments": {"url": "https://example.com/geometry.json", "format": "GeoJSON"}, + "result": True, + } + } diff --git a/tests/rest/test_connection.py b/tests/rest/test_connection.py index d6dfed25e..8dce57086 100644 --- a/tests/rest/test_connection.py +++ b/tests/rest/test_connection.py @@ -30,6 +30,7 @@ connect, paginate, ) +from openeo.rest.vectorcube import VectorCube from openeo.util import ContextTimer from .. import load_json_resource @@ -2370,6 +2371,7 @@ def test_extents(self, con120): ) def test_load_geojson(con100, data, dummy_backend): vc = con100.load_geojson(data) + assert isinstance(vc, VectorCube) vc.execute() assert dummy_backend.get_pg() == { "loadgeojson1": { @@ -2383,6 +2385,23 @@ def test_load_geojson(con100, data, dummy_backend): } +def test_load_url(con100, dummy_backend, requests_mock): + file_formats = { + "input": {"GeoJSON": {"gis_data_type": ["vector"]}}, + } + requests_mock.get(API_URL + "file_formats", json=file_formats) + vc = con100.load_url("https://example.com/geometry.json", format="GeoJSON") + assert isinstance(vc, VectorCube) + vc.execute() + assert dummy_backend.get_pg() == { + "loadurl1": { + "process_id": "load_url", + "arguments": {"url": "https://example.com/geometry.json", "format": "GeoJSON"}, + "result": True, + } + } + + def test_list_file_formats(requests_mock): requests_mock.get(API_URL, json={"api_version": "1.0.0"}) conn = Connection(API_URL)