diff --git a/docker-compose-test.yaml b/docker-compose-test.yaml index d86c2c4..1713cc8 100644 --- a/docker-compose-test.yaml +++ b/docker-compose-test.yaml @@ -32,7 +32,7 @@ services: # Geoserver backend geoserver: - image: geonode/geoserver:2.23.0 + image: geonode/geoserver:2.24.3-latest container_name: geoserver4importer healthcheck: test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows" @@ -42,8 +42,6 @@ services: retries: 5 env_file: - .env_test - ports: - - "8080:8080" volumes: - statics:/mnt/volumes/statics - geoserver-data-dir:/geoserver_data/data @@ -55,7 +53,7 @@ services: condition: service_healthy data-dir-conf: - image: geonode/geoserver_data:2.23.0 + image: geonode/geoserver_data:2.24.3-latest container_name: gsconf4importer entrypoint: sleep infinity volumes: @@ -79,8 +77,6 @@ services: healthcheck: test: "pg_isready -d postgres -U postgres" # uncomment to enable remote connections to postgres - ports: - - "5432:5432" volumes: diff --git a/importer/api/views.py b/importer/api/views.py index 10a372f..09720e4 100644 --- a/importer/api/views.py +++ b/importer/api/views.py @@ -141,7 +141,7 @@ def create(self, request, *args, **kwargs): request, asset_dir, storage_manager, _data, handler ) - self.validate_upload(request, storage_manager) + self.validate_upload(request, storage_manager) action = ExecutionRequestAction.IMPORT.value diff --git a/importer/handlers/common/remote.py b/importer/handlers/common/remote.py index 6a84c9e..c546bbf 100755 --- a/importer/handlers/common/remote.py +++ b/importer/handlers/common/remote.py @@ -177,7 +177,7 @@ def create_geonode_resource( sourcetype=SOURCE_TYPE_REMOTE, alternate=alternate, dirty_state=True, - title=layer_name, + title=params.get("title", layer_name), owner=_exec.user, ), ) diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index 144e017..865dc48 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -222,7 +222,7 @@ def perform_last_step(execution_id): if _exec and not _exec.input_params.get("store_spatial_file", False): resources = ResourceHandlerInfo.objects.filter(execution_request=_exec) # getting all assets list - assets = [get_default_asset(x.resource) for x in resources] + assets = filter(None, [get_default_asset(x.resource) for x in resources]) # we need to loop and cancel one by one to activate the signal # that delete the file from the filesystem for asset in assets: diff --git a/importer/handlers/remote/__init__.py b/importer/handlers/remote/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/importer/handlers/remote/tests/__init__.py b/importer/handlers/remote/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/importer/handlers/remote/tests/test_3dtiles.py b/importer/handlers/remote/tests/test_3dtiles.py new file mode 100644 index 0000000..bb5c569 --- /dev/null +++ b/importer/handlers/remote/tests/test_3dtiles.py @@ -0,0 +1,160 @@ +from django.test import TestCase +from mock import MagicMock, patch +from importer.api.exception import ImportException +from django.contrib.auth import get_user_model +from importer.handlers.common.serializer import RemoteResourceSerializer +from importer.handlers.remote.tiles3d import RemoteTiles3DResourceHandler +from importer.handlers.tiles3d.exceptions import Invalid3DTilesException +from importer.orchestrator import orchestrator +from geonode.base.populate_test_data import create_single_dataset +from geonode.resource.models import ExecutionRequest + + +class TestRemoteTiles3DFileHandler(TestCase): + databases = ("default", "datastore") + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.handler = RemoteTiles3DResourceHandler() + cls.valid_url = "https://raw.githubusercontent.com/CesiumGS/3d-tiles-samples/main/1.1/TilesetWithFullMetadata/tileset.json" + cls.user, _ = get_user_model().objects.get_or_create(username="admin") + cls.invalid_files = { + "url": "http://abc123defsadsa.org", + "title": "Remote Title", + "type": "3dtiles", + } + cls.valid_files = { + "url": cls.valid_url, + "title": "Remote Title", + "type": "3dtiles", + } + cls.owner = get_user_model().objects.first() + cls.layer = create_single_dataset( + name="stazioni_metropolitana", owner=cls.owner + ) + + def test_can_handle_should_return_true_for_remote(self): + actual = self.handler.can_handle(self.valid_files) + self.assertTrue(actual) + + def test_can_handle_should_return_false_for_other_files(self): + actual = self.handler.can_handle({"base_file": "random.file"}) + self.assertFalse(actual) + + def test_should_get_the_specific_serializer(self): + actual = self.handler.has_serializer(self.valid_files) + self.assertEqual(type(actual), type(RemoteResourceSerializer)) + + def test_create_error_log(self): + """ + Should return the formatted way for the log of the handler + """ + actual = self.handler.create_error_log( + Exception("my exception"), + "foo_task_name", + *["exec_id", "layer_name", "alternate"], + ) + expected = "Task: foo_task_name raised an error during actions for layer: alternate: my exception" + self.assertEqual(expected, actual) + + def test_task_list_is_the_expected_one(self): + expected = ( + "start_import", + "importer.import_resource", + "importer.create_geonode_resource", + ) + self.assertEqual(len(self.handler.ACTIONS["import"]), 3) + self.assertTupleEqual(expected, self.handler.ACTIONS["import"]) + + def test_task_list_is_the_expected_one_geojson(self): + expected = ( + "start_copy", + "importer.copy_geonode_resource", + ) + self.assertEqual(len(self.handler.ACTIONS["copy"]), 2) + self.assertTupleEqual(expected, self.handler.ACTIONS["copy"]) + + def test_is_valid_should_raise_exception_if_the_url_is_invalid(self): + with self.assertRaises(ImportException) as _exc: + self.handler.is_valid_url(url=self.invalid_files["url"]) + + self.assertIsNotNone(_exc) + self.assertTrue("The provided url is not reachable") + + def test_is_valid_should_pass_with_valid_url(self): + self.handler.is_valid_url(url=self.valid_files["url"]) + + def test_extract_params_from_data(self): + actual, _data = self.handler.extract_params_from_data( + _data={ + "defaults": '{"url": "http://abc123defsadsa.org", "title": "Remote Title", "type": "3dtiles"}' + }, + action="import", + ) + self.assertTrue("title" in actual) + self.assertTrue("url" in actual) + self.assertTrue("type" in actual) + + @patch("importer.handlers.common.remote.import_orchestrator") + def test_import_resource_should_work(self, patch_upload): + patch_upload.apply_async.side_effect = MagicMock() + try: + exec_id = orchestrator.create_execution_request( + user=get_user_model().objects.first(), + func_name="funct1", + step="step", + input_params=self.valid_files, + ) + + # start the resource import + self.handler.import_resource( + files=self.valid_files, execution_id=str(exec_id) + ) + patch_upload.apply_async.assert_called_once() + finally: + if exec_id: + ExecutionRequest.objects.filter(exec_id=exec_id).delete() + + def test_create_geonode_resource_raise_error_if_url_is_not_reachabel(self): + with self.assertRaises(Invalid3DTilesException) as error: + exec_id = orchestrator.create_execution_request( + user=self.owner, + func_name="funct1", + step="step", + input_params={ + "url": "http://abc123defsadsa.org", + "title": "Remote Title", + "type": "3dtiles", + }, + ) + + resource = self.handler.create_geonode_resource( + "layername", + "layeralternate", + execution_id=exec_id, + resource_type="ResourceBase", + asset=None, + ) + + def test_create_geonode_resource(self): + exec_id = orchestrator.create_execution_request( + user=self.owner, + func_name="funct1", + step="step", + input_params={ + "url": "https://dummyjson.com/users", + "title": "Remote Title", + "type": "3dtiles", + }, + ) + + resource = self.handler.create_geonode_resource( + "layername", + "layeralternate", + execution_id=exec_id, + resource_type="ResourceBase", + asset=None, + ) + self.assertIsNotNone(resource) + self.assertEqual(resource.subtype, "3dtiles") diff --git a/importer/handlers/remote/tiles3d.py b/importer/handlers/remote/tiles3d.py new file mode 100644 index 0000000..517b691 --- /dev/null +++ b/importer/handlers/remote/tiles3d.py @@ -0,0 +1,77 @@ +import logging + +import requests +from geonode.layers.models import Dataset +from importer.handlers.common.remote import BaseRemoteResourceHandler +from importer.handlers.tiles3d.handler import Tiles3DFileHandler +from importer.orchestrator import orchestrator +from importer.handlers.tiles3d.exceptions import Invalid3DTilesException + +logger = logging.getLogger(__name__) + + +class RemoteTiles3DResourceHandler(BaseRemoteResourceHandler, Tiles3DFileHandler): + + @staticmethod + def can_handle(_data) -> bool: + """ + This endpoint will return True or False if with the info provided + the handler is able to handle the file or not + """ + if "url" in _data and "3dtiles" in _data.get("type"): + return True + return False + + @staticmethod + def is_valid_url(url): + BaseRemoteResourceHandler.is_valid_url(url) + try: + payload = requests.get(url, timeout=10).json() + # required key described in the specification of 3dtiles + # https://docs.ogc.org/cs/22-025r4/22-025r4.html#toc92 + is_valid = all( + key in payload.keys() for key in ("asset", "geometricError", "root") + ) + + if not is_valid: + raise Invalid3DTilesException( + "The provided 3DTiles is not valid, some of the mandatory keys are missing. Mandatory keys are: 'asset', 'geometricError', 'root'" + ) + + Tiles3DFileHandler.validate_3dtile_payload(payload=payload) + + except Exception as e: + raise Invalid3DTilesException(e) + + return True + + def create_geonode_resource( + self, + layer_name: str, + alternate: str, + execution_id: str, + resource_type: Dataset = ..., + asset=None, + ): + resource = super().create_geonode_resource( + layer_name, alternate, execution_id, resource_type, asset + ) + _exec = orchestrator.get_execution_object(exec_id=execution_id) + try: + js_file = requests.get(_exec.input_params.get("url"), timeout=10).json() + except Exception as e: + raise Invalid3DTilesException(e) + + if not js_file: + raise Invalid3DTilesException("The JSON file returned by the URL is empty") + + if self._has_region(js_file): + resource = self.set_bbox_from_region(js_file, resource=resource) + elif self._has_sphere(js_file): + resource = self.set_bbox_from_boundingVolume_sphere( + js_file, resource=resource + ) + else: + resource = self.set_bbox_from_boundingVolume(js_file, resource=resource) + + return resource diff --git a/importer/handlers/tiles3d/handler.py b/importer/handlers/tiles3d/handler.py index 1a9ff92..7889ddd 100755 --- a/importer/handlers/tiles3d/handler.py +++ b/importer/handlers/tiles3d/handler.py @@ -103,29 +103,33 @@ def is_valid(files, user): "The provided 3DTiles is not valid, some of the mandatory keys are missing. Mandatory keys are: 'asset', 'geometricError', 'root'" ) - # if the keys are there, let's check if the mandatory child are there too - asset = _file.get("asset", {}).get("version", None) - if not asset: - raise Invalid3DTilesException( - "The mandatory 'version' for the key 'asset' is missing" - ) - volume = _file.get("root", {}).get("boundingVolume", None) - if not volume: - raise Invalid3DTilesException( - "The mandatory 'boundingVolume' for the key 'root' is missing" - ) - - error = _file.get("root", {}).get("geometricError", None) - if error is None: - raise Invalid3DTilesException( - "The mandatory 'geometricError' for the key 'root' is missing" - ) + Tiles3DFileHandler.validate_3dtile_payload(payload=_file) except Exception as e: raise Invalid3DTilesException(e) return True + @staticmethod + def validate_3dtile_payload(payload): + # if the keys are there, let's check if the mandatory child are there too + asset = payload.get("asset", {}).get("version", None) + if not asset: + raise Invalid3DTilesException( + "The mandatory 'version' for the key 'asset' is missing" + ) + volume = payload.get("root", {}).get("boundingVolume", None) + if not volume: + raise Invalid3DTilesException( + "The mandatory 'boundingVolume' for the key 'root' is missing" + ) + + error = payload.get("root", {}).get("geometricError", None) + if error is None: + raise Invalid3DTilesException( + "The mandatory 'geometricError' for the key 'root' is missing" + ) + @staticmethod def extract_params_from_data(_data, action=None): """