Merge pull request #259 from GeoNode/ISSUE_257_3dtiles

[Fixes #257] Add 3dtiles remote service

docker-compose-test.yaml

@@ -32,7 +32,7 @@ services:
 
   # Geoserver backend
   geoserver:
-    image: geonode/geoserver:2.23.0
+    image: geonode/geoserver:2.24.3-latest
     container_name: geoserver4importer
     healthcheck:
       test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows"
@@ -42,8 +42,6 @@ services:
       retries: 5
     env_file:
       - .env_test
-    ports:
-      - "8080:8080"
     volumes:
       - statics:/mnt/volumes/statics
       - geoserver-data-dir:/geoserver_data/data
@@ -55,7 +53,7 @@ services:
         condition: service_healthy
 
   data-dir-conf:
-    image: geonode/geoserver_data:2.23.0
+    image: geonode/geoserver_data:2.24.3-latest
     container_name: gsconf4importer
     entrypoint: sleep infinity
     volumes:
@@ -79,8 +77,6 @@ services:
     healthcheck:
       test: "pg_isready -d postgres -U postgres"
     # uncomment to enable remote connections to postgres
-    ports:
-      - "5432:5432"
 
 
 volumes:

importer/api/views.py

@@ -141,7 +141,7 @@ def create(self, request, *args, **kwargs):
                         request, asset_dir, storage_manager, _data, handler
                     )
 
-                self.validate_upload(request, storage_manager)
+                    self.validate_upload(request, storage_manager)
 
                 action = ExecutionRequestAction.IMPORT.value
 

importer/handlers/common/remote.py

@@ -177,7 +177,7 @@ def create_geonode_resource(
                 sourcetype=SOURCE_TYPE_REMOTE,
                 alternate=alternate,
                 dirty_state=True,
-                title=layer_name,
+                title=params.get("title", layer_name),
                 owner=_exec.user,
             ),
         )

importer/handlers/common/vector.py

@@ -222,7 +222,7 @@ def perform_last_step(execution_id):
         if _exec and not _exec.input_params.get("store_spatial_file", False):
             resources = ResourceHandlerInfo.objects.filter(execution_request=_exec)
             # getting all assets list
-            assets = [get_default_asset(x.resource) for x in resources]
+            assets = filter(None, [get_default_asset(x.resource) for x in resources])
             # we need to loop and cancel one by one to activate the signal
             # that delete the file from the filesystem
             for asset in assets:

importer/handlers/remote/tests/test_3dtiles.py

@@ -0,0 +1,160 @@
+from django.test import TestCase
+from mock import MagicMock, patch
+from importer.api.exception import ImportException
+from django.contrib.auth import get_user_model
+from importer.handlers.common.serializer import RemoteResourceSerializer
+from importer.handlers.remote.tiles3d import RemoteTiles3DResourceHandler
+from importer.handlers.tiles3d.exceptions import Invalid3DTilesException
+from importer.orchestrator import orchestrator
+from geonode.base.populate_test_data import create_single_dataset
+from geonode.resource.models import ExecutionRequest
+
+
+class TestRemoteTiles3DFileHandler(TestCase):
+    databases = ("default", "datastore")
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        cls.handler = RemoteTiles3DResourceHandler()
+        cls.valid_url = "https://raw.githubusercontent.com/CesiumGS/3d-tiles-samples/main/1.1/TilesetWithFullMetadata/tileset.json"
+        cls.user, _ = get_user_model().objects.get_or_create(username="admin")
+        cls.invalid_files = {
+            "url": "http://abc123defsadsa.org",
+            "title": "Remote Title",
+            "type": "3dtiles",
+        }
+        cls.valid_files = {
+            "url": cls.valid_url,
+            "title": "Remote Title",
+            "type": "3dtiles",
+        }
+        cls.owner = get_user_model().objects.first()
+        cls.layer = create_single_dataset(
+            name="stazioni_metropolitana", owner=cls.owner
+        )
+
+    def test_can_handle_should_return_true_for_remote(self):
+        actual = self.handler.can_handle(self.valid_files)
+        self.assertTrue(actual)
+
+    def test_can_handle_should_return_false_for_other_files(self):
+        actual = self.handler.can_handle({"base_file": "random.file"})
+        self.assertFalse(actual)
+
+    def test_should_get_the_specific_serializer(self):
+        actual = self.handler.has_serializer(self.valid_files)
+        self.assertEqual(type(actual), type(RemoteResourceSerializer))
+
+    def test_create_error_log(self):
+        """
+        Should return the formatted way for the log of the handler
+        """
+        actual = self.handler.create_error_log(
+            Exception("my exception"),
+            "foo_task_name",
+            *["exec_id", "layer_name", "alternate"],
+        )
+        expected = "Task: foo_task_name raised an error during actions for layer: alternate: my exception"
+        self.assertEqual(expected, actual)
+
+    def test_task_list_is_the_expected_one(self):
+        expected = (
+            "start_import",
+            "importer.import_resource",
+            "importer.create_geonode_resource",
+        )
+        self.assertEqual(len(self.handler.ACTIONS["import"]), 3)
+        self.assertTupleEqual(expected, self.handler.ACTIONS["import"])
+
+    def test_task_list_is_the_expected_one_geojson(self):
+        expected = (
+            "start_copy",
+            "importer.copy_geonode_resource",
+        )
+        self.assertEqual(len(self.handler.ACTIONS["copy"]), 2)
+        self.assertTupleEqual(expected, self.handler.ACTIONS["copy"])
+
+    def test_is_valid_should_raise_exception_if_the_url_is_invalid(self):
+        with self.assertRaises(ImportException) as _exc:
+            self.handler.is_valid_url(url=self.invalid_files["url"])
+
+        self.assertIsNotNone(_exc)
+        self.assertTrue("The provided url is not reachable")
+
+    def test_is_valid_should_pass_with_valid_url(self):
+        self.handler.is_valid_url(url=self.valid_files["url"])
+
+    def test_extract_params_from_data(self):
+        actual, _data = self.handler.extract_params_from_data(
+            _data={
+                "defaults": '{"url": "http://abc123defsadsa.org", "title": "Remote Title", "type": "3dtiles"}'
+            },
+            action="import",
+        )
+        self.assertTrue("title" in actual)
+        self.assertTrue("url" in actual)
+        self.assertTrue("type" in actual)
+
+    @patch("importer.handlers.common.remote.import_orchestrator")
+    def test_import_resource_should_work(self, patch_upload):
+        patch_upload.apply_async.side_effect = MagicMock()
+        try:
+            exec_id = orchestrator.create_execution_request(
+                user=get_user_model().objects.first(),
+                func_name="funct1",
+                step="step",
+                input_params=self.valid_files,
+            )
+
+            # start the resource import
+            self.handler.import_resource(
+                files=self.valid_files, execution_id=str(exec_id)
+            )
+            patch_upload.apply_async.assert_called_once()
+        finally:
+            if exec_id:
+                ExecutionRequest.objects.filter(exec_id=exec_id).delete()
+
+    def test_create_geonode_resource_raise_error_if_url_is_not_reachabel(self):
+        with self.assertRaises(Invalid3DTilesException) as error:
+            exec_id = orchestrator.create_execution_request(
+                user=self.owner,
+                func_name="funct1",
+                step="step",
+                input_params={
+                    "url": "http://abc123defsadsa.org",
+                    "title": "Remote Title",
+                    "type": "3dtiles",
+                },
+            )
+
+            resource = self.handler.create_geonode_resource(
+                "layername",
+                "layeralternate",
+                execution_id=exec_id,
+                resource_type="ResourceBase",
+                asset=None,
+            )
+
+    def test_create_geonode_resource(self):
+        exec_id = orchestrator.create_execution_request(
+            user=self.owner,
+            func_name="funct1",
+            step="step",
+            input_params={
+                "url": "https://dummyjson.com/users",
+                "title": "Remote Title",
+                "type": "3dtiles",
+            },
+        )
+
+        resource = self.handler.create_geonode_resource(
+            "layername",
+            "layeralternate",
+            execution_id=exec_id,
+            resource_type="ResourceBase",
+            asset=None,
+        )
+        self.assertIsNotNone(resource)
+        self.assertEqual(resource.subtype, "3dtiles")

importer/handlers/remote/tiles3d.py

@@ -0,0 +1,77 @@
+import logging
+
+import requests
+from geonode.layers.models import Dataset
+from importer.handlers.common.remote import BaseRemoteResourceHandler
+from importer.handlers.tiles3d.handler import Tiles3DFileHandler
+from importer.orchestrator import orchestrator
+from importer.handlers.tiles3d.exceptions import Invalid3DTilesException
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteTiles3DResourceHandler(BaseRemoteResourceHandler, Tiles3DFileHandler):
+
+    @staticmethod
+    def can_handle(_data) -> bool:
+        """
+        This endpoint will return True or False if with the info provided
+        the handler is able to handle the file or not
+        """
+        if "url" in _data and "3dtiles" in _data.get("type"):
+            return True
+        return False
+
+    @staticmethod
+    def is_valid_url(url):
+        BaseRemoteResourceHandler.is_valid_url(url)
+        try:
+            payload = requests.get(url, timeout=10).json()
+            # required key described in the specification of 3dtiles
+            # https://docs.ogc.org/cs/22-025r4/22-025r4.html#toc92
+            is_valid = all(
+                key in payload.keys() for key in ("asset", "geometricError", "root")
+            )
+
+            if not is_valid:
+                raise Invalid3DTilesException(
+                    "The provided 3DTiles is not valid, some of the mandatory keys are missing. Mandatory keys are: 'asset', 'geometricError', 'root'"
+                )
+
+            Tiles3DFileHandler.validate_3dtile_payload(payload=payload)
+
+        except Exception as e:
+            raise Invalid3DTilesException(e)
+
+        return True
+
+    def create_geonode_resource(
+        self,
+        layer_name: str,
+        alternate: str,
+        execution_id: str,
+        resource_type: Dataset = ...,
+        asset=None,
+    ):
+        resource = super().create_geonode_resource(
+            layer_name, alternate, execution_id, resource_type, asset
+        )
+        _exec = orchestrator.get_execution_object(exec_id=execution_id)
+        try:
+            js_file = requests.get(_exec.input_params.get("url"), timeout=10).json()
+        except Exception as e:
+            raise Invalid3DTilesException(e)
+
+        if not js_file:
+            raise Invalid3DTilesException("The JSON file returned by the URL is empty")
+
+        if self._has_region(js_file):
+            resource = self.set_bbox_from_region(js_file, resource=resource)
+        elif self._has_sphere(js_file):
+            resource = self.set_bbox_from_boundingVolume_sphere(
+                js_file, resource=resource
+            )
+        else:
+            resource = self.set_bbox_from_boundingVolume(js_file, resource=resource)
+
+        return resource

importer/handlers/tiles3d/handler.py

@@ -103,29 +103,33 @@ def is_valid(files, user):
                     "The provided 3DTiles is not valid, some of the mandatory keys are missing. Mandatory keys are: 'asset', 'geometricError', 'root'"
                 )
 
-            # if the keys are there, let's check if the mandatory child are there too
-            asset = _file.get("asset", {}).get("version", None)
-            if not asset:
-                raise Invalid3DTilesException(
-                    "The mandatory 'version' for the key 'asset' is missing"
-                )
-            volume = _file.get("root", {}).get("boundingVolume", None)
-            if not volume:
-                raise Invalid3DTilesException(
-                    "The mandatory 'boundingVolume' for the key 'root' is missing"
-                )
-
-            error = _file.get("root", {}).get("geometricError", None)
-            if error is None:
-                raise Invalid3DTilesException(
-                    "The mandatory 'geometricError' for the key 'root' is missing"
-                )
+            Tiles3DFileHandler.validate_3dtile_payload(payload=_file)
 
         except Exception as e:
             raise Invalid3DTilesException(e)
 
         return True
 
+    @staticmethod
+    def validate_3dtile_payload(payload):
+        # if the keys are there, let's check if the mandatory child are there too
+        asset = payload.get("asset", {}).get("version", None)
+        if not asset:
+            raise Invalid3DTilesException(
+                "The mandatory 'version' for the key 'asset' is missing"
+            )
+        volume = payload.get("root", {}).get("boundingVolume", None)
+        if not volume:
+            raise Invalid3DTilesException(
+                "The mandatory 'boundingVolume' for the key 'root' is missing"
+            )
+
+        error = payload.get("root", {}).get("geometricError", None)
+        if error is None:
+            raise Invalid3DTilesException(
+                "The mandatory 'geometricError' for the key 'root' is missing"
+            )
+
     @staticmethod
     def extract_params_from_data(_data, action=None):
         """