task/fix handling of missing image geolocation (#121)

* Fix geolocation checks for image importing

This was due to changes in DES-2213 (#97) where python packages were
updated

* Increase rate of file importing

* Improve docstr

* fix linting

geoapi/services/images.py

@@ -44,17 +44,17 @@ def processBase64(encoded: AnyStr) -> ImageData:
     def processImage(fileObj: IO) -> ImageData:
         """
         Resize and get the EXIF GeoLocation from an image
+
+        Image need geolocation information. If missing, then
+        get_exif_location raises InvalidEXIFData.
+
         :param fileObj:
         :return:
         """
-
-        try:
-            imdata = ImageService.resizeImage(fileObj)
-            exif_loc = get_exif_location(fileObj)
-            imdata.coordinates = exif_loc
-            return imdata
-        except:  # noqa: E722
-            raise InvalidEXIFData()
+        imdata = ImageService.resizeImage(fileObj)
+        exif_loc = get_exif_location(fileObj)
+        imdata.coordinates = exif_loc
+        return imdata
 
     @staticmethod
     def resizeImage(fileObj: IO) -> ImageData:
@@ -155,6 +155,9 @@ def _convert_to_degress(value):
 def get_exif_location(image):
     """
     Returns the latitude and longitude, if available, from the provided exif_data (obtained through get_exif_data above)
+
+    raises: InvalidEXIFData: if geospatial data missing
+
     """
     exif_data = exifread.process_file(image)
     lat = None
@@ -174,4 +177,6 @@ def get_exif_location(image):
         if gps_longitude_ref.values[0] != 'E':
             lon = 0 - lon
 
+    if not lat or not lon:
+        raise InvalidEXIFData
     return lon, lat

geoapi/tasks/external_data.py

@@ -101,8 +101,13 @@ def get_additional_files(systemId: str, path: str, client, available_files=None)
     return additional_files
 
 
-@app.task(rate_limit="1/s")
+@app.task(rate_limit="10/s")
 def import_file_from_agave(userId: int, systemId: str, path: str, projectId: int):
+    """
+    Import file from TAPIS system
+
+    Note: all geolocation information is expected to be embedded in the imported file.
+    """
     user = db_session.query(User).get(userId)
     client = AgaveUtils(user.jwt)
     try:
@@ -219,6 +224,20 @@ def import_point_clouds_from_agave(userId: int, files, pointCloudId: int):
 
 @app.task(rate_limit="5/s")
 def import_from_agave(tenant_id: str, userId: int, systemId: str, path: str, projectId: int):
+    """
+    Recursively import files from a system/path.
+
+    If file has already been imported (i.e. during a previously call), we don't re-import it. Likewise,
+    if we have previously failed at importing a file, we do not retry to import the file (unless it was an error like
+    file-access where it makes sense to retry at a later time).
+
+    Files located in /Rapp folder (i.e. created by the RAPP app) are handled differently as their location data is not
+    contained in specific-file-format meta data (e.g. exif for images) but instead the location is stored in Tapis
+    metadata.
+
+    This method is called by refresh_observable_projects()
+    """
+
     user = db_session.query(User).get(userId)
     client = AgaveUtils(user.jwt)
     logger.info("Importing for project:{} directory:{}/{} for user:{}".format(projectId,

geoapi/tests/services/test_image_service.py

@@ -1,5 +1,7 @@
 from geoapi.services.images import ImageService, get_exif_location
+from geoapi.exceptions import InvalidEXIFData
 from PIL import Image, ImageChops
+import pytest
 
 
 def test_image_service_rotations(flipped_image_fixture, corrected_image_fixture):
@@ -28,6 +30,16 @@ def test_get_exif_location(image_file_fixture):
     assert coordinates == (-80.78037499999999, 32.61850555555556)
 
 
+def test_get_exif_location_missing(image_file_no_location_fixture):
+    with pytest.raises(InvalidEXIFData):
+        get_exif_location(image_file_no_location_fixture)
+
+
 def test_process_image(image_file_fixture):
     imdata = ImageService.processImage(image_file_fixture)
     assert imdata.coordinates == (-80.78037499999999, 32.61850555555556)
+
+
+def test_process_image_location_missing(image_file_no_location_fixture):
+    with pytest.raises(InvalidEXIFData):
+        ImageService.processImage(image_file_no_location_fixture)