added a full gdal + jni bindings .deb to python package; updated pom.…

…xml to latest gdal bindings; fixed an issue with argument parsing as a result of moving to gdal 3.10

pom.xml

@@ -111,7 +111,6 @@
         <dependency>
             <groupId>com.uber</groupId>
             <artifactId>h3</artifactId>
-            <!--H3 fixed at 3.7.0 until Photon updates the version-->
             <!--suppress MavenPackageUpdate-->
             <version>3.7.3</version>
         </dependency>
@@ -138,7 +137,7 @@
         <dependency>
             <groupId>org.gdal</groupId>
             <artifactId>gdal</artifactId>
-            <version>3.4.0</version>
+            <version>3.10.0</version>
         </dependency>
 
     </dependencies>

python/setup.cfg

@@ -28,6 +28,8 @@ install_requires =
 mosaic =
     lib/*.jar
     resources/*.png
+    gdal/*.deb
+
 
 [options.extras_require]
 dev =

python/setup.py

@@ -1,6 +1,46 @@
+import os
+import subprocess
+import sys
+
 try:
     from setuptools import setup
 except ImportError:
     from distutils.core import setup
+from setuptools.command.install import install
+
+class CustomInstallCommand(install):
+    """Custom install command to install .deb file."""
+
+    def run(self):
+        # Run the standard installation process
+        install.run(self)
+
+        # Install the .deb file
+        deb_file = os.path.join(os.path.dirname(__file__), 'mosaic', 'gdal', 'gdal_3.10.0-1_amd64.deb')
+
+        if os.path.exists(deb_file):
+            try:
+                # Ensure root privileges for .deb installation
+                if os.geteuid() != 0:
+                    print("You need root privileges to install the .deb package.")
+                    print("Please run this with sudo or as root.")
+                    sys.exit(1)
+
+                # Run dpkg to install the .deb file
+                try:
+                    subprocess.check_call(['dpkg', '-i', deb_file])
+                except subprocess.CalledProcessError as e:
+                    subprocess.check_call(['apt-get', 'install', '-f', '-y'])  # Fix dependencies if needed
+                    subprocess.check_call(['dpkg', '-i', deb_file])
+            except subprocess.CalledProcessError as e:
+                print(f"Error installing .deb package: {e}")
+                sys.exit(1)
+        else:
+            print(f"Error: {deb_file} not found.")
+            sys.exit(1)
 
-setup()
+setup(
+    cmdclass={
+        "install": CustomInstallCommand
+    }
+)

src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/GDALWarp.scala

@@ -6,7 +6,7 @@ import org.gdal.gdalconst.gdalconstConstants
 
 import java.nio.file.{Files, Paths}
 import scala.sys.process._
-import scala.util.Try
+import scala.util.{Failure, Success, Try}
 
 
 /** GDALWarp is a wrapper for the GDAL Warp command. */
@@ -29,7 +29,15 @@ object GDALWarp {
 
         val effectiveCommand = OperatorOptions.appendOptions(command, rasters.head.getWriteOptions)
         val warpOptionsVec = OperatorOptions.parseOptions(effectiveCommand)
-        val warpOptions = new WarpOptions(warpOptionsVec)
+        val warpOptions = Try(new WarpOptions(warpOptionsVec)) match {
+            case Success(value) => value
+            case Failure(exception) =>
+                throw new Exception(
+              "Constructing GDAL warp options object failed " +
+                  s"for command $effectiveCommand " +
+                  s"with error ${gdal.GetLastErrorMsg()}"
+            )
+        }
         val result = gdal.Warp(outputPath, rasters.map(_.getRaster).toArray, warpOptions)
         // Format will always be the same as the first raster
         val errorMsg = gdal.GetLastErrorMsg

src/main/scala/com/databricks/labs/mosaic/core/raster/operator/gdal/OperatorOptions.scala

@@ -14,12 +14,55 @@ object OperatorOptions {
       *   A vector of options.
       */
     def parseOptions(command: String): java.util.Vector[String] = {
-        val args = command.split(" ")
+        val args = parseAndDeduplicate(command)
         val optionsVec = new java.util.Vector[String]()
-        args.drop(1).foreach(optionsVec.add)
+        args.foreach(optionsVec.add)
         optionsVec
     }
 
+    def parseAndDeduplicate(args: String): List[String] = {
+        // Split the input string into an array by whitespace
+        val parts = args.split("\\s+")
+
+        // Mutable structures to track unique flags and allow duplicate prefixes
+        val seenFlags = scala.collection.mutable.Map[String, List[String]]()
+        val preservedMultipleFlags = scala.collection.mutable.ListBuffer[String]()
+
+        val flagRegex = """^-[a-zA-Z]""".r
+
+        // Process the arguments
+        var i = 0
+        while (i < parts.length) {
+            val flag = parts(i)
+            if (flag.startsWith("-")) {
+                // Slice the array for all associated values
+                val values = parts.slice(i + 1, parts.length).takeWhile(v => flagRegex.findFirstIn(v).isEmpty)
+                if (flag.startsWith("-co") || flag.startsWith("-wo")) {
+                    // Allow multiple instances of these (preserve all values)
+                    preservedMultipleFlags += flag
+                    preservedMultipleFlags ++= values
+                } else {
+                    // Deduplicate by keeping only the latest values
+                    seenFlags(flag) = values.toList
+                }
+                i += values.length // Skip over the values
+            }
+            i += 1 // Move to the next flag
+        }
+
+        // Combine the deduplicated flags and preserved multiple flags
+        val deduplicatedArgs = seenFlags.flatMap {
+            case (flag, values) =>
+                if (values.isEmpty) List(flag) // Flags without values
+                else flag +: values // Include flag and its associated values
+        }
+
+        // Return the final deduplicated and ordered list
+        (deduplicatedArgs ++ preservedMultipleFlags).toList
+    }
+
+
+
     /**
       * Add default options to the command. Extract the compression from the
       * raster and append it to the command. This operation does not change the

src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoData.scala

@@ -45,7 +45,7 @@ case class RST_InitNoData(
             .map(GDAL.getNoDataConstant)
             .mkString(" ")
         val resultPath = PathUtils.createTmpFilePath(GDAL.getExtension(tile.getDriver))
-        val cmd = s"""gdalwarp -of ${tile.getDriver} -dstnodata "$dstNoDataValues" -srcnodata "$noDataValues""""
+        val cmd = s"""gdalwarp -dstnodata "$dstNoDataValues" -srcnodata "$noDataValues""""
         tile.copy(
           raster = GDALWarp.executeWarp(
             resultPath,

src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Median.scala

@@ -31,7 +31,7 @@ case class RST_Median(rasterExpr: Expression, expressionConfig: MosaicExpression
         val medRaster = GDALWarp.executeWarp(
           resultFileName,
           Seq(raster),
-          command = s"gdalwarp -r med -tr $width $height -of $outShortName"
+          command = s"gdalwarp -r med -tr ${width} ${height}"
         )
         // Max pixel is a hack since we get a 1x1 raster back
         val maxValues = (1 to medRaster.raster.GetRasterCount()).map(medRaster.getBand(_).maxPixelValue)

src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoData.scala

@@ -52,7 +52,7 @@ case class RST_SetNoData(
             case _ => throw new IllegalArgumentException("No data values must be an array of numerical or a numerical value.")
         }).mkString(" ")
         val resultPath = PathUtils.createTmpFilePath(GDAL.getExtension(tile.getDriver))
-        val cmd = s"""gdalwarp -of ${tile.getDriver} -dstnodata "$dstNoDataValues" -srcnodata "$noDataValues""""
+        val cmd = s"""gdalwarp -dstnodata "$dstNoDataValues" -srcnodata "$noDataValues""""
         tile.copy(
           raster = GDALWarp.executeWarp(
             resultPath,

src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala

@@ -19,13 +19,13 @@ import scala.util.Try
 /** GDAL environment preparation and configuration. Some functions only for driver. */
 object MosaicGDAL extends Logging {
 
-    private val usrlibsoPath = "/usr/lib/libgdal.so"
-    private val usrlibso30Path = "/usr/lib/libgdal.so.30"
-    private val usrlibso3003Path = "/usr/lib/libgdal.so.30.0.3"
-    private val libjnisoPath = "/usr/lib/libgdalalljni.so"
-    private val libjniso30Path = "/usr/lib/libgdalalljni.so.30"
-    private val libjniso3003Path = "/usr/lib/libgdalalljni.so.30.0.3"
-    private val libogdisoPath = "/usr/lib/ogdi/4.1/libgdal.so"
+    private val usrlibsoPath = "/usr/lib/x86_64-linux-gnu/libgdal.so"
+//    private val usrlibso30Path = "/usr/lib/libgdal.so.30"
+//    private val usrlibso3003Path = "/usr/lib/libgdal.so.30.0.3"
+    private val libjnisoPath = "/usr/lib/x86_64-linux-gnu/jni/libgdalalljni.so"
+//    private val libjniso30Path = "/usr/lib/libgdalalljni.so.30"
+//    private val libjniso3003Path = "/usr/lib/libgdalalljni.so.30.0.3"
+//    private val libogdisoPath = "/usr/lib/ogdi/4.1/libgdal.so"
 
     val defaultBlockSize = 1024
     val vrtBlockSize = 128 // This is a must value for VRTs before GDAL 3.7
@@ -236,12 +236,12 @@ object MosaicGDAL extends Logging {
     /** Loads the shared objects required for GDAL. */
     private def loadSharedObjects(): Unit = {
         loadOrNOOP(usrlibsoPath)
-        loadOrNOOP(usrlibso30Path)
-        loadOrNOOP(usrlibso3003Path)
+//        loadOrNOOP(usrlibso30Path)
+//        loadOrNOOP(usrlibso3003Path)
         loadOrNOOP(libjnisoPath)
-        loadOrNOOP(libjniso30Path)
-        loadOrNOOP(libjniso3003Path)
-        loadOrNOOP(libogdisoPath)
+//        loadOrNOOP(libjniso30Path)
+//        loadOrNOOP(libjniso3003Path)
+//        loadOrNOOP(libogdisoPath)
     }
 
     /** Loads the shared object if it exists. */