Merge branch 'databrickslabs-main'

a0x8o · Dec 22, 2023 · 8b4b566 · 8b4b566
2 parents d25c9dc + d48c233
commit 8b4b566
Show file tree

Hide file tree

Showing 191 changed files with 1,515 additions and 4,295 deletions.
diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml
@@ -10,21 +10,21 @@ runs:
     - name: Create download location for Spark
       shell: bash
       run: |
-        sudo mkdir -p /usr/spark-download/unzipped
-        sudo mkdir -p /usr/spark-download/raw
-        sudo chown -R $USER: /usr/spark-download/
+        sudo mkdir -p /usr/spark-download-${{ matrix.spark }}/unzipped
+        sudo mkdir -p /usr/spark-download-${{ matrix.spark }}/raw
+        sudo chown -R $USER: /usr/spark-download-${{ matrix.spark }}/
     - name: Cache Spark download
       id: cache-spark
       uses: actions/cache@v3
       with:
-        path: /usr/spark-download/unzipped
+        path: /usr/spark-download-${{ matrix.spark }}/unzipped
         key: r_build-spark
     - if: ${{ steps.cache-spark.outputs.cache-hit != 'true' }}
       name: Download and unpack Spark
       shell: bash
       run: |
-        wget -P /usr/spark-download/raw https://archive.apache.org/dist/spark/spark-${{ matrix.spark }}/spark-${{ matrix.spark }}-bin-hadoop3.tgz
-        tar zxvf /usr/spark-download/raw/spark-${{ matrix.spark }}-bin-hadoop3.tgz -C /usr/spark-download/unzipped
+        wget -P /usr/spark-download-${{ matrix.spark }}/raw https://archive.apache.org/dist/spark/spark-${{ matrix.spark }}/spark-${{ matrix.spark }}-bin-hadoop3.tgz
+        tar zxvf /usr/spark-download-${{ matrix.spark }}/raw/spark-${{ matrix.spark }}-bin-hadoop3.tgz -C /usr/spark-download-${{ matrix.spark }}/unzipped
     - name: Create R environment
       shell: bash
       run: |
@@ -51,24 +51,28 @@ runs:
         cd R
         Rscript --vanilla generate_docs.R
       env:
-        SPARK_HOME: /usr/spark-download/unzipped/spark-${{ matrix.spark }}-bin-hadoop3
+        SPARK_HOME: /usr/spark-download-${{ matrix.spark }}/unzipped/spark-${{ matrix.spark }}-bin-hadoop3
     - name: Build R package
       shell: bash
       run: |
         cd R
         Rscript --vanilla build_r_package.R
       env:
-        SPARK_HOME: /usr/spark-download/unzipped/spark-${{ matrix.spark }}-bin-hadoop3
+        SPARK_HOME: /usr/spark-download-${{ matrix.spark }}/unzipped/spark-${{ matrix.spark }}-bin-hadoop3
     - name: Test SparkR package
       shell: bash
       run: |
         cd R/sparkR-mosaic
         Rscript --vanilla tests.R
+      env:
+        SPARK_HOME: /usr/spark-download-${{ matrix.spark }}/unzipped/spark-${{ matrix.spark }}-bin-hadoop3
     - name: Test sparklyr package
       shell: bash
       run: |
         cd R/sparklyr-mosaic
         Rscript --vanilla tests.R
+      env:
+        SPARK_HOME: /usr/spark-download-${{ matrix.spark }}/unzipped/spark-${{ matrix.spark }}-bin-hadoop3
     - name: Copy R artifacts to GH Actions run
       shell: bash
       run: |

diff --git a/.github/actions/scala_build/action.yml b/.github/actions/scala_build/action.yml
@@ -1,6 +1,6 @@
 name: build mosaic scala
 description: build mosaic scala
-inputs: 
+inputs:
   skip_tests:
     description: 'Skip Scala Tests? true or false'
     default: 'false'
@@ -19,12 +19,23 @@ runs:
         - name: Add packaged GDAL dependencies
           shell: bash
           run : |
-            sudo apt-get update && sudo apt-get install -y unixodbc libcurl3-gnutls libsnappy-dev libopenjp2-7 
-            pip install databricks-mosaic-gdal==${{ matrix.gdal }}
-            sudo tar -xf /opt/hostedtoolcache/Python/${{ matrix.python }}/x64/lib/python3.9/site-packages/databricks-mosaic-gdal/resources/gdal-${{ matrix.gdal }}-filetree.tar.xz -C /
-            sudo tar -xhf /opt/hostedtoolcache/Python/${{ matrix.python }}/x64/lib/python3.9/site-packages/databricks-mosaic-gdal/resources/gdal-${{ matrix.gdal }}-symlinks.tar.xz -C /
-            pip install numpy==${{ matrix.numpy }}
-            pip install gdal==${{ matrix.gdal }}
+            # - update apt
+            sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc)-backports main universe multiverse restricted"
+            sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc)-updates main universe multiverse restricted"
+            sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc)-security main multiverse restricted universe"
+            sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc) main multiverse restricted universe"
+            sudo apt-get update -y
+            # - install numpy first
+            pip install --upgrade pip
+            pip install 'numpy>=${{ matrix.numpy }}'
+            # - install natives
+            sudo apt-get install -y unixodbc libcurl3-gnutls libsnappy-dev libopenjp2-7
+            sudo apt-get install -y gdal-bin libgdal-dev python3-gdal
+            # - install gdal with numpy
+            pip install --no-cache-dir --force-reinstall 'GDAL[numpy]==${{ matrix.gdal }}'
+            sudo wget -P /usr/lib -nc https://github.com/databrickslabs/mosaic/raw/main/resources/gdal/jammy/libgdalalljni.so
+            sudo wget -P /usr/lib -nc https://github.com/databrickslabs/mosaic/raw/main/resources/gdal/jammy/libgdalalljni.so.30
+            #sudo wget -P /usr/lib -nc https://github.com/databrickslabs/mosaic/raw/main/resources/gdal/jammy/libgdalalljni.so.30.0.3
         - name: Test and build the scala JAR - skip tests is false
           if: inputs.skip_tests == 'false'
           shell: bash

diff --git a/.github/workflows/build_main.yml b/.github/workflows/build_main.yml
@@ -11,16 +11,16 @@ on:
       - "**"
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       matrix:
-        python: [ 3.9.5 ]
+        python: [ 3.10.12 ]
         numpy: [ 1.21.5 ]
-        gdal: [ 3.4.3 ]
-        spark: [ 3.3.2 ]
-        R: [ 4.1.2 ]
+        gdal: [ 3.4.1 ]
+        spark: [ 3.4.0 ]
+        R: [ 4.2.2 ]
     steps:
       - name: checkout code
         uses: actions/checkout@v2
@@ -31,4 +31,4 @@ jobs:
       - name: build R
         uses: ./.github/actions/r_build
       - name: upload artefacts
-        uses: ./.github/actions/upload_artefacts
+        uses: ./.github/actions/upload_artefacts
diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml
@@ -7,24 +7,24 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       matrix:
-        python: [ 3.9.5 ]
+        python: [ 3.10.12 ]
         numpy: [ 1.21.5 ]
-        gdal: [ 3.4.3 ]
-        spark: [ 3.3.2 ]
-        R: [ 4.1.2 ]
+        gdal: [ 3.4.1 ]
+        spark: [ 3.4.0 ]
+        R: [ 4.2.2 ]
     steps:
       - name: checkout code
         uses: actions/checkout@v2
       - name: build scala with skipping tests
         uses: ./.github/actions/scala_build
-        with: 
+        with:
           skip_tests: "true"
       - name: build python
         uses: ./.github/actions/python_build
       - name: upload artefacts
-        uses: ./.github/actions/upload_artefacts
+        uses: ./.github/actions/upload_artefacts
diff --git a/.github/workflows/build_r.yml b/.github/workflows/build_r.yml
@@ -8,24 +8,24 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       matrix:
-        python: [ 3.9.5 ]
+        python: [ 3.10.12 ]
         numpy: [ 1.21.5 ]
-        gdal: [ 3.4.3 ]
-        spark: [ 3.3.2 ]
-        R: [ 4.1.2 ]
+        gdal: [ 3.4.1 ]
+        spark: [ 3.4.0 ]
+        R: [ 4.2.2 ]
     steps:
       - name: checkout code
         uses: actions/checkout@v2
       - name: build scala with skipping tests
         uses: ./.github/actions/scala_build
-        with: 
+        with:
           skip_tests: 'true'
       - name: build r artefacts
         uses: ./.github/actions/r_build
       - name: upload artefacts
-        uses: ./.github/actions/upload_artefacts
+        uses: ./.github/actions/upload_artefacts
diff --git a/.github/workflows/build_scala.yml b/.github/workflows/build_scala.yml
@@ -6,16 +6,16 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       matrix:
-        python: [ 3.9.5 ]
+        python: [ 3.10.12 ]
         numpy: [ 1.21.5 ]
-        gdal: [ 3.4.3 ]
-        spark: [ 3.3.2 ]
-        R: [ 4.1.2 ]
+        gdal: [ 3.4.1 ]
+        spark: [ 3.4.0 ]
+        R: [ 4.2.2 ]
     steps:
       - name: checkout code
         uses: actions/checkout@v2

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -7,10 +7,15 @@ on:
     - feature/new_docs
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python: [ 3.9.5 ]
     steps:
     - name: install pandoc
-      run: sudo apt-get install pandoc
+      run: |
+        sudo apt-get install pandoc
+        sudo apt-get install --reinstall python3-pkg-resources
     - uses: actions/setup-python@v2
     - uses: actions/checkout@v2
       with:

diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml
@@ -6,11 +6,14 @@ on:
 jobs:
   build-n-publish:
     name: Build project and publish to TestPyPI
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python: [3.9]
-        spark: [3.2.1]
+        python: [ 3.10.12 ]
+        numpy: [ 1.21.5 ]
+        gdal: [ 3.4.1 ]
+        spark: [ 3.4.0 ]
+        R: [ 4.2.2 ]
     steps:
       - name: checkout code
         uses: actions/checkout@v2
@@ -24,4 +27,3 @@ jobs:
           user: __token__
           password: ${{ secrets.LABS_PYPI_TOKEN }}
           packages_dir: python/dist/
-
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,16 @@
+## v0.3.14
+- Fixes for Warning and Error messages on mosaic_enable call.
+- Performance improvements for raster functions.
+- Fix support for GDAL configuration via spark config (use 'spark.databricks.labs.mosaic.gdal.' prefix).
+
+## v0.3.13
+- R bindings generation fixed and improved.
+- Remove usage of /vsimem/ drivers for GDAL due to memory leaks.
+- Add support for MapAlgebra expressions via RST_MapAlgebra.
+- Add support for custom combine python functions via RST_DerivedBand.
+- Improve test coverage.
+- Allow for GDAL configuration via spark config (use 'spark.databricks.labs.mosaic.gdal.' prefix).
+
 ## v0.3.12
 - Make JTS default Geometry Provider
 - Add raster tile functions.

diff --git a/R/install_deps.R b/R/install_deps.R
@@ -1,3 +1,3 @@
-options(repos = c(CRAN = "https://packagemanager.posit.co/cran/__linux__/focal/latest"))
+options(repos = c(CRAN = "https://packagemanager.posit.co/cran/__linux__/jammy/latest"))
 
-install.packages(c("pkgbuild", "testthat", "roxygen2", "sparklyr"))
+install.packages(c("pkgbuild", "testthat", "roxygen2", "sparklyr"))
diff --git a/R/sparkR-mosaic/sparkrMosaic/DESCRIPTION b/R/sparkR-mosaic/sparkrMosaic/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: sparkrMosaic
 Title: SparkR bindings for Databricks Mosaic
-Version: 0.3.12
+Version: 0.3.14
 Authors@R: 
     person("Robert", "Whiffin", , "[email protected]", role = c("aut", "cre")
            )

diff --git a/R/sparkR-mosaic/tests.R b/R/sparkR-mosaic/tests.R
@@ -1,7 +1,6 @@
 library(testthat)
 
-spark_location <- "/usr/spark-download/unzipped/spark-3.3.2-bin-hadoop3"
-Sys.setenv(SPARK_HOME = spark_location)
+spark_location <- Sys.getenv("SPARK_HOME")
 library(SparkR, lib.loc = c(file.path(spark_location, "R", "lib")))
 .libPaths(c(file.path(spark_location, "R", "lib"), .libPaths()))
 

diff --git a/R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION b/R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: sparklyrMosaic
 Title: sparklyr bindings for Databricks Mosaic
-Version: 0.3.12
+Version: 0.3.14
 Authors@R: 
     person("Robert", "Whiffin", , "[email protected]", role = c("aut", "cre")
            )

diff --git a/R/sparklyr-mosaic/tests.R b/R/sparklyr-mosaic/tests.R
@@ -9,8 +9,9 @@ if(length(getOption("repos")) < 1) {
 install.packages("sparklyr", repos="")
 library(sparklyr)
 
-spark_home_set("/usr/spark-download/unzipped/spark-3.3.2-bin-hadoop3")
-install.packages("sparklyrMosaic_0.3.12.tar.gz", repos = NULL)
+spark_home <- Sys.getenv("SPARK_HOME")
+spark_home_set(spark_home)
+install.packages("sparklyrMosaic_0.3.14.tar.gz", repos = NULL)
 library(sparklyrMosaic)
 
 # find the mosaic jar in staging

diff --git a/README.md b/README.md
@@ -21,7 +21,7 @@ Mosaic was created to simplify the implementation of scalable geospatial data pi
 Mosaic provides geospatial tools for
 * Data ingestion (WKT, WKB, GeoJSON)
 * Data processing
-    * Geometry and geography `ST_` operations (with default [JTS](https://github.com/locationtech/jts) or [ESRI](https://github.com/Esri/geometry-api-java)) 
+    * Geometry and geography `ST_` operations via [JTS](https://github.com/locationtech/jts)
     * Indexing (with default [H3](https://github.com/uber/h3) or BNG)
     * Chipping of polygons and lines over an indexing grid [co-developed with Ordnance Survey and Microsoft](https://databricks.com/blog/2021/10/11/efficient-point-in-polygon-joins-via-pyspark-and-bng-geospatial-indexing.html)
 * Data visualization ([Kepler](https://github.com/keplergl/kepler.gl))

diff --git a/docs/docs-requirements.txt b/docs/docs-requirements.txt
@@ -1,3 +1,4 @@
+setuptools==68.1.2
 Sphinx==4.4.0
 sphinx-material==0.0.35
 nbsphinx==0.8.8

diff --git a/docs/source/api/api.rst b/docs/source/api/api.rst
@@ -11,4 +11,10 @@ API Documentation
    spatial-functions
    spatial-indexing
    spatial-predicates
-   spatial-aggregations
+<<<<<<< HEAD
+   spatial-aggregations
+=======
+   spatial-aggregations
+   raster-functions
+   rasterio-udfs
+>>>>>>> databrickslabs-main