Merge branch 'main' into feature/st_dimension

databrickslabs · Dec 15, 2023 · 1a088df · 1a088df
2 parents 0ed2a9b + aaf9da1
commit 1a088df
Show file tree

Hide file tree

Showing 389 changed files with 59,288 additions and 8,364 deletions.
diff --git a/.github/actions/python_build/action.yml b/.github/actions/python_build/action.yml
@@ -11,7 +11,9 @@ runs:
       shell: bash
       run: |
         cd python
-        pip install build wheel pyspark==${{ matrix.spark }}
+        pip install build wheel pyspark==${{ matrix.spark }} numpy==${{ matrix.numpy }}
+        pip install numpy==${{ matrix.numpy }}
+        pip install --no-build-isolation --no-cache-dir --force-reinstall gdal==${{ matrix.gdal }}
         pip install .
     - name: Test and build python package
       shell: bash

diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml
@@ -6,29 +6,69 @@ runs:
     - name: Setup R build environment
       shell: bash
       run: |
-        sudo apt-get update && sudo apt-get install -y curl libcurl4-openssl-dev pkg-config libharfbuzz-dev libfribidi-dev 
-    - name: Download and unpack Spark
+        sudo apt-get update && sudo apt-get install -y curl libcurl4-openssl-dev pkg-config libharfbuzz-dev libfribidi-dev
+    - name: Create download location for Spark
       shell: bash
       run: |
-        sudo mkdir -p /usr/spark-download/raw
         sudo mkdir -p /usr/spark-download/unzipped
+        sudo mkdir -p /usr/spark-download/raw
         sudo chown -R $USER: /usr/spark-download/
-        wget -P /usr/spark-download/raw https://archive.apache.org/dist/spark/spark-3.2.1/spark-3.2.1-bin-hadoop2.7.tgz 
-        tar zxvf /usr/spark-download/raw/spark-3.2.1-bin-hadoop2.7.tgz -C /usr/spark-download/unzipped
-
-
-    - name: Build R package
+    - name: Cache Spark download
+      id: cache-spark
+      uses: actions/cache@v3
+      with:
+        path: /usr/spark-download/unzipped
+        key: r_build-spark
+    - if: ${{ steps.cache-spark.outputs.cache-hit != 'true' }}
+      name: Download and unpack Spark
+      shell: bash
+      run: |
+        wget -P /usr/spark-download/raw https://archive.apache.org/dist/spark/spark-${{ matrix.spark }}/spark-${{ matrix.spark }}-bin-hadoop3.tgz
+        tar zxvf /usr/spark-download/raw/spark-${{ matrix.spark }}-bin-hadoop3.tgz -C /usr/spark-download/unzipped
+    - name: Create R environment
       shell: bash
       run: |
-        cd R
         sudo mkdir -p /usr/lib/R/site-library
         sudo chown -R $USER: /usr/lib/R/site-library
+    - name: Setup R
+      uses: r-lib/actions/setup-r@v2
+      with:
+        r-version: ${{ matrix.R }}
+        use-public-rspm: true
+    - name: Install R dependencies
+      shell: bash
+      run: |
+        cd R
+        Rscript --vanilla install_deps.R
+    - name: Generate R bindings
+      shell: bash
+      run: |
+        cd R
+        Rscript --vanilla generate_R_bindings.R ../src/main/scala/com/databricks/labs/mosaic/functions/MosaicContext.scala
+    - name: Build R docs
+      shell: bash
+      run: |
+        cd R
+        Rscript --vanilla generate_docs.R
+      env:
+        SPARK_HOME: /usr/spark-download/unzipped/spark-${{ matrix.spark }}-bin-hadoop3
+    - name: Build R package
+      shell: bash
+      run: |
+        cd R
         Rscript --vanilla build_r_package.R
-    - name: Test R package
+      env:
+        SPARK_HOME: /usr/spark-download/unzipped/spark-${{ matrix.spark }}-bin-hadoop3
+    - name: Test SparkR package
       shell: bash
       run: |
         cd R/sparkR-mosaic
         Rscript --vanilla tests.R
+    - name: Test sparklyr package
+      shell: bash
+      run: |
+        cd R/sparklyr-mosaic
+        Rscript --vanilla tests.R
     - name: Copy R artifacts to GH Actions run
       shell: bash
       run: |

diff --git a/.github/actions/scala_build/action.yml b/.github/actions/scala_build/action.yml
@@ -12,15 +12,29 @@ runs:
           with:
             java-version: '8'
             distribution: 'zulu'
+        - name: Configure python interpreter
+          uses: actions/setup-python@v2
+          with:
+            python-version: ${{ matrix.python }}
+        - name: Add packaged GDAL dependencies
+          shell: bash
+          run : |
+            sudo apt-get update && sudo apt-get install -y unixodbc libcurl3-gnutls libsnappy-dev libopenjp2-7 
+            pip install databricks-mosaic-gdal==${{ matrix.gdal }}
+            sudo tar -xf /opt/hostedtoolcache/Python/${{ matrix.python }}/x64/lib/python3.9/site-packages/databricks-mosaic-gdal/resources/gdal-${{ matrix.gdal }}-filetree.tar.xz -C /
+            sudo tar -xhf /opt/hostedtoolcache/Python/${{ matrix.python }}/x64/lib/python3.9/site-packages/databricks-mosaic-gdal/resources/gdal-${{ matrix.gdal }}-symlinks.tar.xz -C /
+            pip install numpy==${{ matrix.numpy }}
+            pip install gdal==${{ matrix.gdal }}
         - name: Test and build the scala JAR - skip tests is false
           if: inputs.skip_tests == 'false'
           shell: bash
           run: sudo mvn -q clean install
         - name: Build the scala JAR - skip tests is true
           if: inputs.skip_tests == 'true'
           shell: bash
-          run: sudo mvn -q clean install -DskipTests
+          run: sudo mvn -q clean install -DskipTests -Dscoverage.skip
         - name: Publish test coverage
+          if: inputs.skip_tests == 'false'
           uses: codecov/codecov-action@v1
         - name: Copy Scala artifacts to GH Actions run
           shell: bash

diff --git a/.github/workflows/build_main.yml b/.github/workflows/build_main.yml
@@ -2,10 +2,10 @@ name: build main
 on:
   push:
     branches-ignore:
-      - "R/*"
-      - "r/*"
-      - "python/*"
-      - "scala/*"
+      - "R/**"
+      - "r/**"
+      - "python/**"
+      - "scala/**"
   pull_request:
     branches:
       - "**"
@@ -16,8 +16,10 @@ jobs:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       matrix:
-        python: [ 3.8.10 ]
-        spark: [ 3.2.1 ]
+        python: [ 3.9.5 ]
+        numpy: [ 1.21.5 ]
+        gdal: [ 3.4.3 ]
+        spark: [ 3.3.2 ]
         R: [ 4.1.2 ]
     steps:
       - name: checkout code
@@ -26,5 +28,7 @@ jobs:
         uses: ./.github/actions/scala_build
       - name: build python
         uses: ./.github/actions/python_build
+      - name: build R
+        uses: ./.github/actions/r_build
       - name: upload artefacts
         uses: ./.github/actions/upload_artefacts
diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml
@@ -3,7 +3,7 @@ name: build_python
 on:
   push:
     branches:
-      - "python/*"
+      - "python/**"
 
 jobs:
   build:
@@ -12,8 +12,10 @@ jobs:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       matrix:
-        python: [ 3.8.10 ]
-        spark: [ 3.2.1 ]
+        python: [ 3.9.5 ]
+        numpy: [ 1.21.5 ]
+        gdal: [ 3.4.3 ]
+        spark: [ 3.3.2 ]
         R: [ 4.1.2 ]
     steps:
       - name: checkout code

diff --git a/.github/workflows/build_r.yml b/.github/workflows/build_r.yml
@@ -3,8 +3,8 @@ name: build_R
 on:
   push:
     branches:
-      - 'r/*'
-      - 'R/*'
+      - 'r/**'
+      - 'R/**'
 
 jobs:
   build:
@@ -13,8 +13,10 @@ jobs:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       matrix:
-        python: [ 3.8.10 ]
-        spark: [ 3.2.1 ]
+        python: [ 3.9.5 ]
+        numpy: [ 1.21.5 ]
+        gdal: [ 3.4.3 ]
+        spark: [ 3.3.2 ]
         R: [ 4.1.2 ]
     steps:
       - name: checkout code

diff --git a/.github/workflows/build_scala.yml b/.github/workflows/build_scala.yml
@@ -2,7 +2,7 @@ name: build_scala
 on:
   push:
     branches:
-      - "scala/"
+      - "scala/**"
 
 jobs:
   build:
@@ -11,8 +11,10 @@ jobs:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       matrix:
-        python: [ 3.8.10 ]
-        spark: [ 3.2.1 ]
+        python: [ 3.9.5 ]
+        numpy: [ 1.21.5 ]
+        gdal: [ 3.4.3 ]
+        spark: [ 3.3.2 ]
         R: [ 4.1.2 ]
     steps:
       - name: checkout code

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -4,6 +4,7 @@ on:
     branches:
     - main
     - feature/docs_fix
+    - feature/new_docs
 jobs:
   build:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml
@@ -9,8 +9,11 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python: [3.8.10]
-        spark: [3.2.1]
+        python: [ 3.9.5 ]
+        numpy: [ 1.21.5 ]
+        gdal: [ 3.4.3 ]
+        spark: [ 3.3.2 ]
+        R: [ 4.1.2 ]
     steps:
       - name: checkout code
         uses: actions/checkout@v2

diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,7 @@
 #IntelliJ files
 .idea
 *.iml
+tmp_
 
 #VSCode files
 .vscode
@@ -65,6 +66,7 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 /python/test/.run/
+spatial_knn
 
 # Translations
 *.mo

diff --git a/.jvmopts b/.jvmopts
@@ -0,0 +1 @@
+-Djts.overlay=ng
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,27 @@
+## v0.3.14
+- Fixes for Warning and Error messages on mosaic_enable call.
+- Performance improvements for raster functions.
+- Fix support for GDAL configuration via spark config (use 'spark.databricks.labs.mosaic.gdal.' prefix).
+
+## v0.3.13
+- R bindings generation fixed and improved.
+- Remove usage of /vsimem/ drivers for GDAL due to memory leaks.
+- Add support for MapAlgebra expressions via RST_MapAlgebra.
+- Add support for custom combine python functions via RST_DerivedBand.
+- Improve test coverage.
+- Allow for GDAL configuration via spark config (use 'spark.databricks.labs.mosaic.gdal.' prefix).
+
+## v0.3.12
+- Make JTS default Geometry Provider
+- Add raster tile functions.
+- Expand the support for raster manipulation.
+- Add abstractions for running distributed gdal_translate, gdalwarp, gdalcalc, etc.
+- Add RST_BoundingBox, RST_Clip, RST_CombineAvg, RST_CombineAvgAgg, RST_FromBands, RST_FromFile, RST_GetNoData,
+  RST_InitNoData, RST_Merge, RST_MergeAgg, RST_NDVI, RST_ReTile, RST_SetNoData, RST_Subdivide
+- Add RST_Tessellate that generates H3 tiles from rasters.
+- Add RST_ToOverlappingTiles that generates tiles with overlapping areas from rasters.
+- Add GDAL generic format reader.
+
 ## v0.3.11
 - Update the CONTRIBUTING.md to follow the standard process.
 - Fix for issue 383: grid_pointascellid fails with a Java type error when run on an already instantiated point.
@@ -172,4 +196,4 @@
 - Add Geometry validity expressions
 - Create WKT, WKB and Hex conversion expressions
 - Setup the project
-- Define GitHub templates
+- Define GitHub templates