Merge branch 'databrickslabs:main' into mjohns/geoserver-connector

.github/actions/scala_build/action.yml

@@ -15,11 +15,11 @@ runs:
         - name: Test and build the scala JAR - skip tests is false
           if: inputs.skip_tests == 'false'
           shell: bash
-          run: mvn -q clean install
+          run: sudo mvn -q clean install
         - name: Build the scala JAR - skip tests is true
           if: inputs.skip_tests == 'true'
           shell: bash
-          run: mvn -q clean install -DskipTests
+          run: sudo mvn -q clean install -DskipTests
         - name: Publish test coverage
           uses: codecov/codecov-action@v1
         - name: Copy Scala artifacts to GH Actions run

.github/workflows/build_main.yml

@@ -26,7 +26,5 @@ jobs:
         uses: ./.github/actions/scala_build
       - name: build python
         uses: ./.github/actions/python_build
-      - name: build r
-        uses: ./.github/actions/r_build
       - name: upload artefacts
         uses: ./.github/actions/upload_artefacts

.github/workflows/pypi-release.yml

@@ -12,29 +12,12 @@ jobs:
         python: [3.8.10]
         spark: [3.2.1]
     steps:
-      - name: Checkout code
+      - name: checkout code
         uses: actions/checkout@v2
-      - name: Configure JDK
-        uses: actions/setup-java@v2
-        with:
-          java-version: '8'
-          distribution: 'zulu'
-      - name: Test and build the scala JAR
-        run: mvn -q clean install
-      - name: Configure python interpreter
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python }}
-      - name: Install python dependencies
-        run: |
-          cd python
-          pip install build wheel pyspark==${{ matrix.spark }}
-          pip install .
-      - name: Test and build python package
-        run: |
-          cd python 
-          python -m unittest
-          python -m build
+      - name: build scala
+        uses: ./.github/actions/scala_build
+      - name: build python
+        uses: ./.github/actions/python_build
       - name: Publish a Python distribution to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

.gitignore

@@ -17,7 +17,7 @@ __pycache__/
 *$py.class
 
 # C extensions
-*.so
+#*.so
 
 # Distribution / packaging
 .Python
@@ -156,3 +156,4 @@ spark-warehouse
 # MacOS
 .DS_Store
 .Rproj.user
+docker/.m2/

CHANGELOG.md

@@ -1,3 +1,44 @@
+## v0.3.7
+- Fixed pip release publish script
+
+## v0.3.6
+- Added GDAL and 32 rst_* raster functions:
+  - RST_BandMetaData
+  - RST_GeoReference
+  - RST_IsEmpty
+  - RST_MemSize
+  - RST_MetaData
+  - RST_NumBands
+  - RST_PixelWidth
+  - RST_PixelHeight
+  - RST_RasterToGridAvg
+  - RST_RasterToGridMax
+  - RST_RasterToGridMin
+  - RST_RasterToGridMedian
+  - RST_RasterToGridCount
+  - RST_RasterToWorldCoord
+  - RST_RasterToWorldCoordX
+  - RST_RasterToWorldCoordY
+  - RST_ReTile
+  - RST_Rotation
+  - RST_ScaleX
+  - RST_ScaleY
+  - RST_SkewX
+  - RST_SkewY
+  - RST_SRID
+  - RST_Subdatasets
+  - RST_Summary
+  - RST_UpperLeftX
+  - RST_UpperLeftY
+  - RST_Width
+  - RST_Height
+  - RST_WorldToRasterCoord
+  - RST_WorldToRasterCoordX
+  - RST_WorldToRasterCoordY
+- Fixed geometry creation from empty Seq
+- Fixed landmarks_miid and candidates_miid column names parameter in KNN
+- Improved docs
+
 ## v0.3.5
 - Implemented KNN (K Nearest Neighbours) transformer
 - Implemented grid cell functions `grid_cellkring`, `grid_cellkloop`, `grid_cellkringexplode`, and `grid_cellkloopexplode`

R/generate_R_bindings.R

@@ -72,6 +72,9 @@ build_method<-function(input){
     else if(x[2] == 'String'){
       x[2] = "character"
     }
+    else if(x[2] == 'Double'){
+      x[2] = "numeric"
+    }
     x
   }
   # convert scala type to R types
@@ -149,7 +152,7 @@ get_function_names <- function(scala_file_path){
     )
   }
   ))
-  methods_to_bind
+  sort(methods_to_bind, T)
 }
 
 ############################################################

R/sparkR-mosaic/enableMosaic.R

@@ -4,6 +4,7 @@
 #' See \url{https://databrickslabs.github.io/mosaic/} for full documentation
 #' @param geometryAPI character, default="ESRI"
 #' @param indexSystem character, default="H3"
+#' @param indexSystem boolean, default=F
 #' @name enableMosaic
 #' @rdname enableMosaic
 #' @return None
@@ -16,11 +17,15 @@
 enableMosaic <- function(
   geometryAPI="ESRI"
   ,indexSystem="H3"
+  ,rasterAPI="GDAL"
 ){
   geometry_api <- sparkR.callJStatic(x="com.databricks.labs.mosaic.core.geometry.api.GeometryAPI", methodName="apply", geometryAPI)
   index_system_id <- sparkR.callJStatic(x="com.databricks.labs.mosaic.core.index.IndexSystemID", methodName="apply", indexSystem)
   indexing_system <- sparkR.callJStatic(x="com.databricks.labs.mosaic.core.index.IndexSystemID", methodName="getIndexSystem", index_system_id)
-  mosaic_context <- sparkR.newJObject(x="com.databricks.labs.mosaic.functions.MosaicContext", indexing_system, geometry_api)
+
+  raster_api <- sparkR.callJStatic(x="com.databricks.labs.mosaic.core.raster.api.RasterAPI", methodName="apply", rasterAPI)
+
+  mosaic_context <- sparkR.newJObject(x="com.databricks.labs.mosaic.functions.MosaicContext", indexing_system, geometry_api, raster_api)
   functions <<- sparkR.callJMethod(mosaic_context, "functions")
   # register the sql functions for use in sql() commands
   sparkR.callJMethod(mosaic_context, "register")

R/sparkR-mosaic/sparkrMosaic/DESCRIPTION

@@ -1,6 +1,6 @@
 Package: sparkrMosaic
 Title: SparkR bindings for Databricks Mosaic
-Version: 0.3.6
+Version: 0.3.7
 Authors@R: 
     person("Robert", "Whiffin", , "[email protected]", role = c("aut", "cre")
            )

R/sparklyr-mosaic/enableMosaic.R

@@ -19,12 +19,14 @@ enableMosaic <- function(
     sc
     ,geometryAPI="ESRI"
     ,indexSystem="H3"
+    ,rasterAPI="GDAL"
 ){
 
   geometry_api <- sparklyr::invoke_static(sc, class="com.databricks.labs.mosaic.core.geometry.api.GeometryAPI", method="apply", geometryAPI)
   index_system_id <- sparklyr::invoke_static(sc, class="com.databricks.labs.mosaic.core.index.IndexSystemID", method="apply", indexSystem)
+  raster_api <- sparklyr::invoke_static(sc, class="com.databricks.labs.mosaic.core.raster.api.RasterAPI", method="apply", rasterAPI)
   indexing_system <- sparklyr::invoke_static(sc, class="com.databricks.labs.mosaic.core.index.IndexSystemID", method="getIndexSystem", index_system_id)
-  mosaic_context <- sparklyr::invoke_new(sc, class="com.databricks.labs.mosaic.functions.MosaicContext", indexing_system, geometry_api)
+  mosaic_context <- sparklyr::invoke_new(sc, class="com.databricks.labs.mosaic.functions.MosaicContext", indexing_system, geometry_api, raster_api)
   functions <<- sparklyr::invoke(mosaic_context, "functions")
   sparklyr::invoke(mosaic_context, "register")
 

R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION

@@ -1,6 +1,6 @@
 Package: sparklyrMosaic
 Title: sparklyr bindings for Databricks Mosaic
-Version: 0.3.6
+Version: 0.3.7
 Authors@R: 
     person("Robert", "Whiffin", , "[email protected]", role = c("aut", "cre")
            )

README.md

@@ -6,7 +6,7 @@ An extension to the [Apache Spark](https://spark.apache.org/) framework that all
 [![PyPI version](https://badge.fury.io/py/databricks-mosaic.svg)](https://badge.fury.io/py/databricks-mosaic)
 ![PyPI - Downloads](https://img.shields.io/pypi/dm/databricks-mosaic?style=plastic)
 [![codecov](https://codecov.io/gh/databrickslabs/mosaic/branch/main/graph/badge.svg?token=aEzZ8ITxdg)](https://codecov.io/gh/databrickslabs/mosaic)
-[![build](https://github.com/databrickslabs/mosaic/actions/workflows/build.yml/badge.svg)](https://github.com/databrickslabs/mosaic/actions/workflows/build.yml)
+[![build](https://github.com/databrickslabs/mosaic/actions/workflows/build_main.yml/badge.svg)](https://github.com/databrickslabs/mosaic/actions?query=workflow%3A%22build+main%22)
 [![docs](https://github.com/databrickslabs/mosaic/actions/workflows/docs.yml/badge.svg)](https://github.com/databrickslabs/mosaic/actions/workflows/docs.yml)
 [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/databrickslabs/mosaic.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/databrickslabs/mosaic/context:python)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)

docs/code-example-notebooks/spatial-indexes.py

@@ -101,11 +101,11 @@
 
 # COMMAND ----------
 
-from mosaic import polyfill
+from mosaic import grid_polyfill
 
 indexed_neighbourhoods = (
   neighbourhoods
-  .select("*", polyfill("geometry", lit(resolution)).alias("ix_set"))
+  .select("*", grid_polyfill("geometry", lit(resolution)).alias("ix_set"))
   .drop("geometry")
 )
 

docs/source/api/api.rst

@@ -9,4 +9,5 @@ API Documentation
    spatial-functions
    spatial-indexing
    spatial-predicates
-   spatial-aggregations
+   spatial-aggregations
+   raster-functions