Merge branch 'databrickslabs-main'

.github/actions/scala_build/action.yml

@@ -27,7 +27,7 @@ runs:
             sudo apt-get update -y
             # - install natives
             sudo apt-get install -y unixodbc libcurl3-gnutls libsnappy-dev libopenjp2-7
-            sudo apt-get install -y gdal-bin libgdal-dev python3-numpy python3-gdal
+            sudo apt-get install -y gdal-bin libgdal-dev python3-numpy python3-gdal zip unzip
             # - install pip libs
             pip install --upgrade pip
             pip install gdal==${{ matrix.gdal }}

R/sparkR-mosaic/sparkrMosaic/DESCRIPTION

@@ -8,7 +8,7 @@ Description: This package extends SparkR to bring the Databricks Mosaic for geos
 License: Databricks
 Encoding: UTF-8
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.2.3
+RoxygenNote: 7.3.1
 Collate:
     'enableGDAL.R'
     'enableMosaic.R'
@@ -20,4 +20,4 @@ Imports:
 Suggests:
     testthat (>= 3.0.0),
     readr (>= 2.1.5)
-Config/testthat/edition: 3
+Config/testthat/edition: 3

R/sparkR-mosaic/sparkrMosaic/tests/testthat/testRasterFunctions.R

@@ -15,7 +15,7 @@ test_that("mosaic can read single-band GeoTiff", {
   expect_equal(row$srid, 0)
   expect_equal(row$bandCount, 1)
   expect_equal(row$metadata[[1]]$LONGNAME, "MODIS/Terra+Aqua BRDF/Albedo Nadir BRDF-Adjusted Ref Daily L3 Global - 500m")
-  expect_equal(row$tile[[1]]$driver, "GTiff")
+  expect_equal(row$tile[[1]]$metadata$driver, "GTiff")
 
 })
 
@@ -61,7 +61,7 @@ test_that("raster flatmap functions behave as intended", {
   tessellate_sdf <- withColumn(tessellate_sdf, "rst_tessellate", rst_tessellate(column("tile"), lit(3L)))
 
   expect_no_error(write.df(tessellate_sdf, source = "noop", mode = "overwrite"))
-  expect_equal(nrow(tessellate_sdf), 66)
+  expect_equal(nrow(tessellate_sdf), 63)
 
   overlap_sdf <- generate_singleband_raster_df()
   overlap_sdf <- withColumn(overlap_sdf, "rst_to_overlapping_tiles", rst_to_overlapping_tiles(column("tile"), lit(200L), lit(200L), lit(10L)))
@@ -117,7 +117,7 @@ test_that("the tessellate-join-clip-merge flow works on NetCDF files", {
   raster_sdf <- read.df(
     path = "sparkrMosaic/tests/testthat/data/prAdjust_day_HadGEM2-CC_SMHI-DBSrev930-GFD-1981-2010-postproc_rcp45_r1i1p1_20201201-20201231.nc",
     source = "gdal",
-    raster.read.strategy = "retile_on_read"
+    raster.read.strategy = "in_memory"
   )
 
   raster_sdf <- withColumn(raster_sdf, "tile", rst_separatebands(column("tile")))

R/sparkR-mosaic/tests.R

@@ -21,11 +21,15 @@ print("Looking for mosaic jar in")
 mosaic_jar_path <- paste0(staging_dir, mosaic_jar)
 print(mosaic_jar_path)
 
+pwd <- getwd()
 spark <- sparkR.session(
   master = "local[*]"
-  ,sparkJars = mosaic_jar_path
+  ,sparkJars = mosaic_jar_path,
+  sparkConfig = list(
+    spark.databricks.labs.mosaic.raster.tmp.prefix = paste0(pwd, "/mosaic_tmp", sep="")
+    ,spark.databricks.labs.mosaic.raster.checkpoint = paste0(pwd, "/mosaic_checkpoint", sep="")
+  )
 )
-
 enableMosaic()
 
 testthat::test_local(path="./sparkrMosaic")

R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION

@@ -8,7 +8,7 @@ Description: This package extends sparklyr to bring the Databricks Mosaic for ge
 License: Databricks
 Encoding: UTF-8
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.2.3
+RoxygenNote: 7.3.1
 Collate:
     'enableGDAL.R'
     'enableMosaic.R'
@@ -20,4 +20,4 @@ Suggests:
     testthat (>= 3.0.0),
     sparklyr.nested (>= 0.0.4),
     readr (>= 2.1.5)
-Config/testthat/edition: 3
+Config/testthat/edition: 3

R/sparklyr-mosaic/sparklyrMosaic/tests/testthat/testRasterFunctions.R

@@ -18,7 +18,7 @@ test_that("mosaic can read single-band GeoTiff", {
   expect_equal(row$srid, 0)
   expect_equal(row$bandCount, 1)
   expect_equal(row$metadata[[1]]$LONGNAME, "MODIS/Terra+Aqua BRDF/Albedo Nadir BRDF-Adjusted Ref Daily L3 Global - 500m")
-  expect_equal(row$tile[[1]]$driver, "GTiff")
+  expect_equal(row$tile[[1]]$metadata$driver, "GTiff")
 
 })
 
@@ -90,7 +90,7 @@ test_that("raster flatmap functions behave as intended", {
     mutate(rst_tessellate = rst_tessellate(tile, 3L))
 
   expect_no_error(spark_write_source(tessellate_sdf, "noop", mode = "overwrite"))
-  expect_equal(sdf_nrow(tessellate_sdf), 66)
+  expect_equal(sdf_nrow(tessellate_sdf), 63)
 
   overlap_sdf <- generate_singleband_raster_df() %>%
     mutate(rst_to_overlapping_tiles = rst_to_overlapping_tiles(tile, 200L, 200L, 10L))
@@ -157,7 +157,7 @@ test_that("the tessellate-join-clip-merge flow works on NetCDF files", {
       name = "raster_raw",
       source = "gdal",
       path = "data/prAdjust_day_HadGEM2-CC_SMHI-DBSrev930-GFD-1981-2010-postproc_rcp45_r1i1p1_20201201-20201231.nc",
-      options = list("raster.read.strategy" = "retile_on_read")
+      options = list("raster.read.strategy" = "in_memory")
     ) %>%
       mutate(tile = rst_separatebands(tile)) %>%
       sdf_register("raster")

R/sparklyr-mosaic/sparklyrMosaic/tests/testthat/testVectorFunctions.R

@@ -15,6 +15,7 @@ test_that("scalar vector functions behave as intended", {
     st_length = st_length(wkt),
     st_perimeter = st_perimeter(wkt),
     st_buffer = st_buffer(wkt, as.double(1.1)),
+    st_buffer_optparams = st_buffer(wkt, as.double(1.1), "endcap=square quad_segs=2"),
     st_bufferloop = st_bufferloop(wkt, as.double(1.1), as.double(1.2)),
     st_convexhull = st_convexhull(wkt),
     st_dump = st_dump(wkt),
@@ -122,4 +123,4 @@ test_that("aggregate vector functions behave as intended", {
   expect_true(sdf.intersection %>% head(1) %>% sdf_collect %>% .$comparison_intersection)
 
 
-})
+})

R/sparklyr-mosaic/tests.R

@@ -21,6 +21,8 @@ print(paste("Looking for mosaic jar in", mosaic_jar_path))
 
 config <- sparklyr::spark_config()
 config$`sparklyr.jars.default` <- c(mosaic_jar_path)
+config$`spark.databricks.labs.mosaic.raster.tmp.prefix` <- paste0(getwd(), "/mosaic_tmp", sep="")
+config$`spark.databricks.labs.mosaic.raster.checkpoint` <- paste0(getwd(), "/mosaic_checkpoint", sep="")
 
 sc <- spark_connect(master="local[*]", config=config)
 enableMosaic(sc)

README.md

@@ -10,6 +10,7 @@ An extension to the [Apache Spark](https://spark.apache.org/) framework that all
 [![docs](https://github.com/databrickslabs/mosaic/actions/workflows/docs.yml/badge.svg)](https://github.com/databrickslabs/mosaic/actions/workflows/docs.yml)
 [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/databrickslabs/mosaic.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/databrickslabs/mosaic/context:python)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![lines of code](https://tokei.rs/b1/github/databrickslabs/mosaic)]([https://codecov.io/github/databrickslabs/mosaic](https://github.com/databrickslabs/mosaic))
 
 
 ## Why Mosaic?

docs/source/api/spatial-functions.rst

@@ -125,11 +125,17 @@ st_buffer
 .. function:: st_buffer(col, radius)
 
     Buffer the input geometry by radius :code:`radius` and return a new, buffered geometry.
+    The optional parameter buffer_style_parameters='quad_segs=# endcap=round|flat|square' where "#"
+    is the number of line segments used to approximate a quarter circle (default is 8); and endcap
+    style for line features is one of listed (default="round")
+
 
     :param col: Geometry
     :type col: Column
     :param radius: Double
     :type radius: Column (DoubleType)
+    :param buffer_style_parameters: String
+    :type buffer_style_parameters: Column (StringType)
     :rtype: Column: Geometry
 
     :example:
@@ -1535,7 +1541,7 @@ st_transform
 
     df <- createDataFrame(data.frame(wkt = "MULTIPOINT ((10 40), (40 30), (20 20), (30 10))"))
     df <- withColumn(df, 'geom', st_setsrid(st_asgeojson(column('wkt')), lit(4326L)))
-    >>>
+
     showDF(select(df, st_astext(st_transform(column('geom'), lit(3857L)))), truncate=F)
     +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
     |convert_to(st_transform(geom, 3857))                                                                                                                                      |