Getting metadata from a unit test.

Signed-off-by: Jim Hughes <[email protected]>
locationtech · Dec 14, 2020 · ee6cbdb · ee6cbdb
1 parent 56fa1eb
commit ee6cbdb
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 5 deletions.
diff --git a/...-storage-api/src/main/scala/org/locationtech/geomesa/fs/storage/api/StorageMetadata.scala b/...-storage-api/src/main/scala/org/locationtech/geomesa/fs/storage/api/StorageMetadata.scala
@@ -136,6 +136,7 @@ object StorageMetadata {
     * @param timestamp timestamp for the file
     * @param action type of file (append, modify, delete)
     */
+  // TODO:  Add optional metadata bounds/info.
   case class StorageFile(name: String, timestamp: Long, action: StorageFileAction = StorageFileAction.Append)
 
   /**

diff --git a/...src/main/scala/org/locationtech/geomesa/fs/storage/common/AbstractFileSystemStorage.scala b/...src/main/scala/org/locationtech/geomesa/fs/storage/common/AbstractFileSystemStorage.scala
@@ -125,8 +125,16 @@ abstract class AbstractFileSystemStorage(
       }
       // each partition must be read separately, to ensure modifications are handled correctly
       fp.partitions.iterator.flatMap { p =>
-        val files = getFilePaths(p)
-        if (files.isEmpty) { Iterator.empty } else { Iterator.single(reader -> files) }
+        val files: Seq[StorageFilePath] = getFilePaths(p)
+        if (files.isEmpty) { Iterator.empty } else {
+          // if there's file-based metadata available
+          // filter files by metadata
+          // log changes.
+          val partition: Option[PartitionMetadata] = metadata.getPartition(p)
+          val bounds: PartitionBounds = partition.get.bounds.get
+
+          Iterator.single(reader -> files)
+        }
       }
     }
 

diff --git a/...-storage-parquet/src/test/scala/org/locationtech/geomesa/parquet/ParquetStorageTest.scala b/...-storage-parquet/src/test/scala/org/locationtech/geomesa/parquet/ParquetStorageTest.scala
@@ -109,7 +109,8 @@ class ParquetStorageTest extends Specification with AllExpectations with LazyLog
     "read and write complex features" in {
       val sft = SimpleFeatureTypes.createType("parquet-test-complex",
         "name:String,age:Int,time:Long,height:Float,weight:Double,bool:Boolean," +
-            "uuid:UUID,bytes:Bytes,list:List[Int],map:Map[String,Long]," +
+           // "uuid:UUID," +
+          "bytes:Bytes,list:List[Int],map:Map[String,Long]," +
             "line:LineString,mpt:MultiPoint,poly:Polygon,mline:MultiLineString,mpoly:MultiPolygon,g:Geometry," +
             "dtg:Date,*geom:Point:srid=4326")
 
@@ -122,7 +123,7 @@ class ParquetStorageTest extends Specification with AllExpectations with LazyLog
         sf.setAttribute("height", s"$i")
         sf.setAttribute("weight", s"$i")
         sf.setAttribute("bool", Boolean.box(i < 5))
-        sf.setAttribute("uuid", UUID.fromString(s"00000000-0000-0000-0000-00000000000$i"))
+       // sf.setAttribute("uuid", UUID.fromString(s"00000000-0000-0000-0000-00000000000$i"))
         sf.setAttribute("bytes", Array.tabulate[Byte](i)(i => i.toByte))
         sf.setAttribute("list", Seq.tabulate[Integer](i)(i => Int.box(i)))
         sf.setAttribute("map", (0 until i).map(i => i.toString -> Long.box(i)).toMap)

diff --git a/geomesa-spark/geomesa-spark-jts/src/main/scala/org/apache/spark/sql/jts/JTSTypes.scala b/geomesa-spark/geomesa-spark-jts/src/main/scala/org/apache/spark/sql/jts/JTSTypes.scala
@@ -46,7 +46,7 @@ object JTSTypes {
   )
 }
 
-private [spark] class PointUDT extends AbstractGeometryUDT[Point]("point"){
+private [spark] class PointUDT extends AbstractGeometryUDT[Point]("point") {
   override def serialize(obj: Point): InternalRow = {
     new GenericInternalRow(Array[Any](obj.getX, obj.getY))
   }