Merge branch 'release/0.10.1'

Author: metasim

.github/workflows/build-test.yml renamed to .github/disabled-workflows/build-test.yml

@@ -120,12 +120,15 @@ lazy val datasource = project
     moduleName := "rasterframes-datasource",
     libraryDependencies ++= Seq(
       compilerPlugin("org.scalamacros" % "paradise" % "2.1.1" cross CrossVersion.full),
+      compilerPlugin("org.typelevel" % "kind-projector" % "0.13.2" cross CrossVersion.full),
       sttpCatsCe2,
       stac4s,
+      framelessRefined excludeAll ExclusionRule(organization = "com.github.mpilquist"),
       geotrellis("s3").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
       spark("core").value % Provided,
       spark("mllib").value % Provided,
-      spark("sql").value % Provided
+      spark("sql").value % Provided,
+      `better-files`
     ),
     Compile / console / scalacOptions ~= { _.filterNot(Set("-Ywarn-unused-import", "-Ywarn-unused:imports")) },
     Test / console / scalacOptions ~= { _.filterNot(Set("-Ywarn-unused-import", "-Ywarn-unused:imports")) },
@@ -173,7 +176,7 @@ lazy val docs = project
     Compile / paradoxMaterialTheme ~= { _
       .withRepository(uri("https://github.com/locationtech/rasterframes"))
       .withCustomStylesheet("assets/custom.css")
-      .withCopyright("""&copy; 2017-2019 <a href="https://astraea.earth">Astraea</a>, Inc. All rights reserved.""")
+      .withCopyright("""&copy; 2017-2021 <a href="https://astraea.earth">Astraea</a>, Inc. All rights reserved.""")
       .withLogo("assets/images/RF-R.svg")
       .withFavicon("assets/images/RasterFrames_32x32.ico")
       .withColor("blue-grey", "light-blue")

build.sbt

@@ -56,7 +56,7 @@ package object rf {
 
   /** Lookup the registered Catalyst UDT for the given Scala type. */
   def udtOf[T >: Null: TypeTag]: UserDefinedType[T] =
-    UDTRegistration.getUDTFor(typeTag[T].tpe.toString).map(_.newInstance().asInstanceOf[UserDefinedType[T]])
+    UDTRegistration.getUDTFor(typeTag[T].tpe.toString).map(_.getDeclaredConstructor().newInstance().asInstanceOf[UserDefinedType[T]])
       .getOrElse(throw new IllegalArgumentException(typeTag[T].tpe + " doesn't have a corresponding UDT"))
 
   /** Creates a Catalyst expression for flattening the fields in a struct into columns. */

core/src/main/scala/org/apache/spark/sql/rf/package.scala

@@ -33,6 +33,8 @@ import org.locationtech.geomesa.spark.jts.encoders.SpatialEncoders
 import org.locationtech.rasterframes.model.{CellContext, LongExtent, TileContext, TileDataContext}
 import frameless.TypedEncoder
 import geotrellis.raster.mapalgebra.focal.{Kernel, Neighborhood, TargetCell}
+import org.locationtech.rasterframes.ref.RFRasterSource
+import org.locationtech.rasterframes.tiles.ProjectedRasterTile
 
 import java.net.URI
 import java.sql.Timestamp
@@ -45,14 +47,14 @@ trait StandardEncoders extends SpatialEncoders with TypedEncoders {
   implicit def optionalEncoder[T: TypedEncoder]: ExpressionEncoder[Option[T]] = typedExpressionEncoder[Option[T]]
 
   implicit lazy val strMapEncoder: ExpressionEncoder[Map[String, String]] = ExpressionEncoder()
-  implicit lazy val crsExpressionEncoder: ExpressionEncoder[CRS] = ExpressionEncoder()
   implicit lazy val projectedExtentEncoder: ExpressionEncoder[ProjectedExtent] = ExpressionEncoder()
   implicit lazy val temporalProjectedExtentEncoder: ExpressionEncoder[TemporalProjectedExtent] = ExpressionEncoder()
   implicit lazy val timestampEncoder: ExpressionEncoder[Timestamp] = ExpressionEncoder()
   implicit lazy val cellStatsEncoder: ExpressionEncoder[CellStatistics] = ExpressionEncoder()
   implicit lazy val cellHistEncoder: ExpressionEncoder[CellHistogram] = ExpressionEncoder()
   implicit lazy val localCellStatsEncoder: ExpressionEncoder[LocalCellStatistics] = ExpressionEncoder()
 
+  implicit lazy val crsExpressionEncoder: ExpressionEncoder[CRS] = typedExpressionEncoder
   implicit lazy val uriEncoder: ExpressionEncoder[URI] = typedExpressionEncoder[URI]
   implicit lazy val neighborhoodEncoder: ExpressionEncoder[Neighborhood] = typedExpressionEncoder[Neighborhood]
   implicit lazy val targetCellEncoder: ExpressionEncoder[TargetCell] = typedExpressionEncoder[TargetCell]
@@ -78,6 +80,11 @@ trait StandardEncoders extends SpatialEncoders with TypedEncoders {
 
   implicit lazy val tileEncoder: ExpressionEncoder[Tile] = typedExpressionEncoder
   implicit def rasterEncoder[T <: CellGrid[Int]: TypedEncoder]: ExpressionEncoder[Raster[T]] = typedExpressionEncoder[Raster[T]]
+
+  // Intentionally not implicit, defined as implicit in the ProjectedRasterTile companion object
+  lazy val projectedRasterTileEncoder: ExpressionEncoder[ProjectedRasterTile] = typedExpressionEncoder
+  // Intentionally not implicit, defined as implicit in the RFRasterSource companion object
+  lazy val rfRasterSourceEncoder: ExpressionEncoder[RFRasterSource] = typedExpressionEncoder
 }
 
 object StandardEncoders extends StandardEncoders

core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala

@@ -10,6 +10,8 @@ import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.util.QuantileSummaries
 import org.apache.spark.sql.rf.{CrsUDT, RasterSourceUDT, TileUDT}
 import org.locationtech.jts.geom.Envelope
+import org.locationtech.rasterframes.ref.RFRasterSource
+import org.locationtech.rasterframes.tiles.ProjectedRasterTile
 import org.locationtech.rasterframes.util.{FocalNeighborhood, FocalTargetCell, KryoSupport}
 
 import java.net.URI
@@ -23,6 +25,8 @@ trait TypedEncoders {
   implicit val tileUDT = new TileUDT
   implicit val rasterSourceUDT = new RasterSourceUDT
 
+  implicit val crsTypedEncoder: TypedEncoder[CRS] = TypedEncoder.usingUserDefinedType[CRS]
+
   implicit val cellTypeInjection: Injection[CellType, String] = Injection(_.toString, CellType.fromName)
   implicit val cellTypeTypedEncoder: TypedEncoder[CellType] = TypedEncoder.usingInjection[CellType, String]
 
@@ -89,7 +93,20 @@ trait TypedEncoders {
   implicit val tileTypedEncoder: TypedEncoder[Tile] = TypedEncoder.usingUserDefinedType[Tile]
   implicit def rasterTileTypedEncoder[T <: CellGrid[Int]: TypedEncoder]: TypedEncoder[Raster[T]] = TypedEncoder.usingDerivation
 
+  // Derivation is done through frameless to trigger RasterSourceUDT load
+  implicit val rfRasterSourceTypedEncoder: TypedEncoder[RFRasterSource] = TypedEncoder.usingUserDefinedType[RFRasterSource]
+
   implicit val kernelTypedEncoder: TypedEncoder[Kernel] = TypedEncoder.usingDerivation
+
+  // Derivation is done through frameless to trigger the TileUDT and CrsUDT load
+  implicit val projectedRasterTileTypedEncoder: TypedEncoder[ProjectedRasterTile] =
+    ManualTypedEncoder.newInstance[ProjectedRasterTile](
+      fields = List(
+        RecordEncoderField(0, "tile", TypedEncoder[Tile]),
+        RecordEncoderField(1, "extent", TypedEncoder[Extent]),
+        RecordEncoderField(2, "crs", TypedEncoder[CRS])
+      )
+    )
 }
 
 object TypedEncoders extends TypedEncoders

core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala

@@ -34,6 +34,7 @@ import org.apache.spark.sql.types.{DataType, StructType}
 import org.apache.spark.sql.{Column, Row, TypedColumn}
 
 class ProjectedLayerMetadataAggregate(destCRS: CRS, destDims: Dimensions[Int]) extends UserDefinedAggregateFunction {
+
   import ProjectedLayerMetadataAggregate._
 
   def inputSchema: StructType = InputRecord.inputRecordEncoder.schema
@@ -47,10 +48,10 @@ class ProjectedLayerMetadataAggregate(destCRS: CRS, destDims: Dimensions[Int]) e
   def initialize(buffer: MutableAggregationBuffer): Unit = ()
 
   def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
-    if(!input.isNullAt(0)) {
+    if (!input.isNullAt(0)) {
       val in = input.as[InputRecord]
 
-      if(buffer.isNullAt(0)) {
+      if (buffer.isNullAt(0)) {
         in.toBufferRecord(destCRS).write(buffer)
       } else {
         val br = buffer.as[BufferRecord]
@@ -71,16 +72,15 @@ class ProjectedLayerMetadataAggregate(destCRS: CRS, destDims: Dimensions[Int]) e
       case _ => ()
     }
 
-  def evaluate(buffer: Row): Any = {
-    val buf = buffer.as[BufferRecord]
-    if (buf.isEmpty) throw new IllegalArgumentException("Can not collect metadata from empty data frame.")
+  def evaluate(buffer: Row): Any =
+    Option(buffer).map(_.as[BufferRecord]).filter(!_.isEmpty).map(buf => {
+      val re = RasterExtent(buf.extent, buf.cellSize)
+      val layout = LayoutDefinition(re, destDims.cols, destDims.rows)
 
-    val re = RasterExtent(buf.extent, buf.cellSize)
-    val layout = LayoutDefinition(re, destDims.cols, destDims.rows)
+      val kb = KeyBounds(layout.mapTransform(buf.extent))
+      TileLayerMetadata(buf.cellType, layout, buf.extent, destCRS, kb).toRow
 
-    val kb = KeyBounds(layout.mapTransform(buf.extent))
-    TileLayerMetadata(buf.cellType, layout, buf.extent, destCRS, kb).toRow
-  }
+    }).getOrElse(throw new IllegalArgumentException("Can not collect metadata from empty data frame."))
 }
 
 object ProjectedLayerMetadataAggregate {

core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ProjectedLayerMetadataAggregate.scala

@@ -31,7 +31,7 @@ import geotrellis.vector.Extent
 import org.apache.hadoop.conf.Configuration
 import org.apache.spark.annotation.Experimental
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.rf.RasterSourceUDT
+import org.locationtech.rasterframes.encoders.StandardEncoders
 import org.locationtech.rasterframes.model.TileContext
 import org.locationtech.rasterframes.{NOMINAL_TILE_DIMS, rfConfig}
 
@@ -100,10 +100,7 @@ object RFRasterSource extends LazyLogging {
 
   def cacheStats = rsCache.stats()
 
-  implicit def rsEncoder: ExpressionEncoder[RFRasterSource] = {
-    RasterSourceUDT // Makes sure UDT is registered first
-    ExpressionEncoder()
-  }
+  implicit lazy val rsEncoder: ExpressionEncoder[RFRasterSource] = StandardEncoders.rfRasterSourceEncoder
 
   def apply(source: URI): RFRasterSource =
     rsCache.get(

core/src/main/scala/org/locationtech/rasterframes/ref/RFRasterSource.scala

@@ -27,6 +27,7 @@ import geotrellis.vector.{Extent, ProjectedExtent}
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.locationtech.rasterframes.ref.ProjectedRasterLike
 import org.apache.spark.sql.catalyst.DefinedByConstructorParams
+import org.locationtech.rasterframes.encoders.StandardEncoders
 
 /**
  * A Tile that's also like a ProjectedRaster, with delayed evaluation support.
@@ -58,5 +59,6 @@ object ProjectedRasterTile {
 
   def unapply(prt: ProjectedRasterTile): Option[(Tile, Extent, CRS)] = Some((prt.tile, prt.extent, prt.crs))
 
-  implicit lazy val projectedRasterTileEncoder: ExpressionEncoder[ProjectedRasterTile] = ExpressionEncoder()
+  implicit lazy val projectedRasterTileEncoder: ExpressionEncoder[ProjectedRasterTile] =
+    StandardEncoders.projectedRasterTileEncoder
 }

core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala

@@ -24,7 +24,6 @@ package org.locationtech.rasterframes.expressions
 import geotrellis.proj4.{CRS, LatLng, WebMercator}
 import geotrellis.raster.CellType
 import geotrellis.vector._
-import org.apache.spark.sql.Encoders
 import org.apache.spark.sql.jts.JTSTypes
 import org.locationtech.geomesa.curve.{XZ2SFC, Z2SFC}
 import org.locationtech.rasterframes._
@@ -151,7 +150,6 @@ class SFCIndexerSpec extends TestEnvironment with Inspectors {
       val tile = TestData.randomTile(2, 2, CellType.fromName("uint8"))
       val prts = testExtents.map(reproject(crs)).map(ProjectedRasterTile(tile, _, crs))
 
-      implicit val enc = Encoders.tuple(ProjectedRasterTile.projectedRasterTileEncoder, Encoders.scalaInt)
       // The `id` here is to deal with Spark auto projecting single columns dataframes and needing to provide an encoder
       val df = prts.zipWithIndex.toDF("proj_raster", "id")
       withClue("XZ2") {

core/src/test/scala/org/locationtech/rasterframes/expressions/SFCIndexerSpec.scala

@@ -4,3 +4,5 @@ org.locationtech.rasterframes.datasource.geotrellis.GeoTrellisCatalog
 org.locationtech.rasterframes.datasource.raster.RasterSourceDataSource
 org.locationtech.rasterframes.datasource.geojson.GeoJsonDataSource
 org.locationtech.rasterframes.datasource.stac.api.StacApiDataSource
+org.locationtech.rasterframes.datasource.tiles.TilesDataSource
+org.locationtech.rasterframes.datasource.slippy.SlippyDataSource