Merge branch 'release/0.8.5'

Author: metasim

.circleci/config.yml

@@ -5,7 +5,7 @@ _defaults: &defaults
   environment:
     TERM: dumb
   docker:
-    - image: s22s/rasterframes-circleci:latest
+    - image: s22s/rasterframes-circleci:9b7682ef
 
 _setenv: &setenv
   name: set CloudRepo credentials
@@ -195,6 +195,7 @@ workflows:
             branches:
               only:
                 - /feature\/.*docs.*/
+                - /fix\/.*docs.*/
                 - /docs\/.*/
 
   nightly:

RELEASE.md

@@ -0,0 +1,23 @@
+# RasterFrames Release Process
+
+1. Make sure `release-notes.md` is updated.
+2. Use `git flow release start x.y.z` to create release branch.
+3. Manually edit `version.sbt` and `version.py` to set value to `x.y.z` and commit changes.
+4. Do `docker login` if necessary.
+5. `sbt` shell commands:  
+    a. `clean`  
+    b. `test it:test`  
+    c. `makeSite`  
+    d. `publishSigned` (LocationTech credentials required)  
+    e. `sonatypeReleaseAll`. It can take a while, but should eventually show up [here](https://search.maven.org/search?q=g:org.locationtech.rasterframes).  
+    f. `docs/ghpagesPushSite`  
+    g. `rf-notebook/publish`  
+6. `cd pyrasterframes/target/python/dist`
+7. `python3 -m twine upload pyrasterframes-x.y.z-py2.py3-none-any.whl`
+8. Commit any changes that were necessary.
+9. `git-flow finish release`. Make sure to push tags, develop and master
+    branches.
+10. On `develop`, update `version.sbt` and `version.py` to next development
+    version (`x.y.(z+1)-SNAPSHOT` and `x.y.(z+1).dev0`). Commit and push.
+11. In GitHub, create a new release with the created tag. Copy relevant
+    section of release notes into the description.

build.sbt

@@ -34,7 +34,7 @@ lazy val root = project
   .enablePlugins(RFReleasePlugin)
   .settings(
     publish / skip := true,
-    clean := clean.dependsOn(`rf-notebook`/clean).value
+    clean := clean.dependsOn(`rf-notebook`/clean, docs/clean).value
   )
 
 lazy val `rf-notebook` = project
@@ -76,7 +76,6 @@ lazy val core = project
     buildInfoObject := "RFBuildInfo",
     buildInfoOptions := Seq(
       BuildInfoOption.ToMap,
-      BuildInfoOption.BuildTime,
       BuildInfoOption.ToJson
     )
   )

build/circleci/Dockerfile

@@ -2,6 +2,7 @@ FROM circleci/openjdk:8-jdk
 
 ENV OPENJPEG_VERSION 2.3.1
 ENV GDAL_VERSION 2.4.1
+ENV SPATIALINDEX_VERSION 1.9.3
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
 
 # most of these libraries required for
@@ -74,3 +75,15 @@ RUN \
     sudo make install && \
     sudo ldconfig && \
     cd /tmp && sudo rm -Rf gdal*
+
+# Compile and install libspatialindex
+RUN \
+    cd /tmp && \
+    wget https://github.com/libspatialindex/libspatialindex/releases/download/${SPATIALINDEX_VERSION}/spatialindex-src-${SPATIALINDEX_VERSION}.tar.gz && \
+    tar -xf spatialindex-src-${SPATIALINDEX_VERSION}.tar.gz && \
+    cd spatialindex-src-${SPATIALINDEX_VERSION}/ && \
+    cmake  -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
+    make && \
+    sudo make install && \
+    sudo ldconfig && \
+    cd /tmp && sudo rm -Rf spatialindex*

core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala

@@ -30,7 +30,7 @@ import org.locationtech.rasterframes.expressions.aggregates.TileRasterizerAggreg
 
 class RasterRefIT extends TestEnvironment {
   describe("practical subregion reads") {
-    ignore("should construct a natural color composite") {
+    it("should construct a natural color composite") {
       import spark.implicits._
       def scene(idx: Int) = URI.create(s"https://landsat-pds.s3.us-west-2.amazonaws.com" +
         s"/c1/L8/176/039/LC08_L1TP_176039_20190703_20190718_01_T1/LC08_L1TP_176039_20190703_20190718_01_T1_B$idx.TIF")
@@ -55,11 +55,11 @@ class RasterRefIT extends TestEnvironment {
         stats.get.dataCells shouldBe > (1000L)
       }
 
-      //import geotrellis.raster.io.geotiff.{GeoTiffOptions, MultibandGeoTiff, Tiled}
-      //import geotrellis.raster.io.geotiff.compression.{DeflateCompression, NoCompression}
-      //import geotrellis.raster.io.geotiff.tags.codes.ColorSpace
-      //val tiffOptions = GeoTiffOptions(Tiled,  DeflateCompression, ColorSpace.RGB)
-      //MultibandGeoTiff(raster, raster.crs, tiffOptions).write("target/composite.tif")
+      import geotrellis.raster.io.geotiff.compression.DeflateCompression
+      import geotrellis.raster.io.geotiff.tags.codes.ColorSpace
+      import geotrellis.raster.io.geotiff.{GeoTiffOptions, MultibandGeoTiff, Tiled}
+      val tiffOptions = GeoTiffOptions(Tiled,  DeflateCompression, ColorSpace.RGB)
+      MultibandGeoTiff(raster.raster, raster.crs, tiffOptions).write("target/composite.tif")
     }
   }
 }

core/src/main/resources/reference.conf

@@ -17,6 +17,8 @@ vlm.gdal {
       GDAL_CACHEMAX = 512
       GDAL_PAM_ENABLED = "NO"
       CPL_VSIL_CURL_CHUNK_SIZE = 1000000
+      GDAL_HTTP_MAX_RETRY=4
+      GDAL_HTTP_RETRY_DELAY=1
     }
     // set this to `false` if CPL_DEBUG is `ON`
     useExceptions = true

core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala

@@ -38,7 +38,7 @@ import org.locationtech.rasterframes.util.KryoSupport
 @SQLUserDefinedType(udt = classOf[RasterSourceUDT])
 class RasterSourceUDT extends UserDefinedType[RasterSource] {
   import RasterSourceUDT._
-  override def typeName = "rf_rastersource"
+  override def typeName = "rastersource"
 
   override def pyUDT: String = "pyrasterframes.rf_types.RasterSourceUDT"
 
@@ -58,7 +58,6 @@ class RasterSourceUDT extends UserDefinedType[RasterSource] {
       }
       .orNull
 
-
   private[sql] override def acceptsType(dataType: DataType) = dataType match {
     case _: RasterSourceUDT ⇒ true
     case _ ⇒ super.acceptsType(dataType)

core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala

@@ -73,7 +73,7 @@ case object TileUDT  {
 
   final val typeName: String = "tile"
 
-  implicit def tileSerializer: CatalystSerializer[Tile] = new CatalystSerializer[Tile] {
+  implicit val tileSerializer: CatalystSerializer[Tile] = new CatalystSerializer[Tile] {
 
     override val schema: StructType = StructType(Seq(
       StructField("cell_context", schemaOf[TileDataContext], true),