Merge pull request #242 from wri/upgrade-to-gdal3.8.3

GTC-2683 Upgrade to GDAL 3.8.3 and Miniconda3

.github/workflows/ci.yaml

@@ -10,7 +10,8 @@ jobs:
 
     runs-on: ubuntu-latest
     container:
-        image: quay.io/azavea/openjdk-gdal:3.1-jdk8-slim
+        image: docker.io/globalforestwatch/openjdk8-gdal3.8.3-slim:latest
+        options: --user root
 
     steps:
       - uses: actions/checkout@v2

README.md

@@ -233,9 +233,16 @@ The following options are supported:
 
 *   [`build.sbt`](build.sbt): Scala Build Tool build configuration file
 *   [`.sbtopts`](.sbtopts): Command line options for SBT, including JVM parameters
-*   [`project`](project): Additional configuration for SBT project, plugins, utility, versions
+*   [`project`](project): Additional configuration for SBT project, plugins, utility,
+    versions, including metals and bloop
 *   [`src/main/scala`](src/main/scala): Application and utility code
 *   [`src/test/scala`](src/test/scala): Unit test files
+*   [`sbt`][sbt): Script used in github CI to download sbt and all the scala
+    dependencies during CI tests (`.github/workflow/ci.yaml`)
+*   [`ci/Dockerfile`](ci/Dockerfile): Dockerfile used to build the docker image for
+    CI tests
+*   [`scripts/gdal.sh`](scripts/gdal.sh): bootscript needed for EMR runs to update
+    the image to install and use the correct version of GDAL.
 
 ## Spark Job Commands
 

build.sbt

@@ -71,6 +71,7 @@ libraryDependencies ++= Seq(
   sparkFastTests % Test,
   geotrellisS3,
   geotrellisGdal,
+  geotrellisGdalWarp,
   sedonaCore,
   sedonaSQL,
   breeze,

ci/Dockerfile

@@ -0,0 +1,33 @@
+# This Dockerfile is to create the image needed to run tests during
+# .github/workflow/ciyaml. It includes openjdk-8, git, and gdal. For some reason,
+# using Miniconda3 to load gdal broken the libraries needed for git, so we stick with
+# old Miniconda here.
+#
+# Created via:
+#  docker build -t test -f Dockerfile .
+#  docker tag test:latest globalforestwatch/openjdk8-gdal3.8.3-slim:latest
+#  docker login -u username
+#  docker push globalforestwatch/openjdk8-gdal3.8.3-slim:latest
+
+FROM openjdk:8
+
+ENV SCALA_VERSION=2.12.12
+ENV PYTHON_VERSION=3.8
+ENV GDAL_VERSION=3.8.3
+
+USER root
+
+# Install GDAL using Conda
+RUN wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh
+RUN bash Miniconda-latest-Linux-x86_64.sh -b -p /usr/local/miniconda
+
+ENV PATH=/usr/local/miniconda/bin:$PATH
+
+RUN conda config --add channels conda-forge && conda config --set ssl_verify false
+RUN /usr/local/miniconda/bin/conda install python=${PYTHON_VERSION} -y
+RUN pip install tqdm six
+RUN /usr/local/miniconda/bin/conda install -c anaconda hdf5 -y
+RUN /usr/local/miniconda/bin/conda install -c conda-forge libnetcdf gdal=${GDAL_VERSION} -y
+
+ENV LD_LIBRARY_PATH=/usr/local/miniconda/lib/:/usr/local/lib:/usr/lib/hadoop/lib/native:/usr/lib/hadoop-lzo/lib/native:/docker/usr/lib/hadoop/lib/native:/docker/usr/lib/hadoop-lzo/lib/native:/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib
+ENV PROJ_LIB=/usr/local/miniconda/share/proj

project/Dependencies.scala

@@ -49,6 +49,8 @@ object Dependencies {
   val geotrellisSparkTestKit = "org.locationtech.geotrellis" %% "geotrellis-spark-testkit" % Version.geotrellis
   val geotrellisRasterTestkit = "org.locationtech.geotrellis" %% "geotrellis-raster-testkit" % Version.geotrellis
   val geotrellisGdal = "org.locationtech.geotrellis" %% "geotrellis-gdal" % Version.geotrellis
+  // 3.8.0 warp bindings corresponds to gdal 3.8.x; this will not necessarily match `Version.geotrellis`
+  val geotrellisGdalWarp = "com.azavea.geotrellis" % "gdal-warp-bindings" % "3.8.0"
 
   val pureconfig = "com.github.pureconfig" %% "pureconfig" % "0.9.1"
   val logging = "com.typesafe.scala-logging" %% "scala-logging" % "3.9.0"

scripts/gdal.sh

@@ -1,25 +1,25 @@
 #!/bin/bash
 
+# This is the bootstrap script used to install GDAL on individual EMR nodes before
+# they run.
+
 set -ex
 
-# The default GDAL version would be 3.1.2
+# The default GDAL version would be 3.8.3
 GDAL_VERSION=$1
-GDAL_VERSION=${GDAL_VERSION:="3.1.2"}
+GDAL_VERSION=${GDAL_VERSION:="3.8.3"}
+
+# Install GDAL using Miniconda3 (which includes Python 3.12 by default)
+# Avoid using default anaconda repo.
 
-# Install Conda
-wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh
-sudo sh Miniconda-latest-Linux-x86_64.sh -b -p /usr/local/miniconda
+wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+sudo sh Miniconda3-latest-Linux-x86_64.sh -b -p /usr/local/miniconda
 
 source ~/.bashrc
 export PATH=/usr/local/miniconda/bin:$PATH
 
-# Install GDAL with a specific libcurl version (7.76.1)
-conda config --add channels conda-forge
-sudo pip3 install tqdm && \
-sudo /usr/local/miniconda/bin/conda install python=3.6 -y && \
-sudo /usr/local/miniconda/bin/conda install -c anaconda hdf5 -y && \
-sudo /usr/local/miniconda/bin/conda install -c cctbx202105 libcurl=7.76.1 -y && \
-sudo /usr/local/miniconda/bin/conda install -c conda-forge libnetcdf gdal=${GDAL_VERSION} -y
+sudo pip3 install tqdm six && \
+sudo /usr/local/miniconda/bin/conda install -c conda-forge  --override-channels hdf5 libnetcdf gdal=${GDAL_VERSION} -y
 
 echo "export PATH=/usr/local/miniconda/bin:$PATH" >> ~/.bashrc
 echo "export LD_LIBRARY_PATH=/usr/local/miniconda/lib/:/usr/local/lib:/usr/lib/hadoop/lib/native:/usr/lib/hadoop-lzo/lib/native:/docker/usr/lib/hadoop/lib/native:/docker/usr/lib/hadoop-lzo/lib/native:/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib" >> ~/.bashrc

src/main/scala/org/globalforestwatch/summarystats/SummaryMain.scala

@@ -30,6 +30,11 @@ object SummaryMain {
   val command = Command(name, header, true)(main)
 
   final def main(args: Array[String]): Unit = {
+    // Print out environment variables (for debugging purposes)
+    val environmentVars = System.getenv().forEach {
+      case (key, value) => println(s"$key = $value")
+    }
+
     command.parse(args, sys.env) match {
       case Left(help) =>
         System.err.println(help)