Merge pull request #17 from USGS-EROS/develop

0.5 Release

Author: davidvhill

.dockerignore

@@ -1,3 +1,7 @@
 .git/
 .hypothesis/
 lib/
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class

.test_env

@@ -1,4 +1,4 @@
-export SPARK_HOME=$(pwd)/lib/spark
+export SPARK_HOME="${SPARK_HOME:=$(pwd)/lib/spark}"
 export PATH=$PATH:$SPARK_HOME/bin
 export PYTHONPATH=$PYTHONPATH:$SPARK_HOME/python/lib/pyspark.zip
 export PYTHONPATH=$PYTHONPATH:$SPARK_HOME/python/lib/py4j-0.10.4-src.zip

Dockerfile

@@ -1,23 +1,36 @@
 FROM centos:7.3.1611
 
-LABEL maintainer="USGS EROS LCMAP http://eros.usgs.gov http://github.com/usgs-eros/lcmap-firebird"
-LABEL description="CentOS based Spark-Mesos image for LCMAP"
-LABEL org.apache.mesos.version=1.4.0
-LABEL org.apache.spark.version=2.2.0
-LABEL net.java.openjdk.version=1.8.0
-LABEL org.python.version=3.6
-LABEL org.centos=7.3.1611
+LABEL maintainer="USGS EROS LCMAP http://eros.usgs.gov http://github.com/usgs-eros/lcmap-firebird" \
+      description="CentOS based Spark-Mesos image for LCMAP" \
+      org.apache.mesos.version=1.4.0 \
+      org.apache.spark.version=2.2.0 \
+      net.java.openjdk.version=1.8.0 \
+      org.python.version=3.6 \
+      org.centos=7.3.1611
 
 EXPOSE 8081 4040 8888
 
 RUN yum update -y && \
-    yum install -y sudo gcc bzip2 java-1.8.0-openjdk-devel.x86_64 && \
+    yum install -y sudo java-1.8.0-openjdk-devel.x86_64 && \
     yum install -y http://repos.mesosphere.io/el/7/noarch/RPMS/mesosphere-el-repo-7-3.noarch.rpm && \
     yum install -y mesos && \
-    yum -y downgrade mesos-1.4.0
+    yum -y downgrade mesos-1.4.0 && \
+    sudo yum clean all && \
+    sudo rm -rf /var/cache/yum && \
+    localedef -i en_US -f UTF-8 en_US.UTF-8
 
-RUN localedef -i en_US -f UTF-8 en_US.UTF-8
+ENV HOME=/home/lcmap \
+    USER=lcmap \
+    SPARK_HOME=/opt/spark \
+    SPARK_NO_DAEMONIZE=true \
+    PYSPARK_PYTHON=python3 \
+    MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so \    
+    LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8
 
+ENV PATH=$SPARK_HOME/bin:${PATH}:$HOME/miniconda3/bin \
+    PYTHONPATH=$PYTHONPATH:$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.4-src.zip:$SPARK_HOME/python/lib/pyspark.zip
+    
 ##########################################################################
 # Add a user to run as inside the container.  This will prevent accidental
 # foo while mounting volumes.  To enable access between external and
@@ -27,51 +40,51 @@ RUN adduser -ms /bin/bash lcmap && \
     echo "lcmap ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/lcmap && \
     chmod 0440 /etc/sudoers.d/lcmap
 
-ENV HOME=/home/lcmap
-ENV USER=lcmap
 USER $USER
 WORKDIR $HOME
 ##########################################################################
 
-ENV SPARK_HOME=/opt/spark
-ENV SPARK_NO_DAEMONIZE=true
-ENV PYSPARK_PYTHON=$HOME/miniconda3/bin/python3
-ENV MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so
-ENV PATH=$HOME/miniconda3/bin:$SPARK_HOME/bin:${PATH}
-ENV PYTHONPATH=$PYTHONPATH:$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.4-src.zip:$SPARK_HOME/python/lib/pyspark.zip
-ENV LC_ALL=en_US.UTF-8
-ENV LANG=en_US.UTF-8
-
+# Install Spark
 RUN cd /opt && \
     sudo curl https://d3kbcqa49mib13.cloudfront.net/spark-2.2.0-bin-hadoop2.7.tgz -o spark.tgz && \
     sudo tar -zxf spark.tgz && \
     sudo rm -f spark.tgz && \
     sudo ln -s spark-* spark
 
-RUN curl https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o mc.sh && \
+# Install miniconda.
+# Install numpy from default repo for mkl based implementation.
+RUN sudo yum install -y bzip2 && \
+    curl https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o mc.sh && \
     chmod 755 mc.sh && \
     ./mc.sh -b && \
     rm -rf mc.sh && \
-    conda config --add channels conda-forge && \
-    conda install cython gdal --yes
+    sudo yum erase -y bzip2 && \
+    sudo yum clean all && \
+    conda install python=3.6 pip numpy cython scipy scikit-learn=0.18 --yes
 
-COPY firebird firebird
-COPY notebook notebook
-COPY resources resources
-COPY test test
-COPY Makefile .
-COPY pom.xml .
-COPY README.md .
-COPY setup.py .
-COPY version.txt .
+# Copy firebird project artifacts into image
+RUN mkdir -p firebird
+COPY pom.xml .test_env test.sh setup.py version.txt Makefile README.rst ./firebird/
+COPY firebird firebird/firebird
+COPY notebook firebird/notebook
+COPY test firebird/test
 COPY log4j.properties $SPARK_HOME/conf/log4j.properties
-RUN sudo chown -R lcmap:lcmap .
 
-RUN pip install -e .[test,dev]
-
-RUN cd $HOME && \
-    sudo yum install -y maven && \
+# Install spark-cassandra-connector
+RUN cd $HOME/firebird && \
+    sudo yum install -y maven  && \
     sudo mvn dependency:copy-dependencies -DoutputDirectory=$SPARK_HOME/jars && \
     sudo yum erase -y maven && \
     sudo yum clean all && \
-    conda clean -all
+    sudo rm -rf /var/cache/yum && \
+    sudo rm -rf /root/.cache /root/.m2
+
+# Do not install the test or dev profiles in this image, control image size
+#RUN sudo /usr/local/bin/pip install -e .[test,dev]
+RUN sudo chown -R lcmap:lcmap . && \
+    sudo yum -y install gcc Cython && \
+    sudo $HOME/miniconda3/bin/pip install -e firebird/.[test,dev] && \
+    sudo sh -c 'find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf' && \
+    sudo yum erase -y gcc && \
+    sudo yum clean all && \
+    sudo rm -rf /var/cache/yum

Makefile

@@ -1,20 +1,21 @@
 # pull the tag from version.txt
 TAG:=`cat version.txt`
-WORKERIMAGE:=usgseros/lcmap-firebird:$(TAG)
+IMAGE:=usgseros/lcmap-firebird
 
 vertest:
 	@echo TAG:$(TAG)
-	@echo WORKERIMAGE:$(WORKERIMAGE)
+	@echo IMAGE:$(IMAGE)
 
 docker-build:
-	docker build -t $(WORKERIMAGE) $(PWD)
+	docker build -t $(IMAGE):$(TAG) -t $(IMAGE):latest $(PWD)
 
 docker-push:
 	docker login
-	docker push $(WORKERIMAGE)
+	docker push $(IMAGE):$(TAG)
+	docker push $(IMAGE):latest
 
 docker-shell:
-	docker run -it --entrypoint=/bin/bash usgseros/$(WORKERIMAGE)
+	docker run -it --entrypoint=/bin/bash $(IMAGE):latest
 
 deps-up:
 	docker-compose -f test/resources/docker-compose.yml up
@@ -34,7 +35,7 @@ spark-lib:
 	tar -C lib -xvf lib/spark-2.2.0-bin-hadoop2.7.tar
 	rm lib/*tar
 	ln -s spark-2.2.0-bin-hadoop2.7 lib/spark
-	mvn dependency:copy-dependencies -DoutputDirectory=lib/spark/jars
+	mvn dependency:copy-dependencies -f pom.xml -DoutputDirectory=lib/spark/jars
 
 tests:
 	./test.sh

README.md renamed to README.rst

@@ -1,78 +1,90 @@
-# lcmap-firebird
+lcmap-firebird
+==============
 LCMAP Science Execution Environment
 
-## What is lcmap-firebird?
+What is lcmap-firebird?
+-----------------------
 * LCMAP product generation, development and analysis
 * Built on Apache Spark, Apache Mesos, Docker and Python3
 * Runs on 2000 cores as easily as it runs on 1
 
-## As an operations environment
+As an operations environment
+----------------------------
 * Command line interface
 * System requirements: Bash & Docker
 
-## As a development and analysis environment
+As a development and analysis environment
+-----------------------------------------
 * Jupyter Notebooks & Apache Cassandra (included)
 * Notebooks uploaded or mounted via a volume 
 * Results are savable to Cassandra anywhere: Cloud, cluster, dev server, local, etc
 * Comes with the Firebird Spark Python library for working with chips, chip-specs and creating time-series data
 
-## Streamlined Research to Operations
+Streamlined Research to Operations
+----------------------------------
 * Development and analysis Notebooks are controlled outside Firebird.
-* Code may be moved to operations [by merging it directly into lcmap-firebird or included it via library.](#developing-firebird)
+* Code may be moved to operations `by merging it directly into lcmap-firebird or included it via library. <#developing-firebird>`_
 
 
-## Get Started
-```bash
+Get Started
+-----------
+.. code-block:: bash
+
    $ wget https://raw.githubusercontent.com/USGS-EROS/lcmap-firebird/master/firebird.install.example -O firebird.install
    $ emacs firebird.install
    $ source firebird.install
    $ firebird-save -a 1980-01-01/2017-01-01 -b -1821585,2891595 -p seglength -p ccd -d 2014-01-01 
-```
 
-## Read [Frequently Asked Questions](faq.md)
+`Frequently Asked Questions <docs/faq.rst>`_
+----------------------------------------------
+
+`Roadmap <docs/roadmap.rst>`_
+-----------------------------
 
-## Developing Firebird
+Developing Firebird
+-------------------
 
 * Install Docker, Maven and Conda
 
 * Create and activate a conda environment
-```bash
+.. code-block:: bash
+
    $ conda config --add channels conda-forge
    $ conda create --name firebird python=3.6 numpy pandas scipy gdal -y
    $ source activate firebird
-```
 
 * Clone this repo, install deps
-```bash
+.. code-block:: bash
+
    $ git clone [email protected]:usgs-eros/lcmap-firebird
    $ cd lcmap-firebird
    $ pip install -e .[test,dev]
-```
 
 * Run tests
-```bash
- $ make spark-lib
- $ make deps-up
- $ make db-schema
- $ make tests
- $ make deps-down
-```
+.. code-block:: bash
+
+   $ make spark-lib
+   $ make deps-up
+   $ make db-schema
+   $ make tests
+   $ make deps-down
 
 * Cut a branch, do some work, write some tests, update the docs, push to github
 
 * Build a Docker image to test locally
-```bash
-    $ emacs version.txt
-    $ make docker-build
-    $ emacs firebird.install # point to new version that was just built
-```
+.. code-block:: bash
+
+   $ emacs version.txt
+   $ make docker-build
+   $ emacs firebird.install # point to new version that was just built
 
 * Publish the Docker image so it will be available to a cluster
-```bash
-    $ make docker-push
-```
+.. code-block:: bash
+
+   $ make docker-push
 
-## Development Philosophy
+Development Philosophy
+----------------------
 Apache Spark is functional programming for cluster computing therefore
 Firebird strives to ensure all of it's code follows functional principles:
 data is immutable, functions are the primary unit of abstraction, and functional