Update jupyter (#482)

* updating jupyter version and stack dependencies

this enables many new ipython widgets and jupyter notebooks features

* force fetch on Makefile:default_stack repos

* added cycler package

* added backports-shutil dep

* added more ipython deps to stack

* update stack commit again

* fixing shapely in stack

* removed scorec from default stack

* added python-lzma

* fixing default_stack target

* updated Dockerfile to work with current jupyterhub

* updated hashdist and hashstack commits

* close hdf5 files after each test runs

* updated test output, fixed gauge MPI bug,  removed some warnings

* tried to fix int conversion problem in WaveTools.py

* added some system dependencies for travis builds

* fixed  spacing in .travis.yaml--got to get a yml emacs mode

* trying pip install of pyliblzma so hit can do .xz files

* turning down hit verbosity

* updated hashdist to turn off progress bar that fills up logs

* updated Dockerfile to allow multiple modes of running the container

supports terminal, ipython, local notebook, and jupyterhub

* updated Dockerfile with more OS packages and hashdist_default

* updated zmq in stack and how pyzmq is build in Dockerfile

* cleaned up Dockerfile and tried to fix pip3 pyzmq build

* more stack and hashdist fixes

* added new test data

* fixed nbextensions in jupyterhub config

* Close .h5 file in import module.

* Delete renamed output files.

* updated to petsc 3.7.5

* update version numbers and Dockerfile

* added PROTEUS_RELEASE environment variable in .travis.yml

* added better version extraction

Author: cekees

.hashdist_default

@@ -1,4 +1,4 @@
-a23cfaa98e8fff6c4e9fc70f09417841e9e398b6
+95601061658628ad9da3be1a5538681ed1092634
 
 
 

.hashstack_default

@@ -1,8 +1,4 @@
-ca8376dae614a5192d012bc0fd0124c596248723
-
-
-
-
+2ca88aa0ef486b86ad45191ea56c72ce641f8dda
 
 
 

.travis.yml

@@ -13,11 +13,15 @@ addons:
   apt:
     packages:
     - gfortran
+    - libffi-dev
+    - python-lzma
+    - libssl-dev
 
 before_install:
 - wget https://dl.dropboxusercontent.com/u/26353144/hashdist_trusty.tgz
 - tar xzf hashdist_trusty.tgz 
 - mv .hashdist $HOME
+- pip install pyliblzma
 
 install:
 - curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
@@ -32,7 +36,7 @@ install:
 - ls $SSL_CERT_DIR
 
 script:
-- py.test --boxed -v linux2/lib/python2.7/site-packages/proteus-1.1.0-py2.7-linux-x86_64.egg/proteus/tests
+- py.test --boxed -v linux2/lib/python2.7/site-packages/proteus-$(python -c "import proteus; print proteus.__version__")-py2.7-linux-x86_64.egg/proteus/tests
 
 notifications:
   email:

Dockerfile

@@ -4,9 +4,12 @@ MAINTAINER Proteus Project <[email protected]>
 
 USER root
 
-# Install all OS dependencies for fully functional notebook server
 ENV DEBIAN_FRONTEND noninteractive
-RUN apt-get update && apt-get install -yq --no-install-recommends --fix-missing \
+
+RUN REPO=http://cdn-fastly.deb.debian.org \
+    && echo "deb $REPO/debian jessie main\ndeb $REPO/debian-security jessie/updates main" > /etc/apt/sources.list \
+    && apt-get update && apt-get -yq dist-upgrade \
+    && apt-get install -yq --no-install-recommends --fix-missing \
     git \
     vim \
     jed \
@@ -30,7 +33,15 @@ RUN apt-get update && apt-get install -yq --no-install-recommends --fix-missing
     libav-tools \
     libmpich2-dev \
     liblapack-dev \
+    freeglut3 \
     freeglut3-dev \
+    libglew1.5 \
+    libglew1.5-dev \
+    libglu1-mesa \
+    libglu1-mesa-dev \
+    libgl1-mesa-glx \
+    libgl1-mesa-dev \
+    curl \
     libjpeg-dev \
     m4 \
     libssl-dev \
@@ -50,16 +61,21 @@ RUN apt-get update && apt-get install -yq --no-install-recommends --fix-missing
     binfmt-support \
     python3-dev \
     python3-wheel \
-    && apt-get clean
+    libffi-dev \
+    python-lzma \
+    python-pip \
+    cmake \
+    gfortran \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-RUN pip3 install notebook terminado
 
 RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
     locale-gen
 
 # Install Tini
-RUN wget --quiet https://github.com/krallin/tini/releases/download/v0.9.0/tini && \
-    echo "faafbfb5b079303691a939a747d7f60591f2143164093727e870b289a44d9872 *tini" | sha256sum -c - && \
+RUN wget --quiet https://github.com/krallin/tini/releases/download/v0.10.0/tini && \
+    echo "1361527f39190a7338a0b434bd8c88ff7233ce7b9a4876f3315c22fce7eca1b0 *tini" | sha256sum -c - && \
     mv tini /usr/local/bin/tini && \
     chmod +x /usr/local/bin/tini
 
@@ -72,34 +88,29 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US.UTF-8
 
 # Create jovyan user with UID=1000 and in the 'users' group
-RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \
-    chown -R $NB_USER:users /home/$NB_USER
+RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER
 
-USER jovyan
+RUN mkdir /home/$NB_USER/.jupyter && \
+    mkdir /home/$NB_USER/.local && \
+    mkdir /home/$NB_USER/.hashdist && \
+    echo "cacert=/etc/ssl/certs/ca-certificates.crt" > /home/$NB_USER/.curlrc
 
-RUN mkdir /home/$NB_USER/work && \
-    mkdir /home/$NB_USER/.jupyter && \
-    mkdir /home/$NB_USER/.local
+ADD https://dl.dropboxusercontent.com/u/26353144/hashdist_config_jovyan.yaml /home/$NB_USER/.hashdist/config.yaml
 
-#RUN wget https://dl.dropboxusercontent.com/u/26353144/hashdist_config_jovyan.yaml && \
-#    mkdir /home/$NB_USER/.hashdist && \
-#    mv hashdist_config_jovyan.yaml /home/$NB_USER/.hashdist/config.yaml && \
-#    cat /home/$NB_USER/.hashdist/config.yaml
+RUN chown -R $NB_USER:users /home/$NB_USER
 
-WORKDIR /home/$NB_USER
+USER jovyan
+
+RUN ls /home/$NB_USER/.hashdist && \
+    cat /home/$NB_USER/.hashdist/config.yaml
 
-RUN git clone https://github.com/erdc-cm/workshops -b erdc-fsi-tutorials
+WORKDIR /home/$NB_USER
 
-RUN git clone https://github.com/erdc-cm/proteus && \
+RUN cat /home/$NB_USER/.hashdist/config.yaml && \
+    git clone https://github.com/erdc-cm/proteus && \
     cd proteus && \
     make hashdist stack stack/default.yaml && \
     cd stack && \
-    git remote add erdc-cm http://github.com/erdc-cm/hashstack-private && \
-    git fetch --all && \
-    git checkout erdc-cm/cekees/kitchen_sink && \
-    /usr/bin/mpicc -show && \
-    which mpiexec && \
-    which mpirun && \
     ../hashdist/bin/hit build default.yaml -v
 
 ENV CC mpicc
@@ -112,54 +123,55 @@ RUN cd proteus && make develop
 ENV PATH /home/$NB_USER/proteus/linux2/bin:$PATH
 ENV LD_LIBRARY_PATH /home/$NB_USER/proteus/linux2/lib:$LD_LIBRARY_PATH
 
+RUN cd proteus && make jupyter
+
 USER root
 
-# Configure container startup as root
+RUN pip3 install pyzmq --install-option="--zmq=/home/$NB_USER/proteus/linux2"
+RUN pip3 install notebook jupyterhub jupyterlab terminado ipyparallel ipywidgets ipyleaflet jupyter_dashboards pythreejs cesiumpy
+RUN /usr/local/bin/jupyter serverextension enable --py jupyterlab --sys-prefix \
+    && /usr/local/bin/jupyter nbextension enable --py --sys-prefix widgetsnbextension \
+    && /usr/local/bin/jupyter nbextension enable --py --sys-prefix pythreejs \
+    && /usr/local/bin/jupyter nbextension enable --py --sys-prefix ipyleaflet \
+    && /usr/local/bin/jupyter dashboards quick-setup --sys-prefix
+
 EXPOSE 8888
-WORKDIR /home/$NB_USER/work
+WORKDIR /home/$NB_USER
+
 ENTRYPOINT ["tini", "--"]
 CMD ["start-notebook.sh"]
 
-RUN cd /usr/local/bin && \
-    wget https://raw.githubusercontent.com/jupyter/docker-stacks/master/minimal-notebook/start-notebook.sh
+# Add local files as late as possible to avoid cache busting
+ADD https://raw.githubusercontent.com/jupyter/docker-stacks/master/base-notebook/start.sh /usr/local/bin/start.sh
+ADD https://raw.githubusercontent.com/jupyter/docker-stacks/master/base-notebook/start-notebook.sh /usr/local/bin/start-notebook.sh
+ADD https://raw.githubusercontent.com/jupyter/docker-stacks/master/base-notebook/start-singleuser.sh /usr/local/bin/start-singleuser.sh
+ADD https://raw.githubusercontent.com/jupyter/docker-stacks/master/base-notebook/jupyter_notebook_config.py /home/$NB_USER/.jupyter/jupyter_notebook_config.py
 
-ADD https://raw.githubusercontent.com/jupyter/docker-stacks/master/minimal-notebook/jupyter_notebook_config.py /home/$NB_USER/.jupyter/jupyter_notebook_config.py
+RUN chmod a+rx /usr/local/bin/*
 
-RUN mkdir /etc/jupyter && \
-    chmod a+rwX /etc/jupyter && \
-    chown -R $NB_USER:users /home/$NB_USER
+RUN chown -R $NB_USER:users /home/$NB_USER/.jupyter
 
-#jupyter/ipython extensions
-RUN pip3 install \
-    ipyparallel \
-    ipywidgets
+RUN jupyter kernelspec install-self
 
 # Switch back to jovyan to avoid accidental container runs as root
-USER jovyan
-    
+USER $NB_USER
+
 RUN cd ~/.jupyter && \
     ipython profile create mpi --parallel && \
-    ipcluster nbextension enable && \
+    ipcluster nbextension enable --user && \
     echo '\nc.NotebookApp.server_extensions.append("ipyparallel.nbextension")' >> /home/$NB_USER/.jupyter/jupyter_notebook_config.py && \
-    cp jupyter_notebook_config.py /etc/jupyter/ && \
     echo "c.LocalControllerLauncher.controller_cmd = ['python2', '-m', 'ipyparallel.controller']\nc.LocalEngineSetLauncher.engine_cmd = ['python2', '-m', 'ipyparallel.engine']\n" \
-          >> /home/$NB_USER/.ipython/profile_mpi/ipcluster_config.py  
-
-USER root
-
-RUN jupyter kernelspec install-self
-
-# fetch juptyerhub-singleuser entrypoint
-RUN wget -q https://raw.githubusercontent.com/jupyter/jupyterhub/master/scripts/jupyterhub-singleuser -O /usr/local/bin/jupyterhub-singleuser && \
-    chmod 755 /usr/local/bin/jupyterhub-singleuser
-
-ADD https://raw.githubusercontent.com/jupyter/dockerspawner/master/singleuser/singleuser.sh /srv/singleuser/singleuser.sh
-
-RUN chmod 755 /srv/singleuser/singleuser.sh
-
-USER jovyan
-
-RUN cat /srv/singleuser/singleuser.sh
-# smoke test that it's importable at least
-RUN sh /srv/singleuser/singleuser.sh -h
-CMD ["sh", "/srv/singleuser/singleuser.sh"]
+          >> /home/$NB_USER/.ipython/profile_mpi/ipcluster_config.py \
+    && jupyter serverextension enable --py jupyterlab --user \
+    && jupyter nbextension enable --py --user widgetsnbextension\
+    && jupyter nbextension install --py --user mayavi \
+    && jupyter nbextension enable --py --user bqplot \
+    && jupyter nbextension enable --py --user pythreejs \
+    && jupyter nbextension enable --py --user ipyleaflet \
+    && jupyter nbextension install --py --user rise \
+    && jupyter nbextension enable --py --user rise \
+    && jupyter dashboards quick-setup --user
+
+# Import matplotlib the first time to build the font cache.
+ENV XDG_CACHE_HOME /home/$NB_USER/.cache/
+RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot"

Makefile

@@ -108,10 +108,13 @@ update:
 	@echo "Type: git checkout -b branch_name to save changes" 
 	@echo "+======================================================================================================+"
 
-default_stack: stack
+default_stack: stack hashdist
 	cd stack; git fetch origin; git checkout -q ${HASHSTACK_DEFAULT_VERSION}
 	@echo "Stack repository updated to .hashstack_default"
 	HASHSTACK_VERSION=${HASHSTACK_DEFAULT_VERSION}
+	@echo "hashdist repository updated to .hashdist_default"
+	cd hashdist; git fetch origin; git checkout -q ${HASHDIST_DEFAULT_VERSION}
+	HASHDIST_VERSION=${HASHDIST_DEFAULT_VERSION}
 
 hashdist: 
 	@echo "No hashdist found.  Cloning hashdist from GitHub"
@@ -155,7 +158,7 @@ ${PROTEUS_PREFIX}/artifact.json: stack/default.yaml stack hashdist $(shell find
 
 	$(call show_info)
 
-	cd stack && ${PROTEUS}/hashdist/bin/hit develop ${HIT_FLAGS} -v -f -k error default.yaml ${PROTEUS_PREFIX}
+	cd stack && ${PROTEUS}/hashdist/bin/hit develop ${HIT_FLAGS} -f -k error default.yaml ${PROTEUS_PREFIX}
 
 	@echo "************************"
 	@echo "Dependency build complete"
@@ -279,3 +282,18 @@ test:
 	source ${PROTEUS_PREFIX}/bin/proteus_env.sh; py.test --boxed -v proteus/tests --ignore proteus/tests/POD
 	@echo "Tests complete "
 	@echo "************************************"
+
+jupyter:
+	@echo "************************************"
+	@echo "Enabling jupyter notebook/lab/widgets"
+	source ${PROTEUS_PREFIX}/bin/proteus_env.sh
+	pip install jupyter jupyterlab  ipywidgets ipyleaflet jupyter_dashboards pythreejs RISE cesiumpy bqplot mayavi
+	jupyter serverextension enable --py jupyterlab --sys-prefix
+	jupyter nbextension enable --py --sys-prefix widgetsnbextension
+	jupyter nbextension install --py --sys-prefix mayavi
+	jupyter nbextension enable --py --sys-prefix bqplot
+	jupyter nbextension enable --py --sys-prefix pythreejs
+	jupyter nbextension enable --py --sys-prefix ipyleaflet
+	jupyter nbextension install --py --sys-prefix rise
+	jupyter nbextension enable --py --sys-prefix rise
+	jupyter dashboards quick-setup --sys-prefix

doc/source/_templates/layout.html

@@ -104,12 +104,12 @@
      style="height:1in;"> A Python toolkit for computational methods and simulation</p>
     <div class="btn-group" role="group">
       <a class="btn btn-success"
-          href="https://github.com/erdc-cm/proteus/archive/v0.9.0.tar.gz">
+          href="https://github.com/erdc-cm/proteus/archive/v1.2.0.tar.gz">
             <span class="glyphicon glyphicon-download"></span> Download
       </a>
       <button type="button" class="btn btn-success dropdown-toggle" data-toggle="dropdown"><span class="caret"></span></button>
       <ul class="dropdown-menu">
-        <li><a https://github.com/erdc-cm/proteus/archive/v0.9.0.tar.gz">Version 0.9.0</a></li>
+        <li><a https://github.com/erdc-cm/proteus/archive/v1.2.0.tar.gz">Version 1.2.0</a></li>
       </ul>
     </div>
   </div>

doc/source/conf.py

@@ -238,7 +238,7 @@ def linkcode_resolve(domain, info):
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-html_favicon = "tritonBorder.ico"
+html_favicon = "_static/tritonBorder.ico"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -327,7 +327,7 @@ def linkcode_resolve(domain, info):
 #latex_use_parts = False
 
 # Additional stuff for the LaTeX preamble.
-latex_preamble = '\input{macros}'
+latex_elements['preamble'] = '\input{macros}'
 
 # If true, show page references after internal links.
 #latex_show_pagerefs = False

proteus/Gauges.py

@@ -221,7 +221,8 @@ def findNearestNode(self, femSpace, location):
 
         # determine global nearest node
         haveElement = int(local_element is not None)
-        global_have_element, owning_proc = comm.allreduce(haveElement, op=MPI.MAXLOC)
+        global_have_element, owning_proc = comm.allreduce((haveElement, comm.rank),
+                                                          op=MPI.MAXLOC)
         if global_have_element:
             logEvent("Gauges on element at location: [%g %g %g] assigned to %d" % (location[0], location[1], location[2],
                                                                     owning_proc), 3)

proteus/LinearSolvers.py

@@ -99,8 +99,8 @@ def solveInitialize(self,u,r,b,initialGuessIsZero=True):
             r=self.r
         else:
             self.r=r
-        if b == None:
-            if self.b == None:
+        if b is None:
+            if self.b is None:
                 self.b = Vec(self.n)
             b=self.b
         else:

proteus/WaveTools.py

@@ -2535,7 +2535,7 @@ def writeEtaSeries(self,Tstart,Tend,dt,x0,fname, mode="all",setUp=False,Vgen=np.
         Tlag = max(Tlag)
         Tstart = Tstart - Tlag
 
-        Nseries = int(Tend - Tstart)/dt + 1
+        Nseries = int((Tend - Tstart)/dt) + 1
         timelst=np.linspace(Tstart, Tend, Nseries)
         series = np.zeros((Nseries,2),)
         series[:,0] = timelst