Merge branch 'main' into dependabot/pip/tests/local/utils/pillow-10.3.0

.github/workflows/test-pr.yml

@@ -21,10 +21,22 @@ jobs:
         configuration: [nwm_ana, nwm_long_range, gridded, reach]
     runs-on: ubuntu-latest
 
+    env:
+      MPI_HOME: /usr/share/miniconda
+      NETCDF: /usr/share/miniconda
+      NETCDF_INCLUDES: /usr/share/miniconda/include
+      NETCDF_LIBRARIES: /usr/share/miniconda/lib
+
+
     steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
       - name: Checkout candidate (pull request / push)
         if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: candidate
 
@@ -36,14 +48,14 @@ jobs:
 
       - name: Checkout reference (pull request)
         if: ${{ github.event_name == 'pull_request' }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.base.ref }}
           path: reference
 
       - name: Checkout reference (push)
         if: ${{ github.event_name == 'push' }}
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.before }}
           path: reference
@@ -54,29 +66,148 @@ jobs:
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
         run: gh repo clone ${{ github.repository }} reference && cd reference && git checkout origin/$(gh pr view ${{ github.event.inputs.pr }} --json baseRefName --jq '.baseRefName')
 
-      - name: Run testing container
+      - name: Install dependencies with apt-get
         run: |
-          docker run -e TRAVIS=1 -t --name test_container \
-           -v $GITHUB_WORKSPACE/candidate:/home/docker/candidate \
-           -v $GITHUB_WORKSPACE/reference:/home/docker/reference \
-           wrfhydro/dev:modeltesting --config ${{ matrix.configuration }} --domain_tag dev
+          sudo apt-get update \
+            && sudo apt-get install -yq --no-install-recommends \
+            wget \
+            curl \
+            bzip2 \
+            ca-certificates \
+            libhdf5-dev \
+            gfortran \
+            g++ \
+            m4 \
+            make \
+            libswitch-perl \
+            git \
+            bc \
+            openmpi-bin openmpi-common libopenmpi-dev \
+            libxml2-dev \
+            libnetcdf-dev \
+            libnetcdff-dev
 
-      - name: Copy test results from container
+      - name: Install dependencies with pip
+        run: |
+          python3 -m pip install matplotlib numpy xarray dask netCDF4 pygithub
+
+      - name: Compile reference
+        run: |
+          cd $GITHUB_WORKSPACE/reference
+          cmake -B build
+          make -C build -j
+
+      - name: Compile candidate
+        run: |
+          cd $GITHUB_WORKSPACE/candidate
+          cmake -B build
+          make -C build -j
+
+      - name: Run reference model
+        run: |
+          cd $GITHUB_WORKSPACE/reference/build/Run
+          make run-croton-${{ matrix.configuration }}
+
+      - name: Run candidate model
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          make run-croton-${{ matrix.configuration }}
+
+      - name: Compare HYDRO_RST.* output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          for file in output_${{ matrix.configuration }}/HYDRO_RST.*; do\
+            python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+          done
+
+      - name: Compare RESTART.* output with xrcmp
         if: ${{ always() }}
-        run: docker cp test_container:/home/docker/test_out $GITHUB_WORKSPACE/test_report
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          for file in output_${{ matrix.configuration }}/RESTART.*; do\
+            python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+          done
+
+      - name: Compare last *.CHANOBS_DOMAIN1 output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          file=$(ls -t output_${{ matrix.configuration }}/*.CHANOBS_DOMAIN1 | head -n 1)
+          python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+
+      - name: Compare last *.CHRTOUT_DOMAIN1 output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          file=$(ls -t output_${{ matrix.configuration }}/*.CHRTOUT_DOMAIN1 | head -n 1)
+          python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+
+      - name: Compare last *.LSMOUT_DOMAIN1 output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          file=$(ls -t output_${{ matrix.configuration }}/*.LSMOUT_DOMAIN1 | head -n 1)
+          python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
 
-      - name: Checkout Github API
+      - name: Compare last *.RTOUT_DOMAIN1 output with xrcmp
         if: ${{ always() }}
         run: |
-          pip install pygithub
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          file=$(ls -t output_${{ matrix.configuration }}/*.RTOUT_DOMAIN1 | head -n 1)
+          python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+
+      - name: Compare output with compare_output
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          mkdir output_diff
+          python -c \
+          "import sys; \
+          sys.path.append('${GITHUB_WORKSPACE}/candidate/tests/utils'); \
+          import compare_output; \
+          from pathlib import Path; \
+          compare_output.plot_diffs('${GITHUB_WORKSPACE}/candidate/build/Run/output_diff', \
+            '${GITHUB_WORKSPACE}/candidate/build/Run/output_${{ matrix.configuration }}/', \
+            '${GITHUB_WORKSPACE}/reference/build/Run/output_${{ matrix.configuration }}/', \
+            '${{ matrix.configuration }}')"
+
+      - name: Copy test results from container
+        if: ${{ always() }}
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/test_report
+          cp -r $GITHUB_WORKSPACE/candidate/build/Run/output_${{ matrix.configuration }}/* $GITHUB_WORKSPACE/test_report/
 
       - name: Attach diff plots to PR
         if: ${{ always() }}
         run: |
           cd $GITHUB_WORKSPACE/candidate/tests/local/utils
           bash attach_all_plots.bash $(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") ${{ matrix.configuration }}
 
-
       - name: Archive test results to GitHub
         if: ${{ always() }}
         uses: actions/upload-artifact@v3

tests/ctests/CMakeLists.txt

@@ -1,24 +1,35 @@
-# add Makefile target to setup test runs
-# - croton test options: {Gridded, Gridded_no_lakes, NWM, Reach, ReachLakes}
+# add Makefile targets to setup test runs
+# Croton Test Options: {Gridded, Gridded_no_lakes, NWM, Reach, ReachLakes}
+set(croton_testcases "gridded" "gridded_no_lakes" "nwm" "nwm_ana"
+  "nwm_long_range" "reach" "reach_lakes")
+
+# set default croton setup and run target
+set(croton_default_testcase "gridded")
 add_custom_target(croton
   COMMAND bash
-  ${CMAKE_CURRENT_SOURCE_DIR}/setup_cmake_testcase.sh Gridded ${CMAKE_BINARY_DIR})
-add_custom_target(croton-gridded
-  COMMAND bash
-  ${CMAKE_CURRENT_SOURCE_DIR}/setup_cmake_testcase.sh Gridded ${CMAKE_BINARY_DIR})
-add_custom_target(croton-gridded-no-lakes
-  COMMAND bash
-  ${CMAKE_CURRENT_SOURCE_DIR}/setup_cmake_testcase.sh Gridded_no_lakes ${CMAKE_BINARY_DIR})
-add_custom_target(croton-nwm
-  COMMAND bash
-  ${CMAKE_CURRENT_SOURCE_DIR}/setup_cmake_testcase.sh NWM ${CMAKE_BINARY_DIR})
-add_custom_target(croton-reach
+  ${CMAKE_CURRENT_SOURCE_DIR}/setup_cmake_testcase.sh ${croton_default_testcase} ${CMAKE_BINARY_DIR})
+add_custom_target(run-croton
   COMMAND bash
-  ${CMAKE_CURRENT_SOURCE_DIR}/setup_cmake_testcase.sh Reach ${CMAKE_BINARY_DIR})
-add_custom_target(croton-reach-lakes
+  ${CMAKE_CURRENT_SOURCE_DIR}/run_cmake_testcase.sh ${croton_default_testcase} ${CMAKE_BINARY_DIR} 1)
+add_custom_target(run-croton-parallel
   COMMAND bash
-  ${CMAKE_CURRENT_SOURCE_DIR}/setup_cmake_testcase.sh ReachLakes ${CMAKE_BINARY_DIR})
+  ${CMAKE_CURRENT_SOURCE_DIR}/run_cmake_testcase.sh ${croton_default_testcase} ${CMAKE_BINARY_DIR} 2)
 
+# generate commands for every croton testcase option
+foreach(croton_testcase ${croton_testcases})
+  string(TOLOWER ${croton_testcase} testcase)
+  add_custom_target(croton-${testcase}
+    COMMAND bash
+    ${CMAKE_CURRENT_SOURCE_DIR}/setup_cmake_testcase.sh ${croton_testcase} ${CMAKE_BINARY_DIR})
+  add_custom_target(run-croton-${testcase}
+    DEPENDS croton-${testcase}
+    COMMAND bash
+    ${CMAKE_CURRENT_SOURCE_DIR}/run_cmake_testcase.sh ${croton_testcase} ${CMAKE_BINARY_DIR} 1)
+  add_custom_target(run-croton-${testcase}-parallel
+    DEPENDS croton-${testcase}
+    COMMAND bash
+    ${CMAKE_CURRENT_SOURCE_DIR}/run_cmake_testcase.sh ${croton_testcase} ${CMAKE_BINARY_DIR} 2)
+endforeach()
 
 # generic ctests
 add_executable(fortran_ctest_should_fail

tests/ctests/run_cmake_testcase.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Bash script is meant to be used by CMake. It runs the Croton testcases after
+# setup_cmake_testcase.sh uses them
+
+# setup directory variables in script
+testcase_type=${1}
+binary_dir=${2}
+np=${3}
+run_dir=${binary_dir}/Run
+output_dir=output_${testcase_type}
+
+if [ "$np" -eq "1" ]; then
+    run_cmd="./wrf_hydro.exe"
+else
+    run_cmd="mpiexec -np ${np} ./wrf_hydro.exe"
+fi
+
+# run testcase
+cd $run_dir
+${run_cmd}
+
+# collect output, and fail silently if files not outputted at this point
+mkdir -p ${output_dir}
+mv *.CHANOBS_DOMAIN1 ${output_dir}/ 2>/dev/null
+mv *.CHRTOUT_DOMAIN1 ${output_dir}/ 2>/dev/null
+mv *.CHRTOUT_GRID1 ${output_dir}/ 2>/dev/null
+mv *.GWOUT_DOMAIN1 ${output_dir}/ 2>/dev/null
+mv *.LAKEOUT_DOMAIN1 ${output_dir}/ 2>/dev/null
+mv *.LDASOUT_DOMAIN1 ${output_dir}/ 2>/dev/null
+mv *.LSMOUT_DOMAIN1 ${output_dir}/ 2>/dev/null
+mv *.RTOUT_DOMAIN1 ${output_dir}/ 2>/dev/null
+mv HYDRO_RST.*_DOMAIN1 ${output_dir}/ 2>/dev/null
+mv RESTART.*_DOMAIN1 ${output_dir}/ 2>/dev/null
+mv diag_hydro.* ${output_dir}/ 2>/dev/null

tests/ctests/run_dir_makefile.mk

@@ -10,20 +10,56 @@ run:
 # download and extract Croton, NY test case to Run directory
 # testcase options: {Gridded, Gridded_no_lakes, NWM, Reach, ReachLakes}
 croton:
-	make -C .. croton
+	make -C .. $@
 croton-gridded:
-	make -C .. croton-gridded
+	make -C .. $@
 croton-gridded-no-lakes:
-	make -C .. croton-gridded-no-lakes
+	make -C .. $@
 croton-nwm:
-	make -C .. croton-nwm
+	make -C .. $@
+croton-nwm_ana:
+	make -C .. $@
+croton-nwm_long_range:
+	make -C .. $@
 croton-reach:
-	make -C .. croton-reach
+	make -C .. $@
 croton-reach-lakes:
-	make -C .. croton-reach-lakes
+	make -C .. $@
+# run croton targets
+run-croton:
+	make -C .. $@
+run-croton-gridded:
+	make -C .. $@
+run-croton-gridded-no-lakes:
+	make -C .. $@
+run-croton-nwm:
+	make -C .. $@
+run-croton-nwm_ana:
+	make -C .. $@
+run-croton-nwm_long_range:
+	make -C .. $@
+run-croton-reach:
+	make -C .. $@
+run-croton-reach-lakes:
+	make -C .. $@
+# run in parallel
+run-croton-gridded-parallel:
+	make -C .. $@
+run-croton-gridded-no-lakes-parallel:
+	make -C .. $@
+run-croton-nwm-parallel:
+	make -C .. $@
+run-croton-nwm_ana-parallel:
+	make -C .. $@
+run-croton-nwm_long_range-parallel:
+	make -C .. $@
+run-croton-reach-parallel:
+	make -C .. $@
+run-croton-reach-lakes-parallel:
+	make -C .. $@
 
 clean:
-	rm -f *.LDASOUT_DOMAIN1 *.LAKEOUT_DOMAIN1 *.CHRTOUT_DOMAIN1
-	rm -f *.GWOUT_DOMAIN1 *.RTOUT_DOMAIN1 *.CHANOBS_DOMAIN1
-	rm -f RESTART.*_DOMAIN1 HYDRO_RST.*_DOMAIN1
+	rm -f *.CHANOBS_DOMAIN1 *.CHRTOUT_DOMAIN1 *.GWOUT_DOMAIN1
+	rm -f *.LAKEOUT_DOMAIN1 *.LDASOUT_DOMAIN1 *.LSMOUT_DOMAIN1
+	rm -f *.RTOUT_DOMAIN1 RESTART.*_DOMAIN1 HYDRO_RST.*_DOMAIN1
 	rm -f diag_hydro.*