Merge pull request #97 from PrincetonUniversity/code-along

Code along Known issues - 1. memory misallocation with intel compilers 2. composite boundaries implementation doesn't work with CUDA compiler
PrincetonUniversity · Jan 3, 2024 · 35d2f06 · 35d2f06
2 parents 4bdfac1 + 6520a9e
commit 35d2f06
Show file tree

Hide file tree

Showing 511 changed files with 46,742 additions and 3,700 deletions.
diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml
@@ -14,7 +14,7 @@ jobs:
         run: git submodule init
           &&  git submodule update
       - name: Configure
-        run: cmake -S . -B build
+        run: cmake -S . -B build -DBUILD_TESTS=ON
       - name: Build
         run: cmake --build build
       - name: Run all tests

diff --git a/.gitignore b/.gitignore
@@ -15,3 +15,12 @@ examples/*/*_config.yaml
 tests/regression-tests/*.yaml
 tests/regression-tests/*/cpu/*.yaml
 tests/regression-tests/*/gpu/*.yaml
+*.mod
+*.o
+*/OUTPUT_FILES/*
+*.gnu
+examples/*/Par_File
+*OUTPUT_FILES*
+*.sqlite
+*.nsys-rep
+*.ncu-rep
diff --git a/.jenkins/compiler_checks.gvy b/.jenkins/compiler_checks.gvy
@@ -51,18 +51,19 @@ pipeline{
                                 steps {
                                     echo "Building ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}"
                                     sh """
+                                        module load boost/1.73.0
                                         module load ${CUDA_MODULE}
-                                        cmake3 -S . -B build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}
-                                        cmake3 --build build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}
+                                        cmake3 -S . -B build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}
+                                        cmake3 --build build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}
                                     """
                                     echo ' Build completed '
                                 }
                             }
-                            stage (' Clean '){
-                                steps {
-                                    echo ' Cleaning '
-                                    sh "rm -rf build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}"
-                                }
+                        }
+                        post {
+                            always {
+                                echo ' Cleaning '
+                                sh "rm -rf build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}"
                             }
                         }
                     }
@@ -107,20 +108,21 @@ pipeline{
                                 steps {
                                     echo "Building ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}"
                                     sh """
+                                        module load boost/1.73.0
                                         module load intel/2022.2.0
                                         export CC=icx
                                         export CXX=icpx
-                                        cmake3 -S . -B build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}
-                                        cmake3 --build build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}
+                                        cmake3 -S . -B build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}
+                                        cmake3 --build build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}
                                     """
                                     echo ' Build completed '
                                 }
                             }
-                            stage (' Clean '){
-                                steps {
-                                    echo ' Cleaning '
-                                    sh "rm -rf build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}"
-                                }
+                        }
+                        post {
+                            always {
+                                echo ' Cleaning '
+                                sh "rm -rf build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}"
                             }
                         }
                     }

diff --git a/.jenkins/regression_tests.gvy b/.jenkins/regression_tests.gvy
@@ -13,7 +13,7 @@ pipeline {
                 // Screen is needed since the sessions need to remain active even when this stage exits
                 sh """
                     screen -dm salloc -J jenkins_cpu -N 1 -n 1 -t 00:30:00 --constraint=broadwell
-                    screen -dm salloc -J jenkins_gpu -N 1 -c 10 -t 00:30:00 --gres=gpu:1 --constraint=a100 &
+                    screen -dm salloc -J jenkins_gpu -N 1 -c 10 -t 00:30:00 --gres=gpu:1 --constraint=a100
                 """
             }
         }
@@ -145,9 +145,9 @@ pipeline {
                 stage (' Checkout main branch '){
                     steps {
                         checkout([$class: 'GitSCM',
-                                branches: [[name: 'devel']],
+                                branches: [[name: 'code-along']],
                                 extensions: [lfs()],
-                                userRemoteConfigs: [[url: 'https://github.com/PrincetonUniversity/specfem2d_kokkos']]])
+                                userRemoteConfigs: [[url: 'https://github.com/PrincetonUniversity/specfempp']]])
                     }
                 }
 

diff --git a/.jenkins/unittests.gvy b/.jenkins/unittests.gvy
@@ -61,9 +61,10 @@ pipeline {
                                 steps {
                                     echo "Building ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}"
                                     sh """
+                                        module load boost/1.73.0
                                         module load ${CUDA_MODULE}
-                                        cmake3 -S . -B build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}
-                                        cmake3 --build build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}
+                                        cmake3 -S . -B build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS} -DBUILD_TESTS=ON
+                                        cmake3 --build build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}
                                     """
                                     echo ' Build completed '
                                 }
@@ -72,16 +73,17 @@ pipeline {
                                 steps {
                                     echo " Running Unittests "
                                     sh """
-                                        cd build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}/tests/unit-tests
-                                        srun -N 1 -t 00:10:00 ${HOST_RUN_FLAGS} ${DEVICE_RUN_FLAGS} bash -c 'export OMP_PROC_BIND=spread; export OMP_THREADS=places; ctest'
+                                        module load boost/1.73.0
+                                        cd build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}/tests/unit-tests
+                                        srun -N 1 -t 00:10:00 ${HOST_RUN_FLAGS} ${DEVICE_RUN_FLAGS} bash -c 'export OMP_PROC_BIND=spread; export OMP_THREADS=places; ctest; ctest --rerun-failed --output-on-failure;'
                                     """
                                 }
                             }
-                            stage (' Clean '){
-                                steps {
-                                    echo ' Cleaning '
-                                    sh "rm -rf build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}"
-                                }
+                        }
+                        post {
+                            always {
+                                echo ' Cleaning '
+                                sh "rm -rf build_GNU_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}"
                             }
                         }
                     }
@@ -138,8 +140,8 @@ pipeline {
                                         module load intel/2022.2.0
                                         export CC=icx
                                         export CXX=icpx
-                                        cmake3 -S . -B build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS}
-                                        cmake3 --build build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}
+                                        cmake3 -S . -B build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT} -DCMAKE_BUILD_TYPE=Release ${CMAKE_HOST_FLAGS} ${CMAKE_DEVICE_FLAGS} -DBUILD_TESTS=ON
+                                        cmake3 --build build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}
                                     """
                                     echo ' Build completed '
                                 }
@@ -148,17 +150,18 @@ pipeline {
                                 steps {
                                     echo " Running Unittests "
                                     sh """
+                                        module load boost/1.73.0
                                         module load intel/2022.2.0
-                                        cd build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}/tests/unit-tests
-                                        srun -N 1 -t 00:10:00 ${HOST_RUN_FLAGS} ${DEVICE_RUN_FLAGS} bash -c 'export OMP_PROC_BIND=spread; export OMP_THREADS=places; ctest'
+                                        cd build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}/tests/unit-tests
+                                        srun -N 1 -t 00:10:00 ${HOST_RUN_FLAGS} ${DEVICE_RUN_FLAGS} bash -c 'export OMP_PROC_BIND=spread; export OMP_THREADS=places; ctest; ctest --rerun-failed --output-on-failure;'
                                     """
                                 }
                             }
-                            stage (' Clean '){
-                                steps {
-                                    echo ' Cleaning '
-                                    sh "rm -rf build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}"
-                                }
+                        }
+                        post {
+                            always {
+                                echo ' Cleaning '
+                                sh "rm -rf build_INTEL_${CMAKE_HOST_NAME}_${CMAKE_DEVICE_NAME}_${env.GIT_COMMIT}"
                             }
                         }
                     }

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -4,6 +4,8 @@ project(specfem2d_kokkos VERSION 0.1.0)
 
 set(CMAKE_CXX_STANDARD 17)
 option(MPI_PARALLEL "MPI enabled" OFF)
+option(BUILD_TESTS "Tests included" OFF)
+option(BUILD_EXAMPLES "Examples included" OFF)
 
 if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
         set(CMAKE_CXX_FLAGS "-fp-model=precise")
@@ -25,25 +27,37 @@ endif()
 ## TODO: Add options for on utilizing in house builds
 include(FetchContent)
 FetchContent_Declare(
-  kokkos
-  URL https://github.com/kokkos/kokkos/archive/refs/tags/4.0.00.zip
+kokkos
+URL https://github.com/kokkos/kokkos/archive/refs/tags/4.0.00.zip
 )
 FetchContent_MakeAvailable(kokkos)
 
 FetchContent_Declare(
         yaml
-	URL https://github.com/jbeder/yaml-cpp/archive/refs/tags/yaml-cpp-0.7.0.zip
+        URL https://github.com/jbeder/yaml-cpp/archive/refs/tags/0.8.0.tar.gz
 )
 FetchContent_MakeAvailable(yaml)
-include_directories(BEFORE SYSTEM ${yaml_SOURCE_DIR} ${yaml_BINARY_DIR}/include)
 
-add_subdirectory(boost-cmake)
+include_directories(BEFORE SYSTEM ${yaml_BINARY_DIR} ${yaml_SOURCE_DIR}/include)
+
+# Try finding boost and if not found install.
+find_package(Boost 1.73.0 COMPONENTS program_options filesystem system)
+
+if (NOT ${Boost_FOUND})
+        add_subdirectory(boost-cmake)
+else ()
+        message(STATUS "    LIB:   ${Boost_LIBRARY_DIRS}")
+        message(STATUS "    INC:   ${Boost_INCLUDE_DIRS}")
+        message(STATUS "    LIBSO: ${Boost_LIBRARIES}")
+endif()
 
 configure_file(constants.hpp.in constants.hpp)
 
 include_directories(include)
 include_directories(${CMAKE_BINARY_DIR})
 
+add_subdirectory(meshfem2d)
+
 # Build specfem2d libraries
 add_library(
         quadrature
@@ -85,16 +99,6 @@ else()
         message("-- Compiling SPECFEM without MPI")
 endif(MPI_PARALLEL)
 
-add_library(
-        operators
-        src/mathematical_operators/mathematical_operators.cpp
-)
-
-target_link_libraries(
-        operators
-        Kokkos::kokkos
-)
-
 add_library(
         material_class
         src/material/elastic_material.cpp
@@ -113,12 +117,15 @@ add_library(
         src/mesh/IO/fortran/read_material_properties.cpp
         src/mesh/boundaries/forcing_boundaries.cpp
         src/mesh/boundaries/absorbing_boundaries.cpp
+        src/mesh/boundaries/acoustic_free_surface.cpp
         src/mesh/elements/tangential_elements.cpp
         src/mesh/elements/axial_elements.cpp
         src/mesh/properties/properties.cpp
         src/mesh/mpi_interfaces/mpi_interfaces.cpp
         src/mesh/material_indic/material_indic.cpp
-        src/mesh/surfaces/acoustic_free_surface.cpp
+        src/mesh/coupled_interfaces/elastic_acoustic.cpp
+        src/mesh/coupled_interfaces/elastic_poroelastic.cpp
+        src/mesh/coupled_interfaces/acoustic_poroelastic.cpp
         src/mesh/mesh.cpp
 )
 
@@ -157,6 +164,7 @@ target_link_libraries(
 add_library(
         source_time_function
         src/source_time_function/dirac.cpp
+        src/source_time_function/ricker.cpp
 )
 
 target_link_libraries(
@@ -206,6 +214,8 @@ add_library(
         src/compute/compute_properties.cpp
         src/compute/compute_sources.cpp
         src/compute/compute_receivers.cpp
+        src/compute/coupled_interfaces.cpp
+        src/compute/compute_boundaries.cpp
 )
 
 target_link_libraries(
@@ -218,22 +228,6 @@ target_link_libraries(
         Kokkos::kokkos
 )
 
-# there seems to be a bug when compiling compute with -O3 mode using the icpx compiler.
-# set_target_properties(compute PROPERTIES COMPILE_OPTIONS "$<$<CONFIG:RELEASE>:-fp-model=precise>")
-
-add_library(
-        domain
-        src/domain/elastic_domain.cpp
-)
-
-target_link_libraries(
-        domain
-        compute
-        quadrature
-        operators
-        Kokkos::kokkos
-)
-
 add_library(
         timescheme
         src/timescheme/timescheme.cpp
@@ -242,7 +236,9 @@ add_library(
 
 target_link_libraries(
         timescheme
-        domain
+        Kokkos::kokkos
+        yaml-cpp
+        compute
 )
 
 add_library(
@@ -256,18 +252,6 @@ target_link_libraries(
         receiver_class
 )
 
-add_library(
-        solver
-        src/solver/time_marching.cpp
-)
-
-target_link_libraries(
-        solver
-        domain
-        timescheme
-        writer
-)
-
 add_library(
         parameter_reader
         src/parameter_parser/run_setup.cpp
@@ -288,6 +272,7 @@ target_link_libraries(
         timescheme
         receiver_class
         yaml-cpp
+        writer
         Boost::filesystem
 )
 
@@ -306,17 +291,22 @@ target_link_libraries(
         compute
         source_class
         parameter_reader
-        domain
-        solver
         receiver_class
         writer
         Boost::program_options
 )
 
 # Include tests
-add_subdirectory(tests/unit-tests)
-add_subdirectory(tests/regression-tests)
-add_subdirectory(examples)
+if (BUILD_TESTS)
+        message("-- Including tests.")
+        add_subdirectory(tests/unit-tests)
+        add_subdirectory(tests/regression-tests)
+endif()
+
+if (BUILD_EXAMPLES)
+        message("-- Including examples.")
+        add_subdirectory(examples)
+endif()
 
 # Doxygen