Add CMake test for GPU-aware MPI (#17)

Try to compile and run send_recv_usm.cpp to determine
whether the MPI implementation is GPU-aware. Enable
building the MPI_with_SYCL examples only if at least
one backend is found to work with MPI over GPU.

Author: rafbiels

CMakeLists.txt

@@ -2,7 +2,7 @@
 # This work is licensed under the Apache License, Version 2.0.
 # For a copy, see http://www.apache.org/licenses/LICENSE-2.0
 
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.12)
 project(SYCL-samples)
 
 # Set build type to Release if unset

README.md

@@ -85,8 +85,12 @@ MPI installation. E.g.
 ```
 
 Additionally, in order to run the examples, the MPI implementation needs
-to be device-aware. This is only detectable at runtime, so the examples may build
-fine but crash on execution if the linked MPI library isn't device-aware.
+to be device-aware. The CMake configuration attempts to build and execute the
+simplest example to evaluate whether the found MPI library supports any of the
+enabled backends. This demo will be automatically skipped if this check does not
+pass and a corresponding message will appear in the CMake configuration output.
+The result of this check can be overwritten with the `-DMPI_DEVICE_AWARE=ON/OFF`
+option.
 
 ### Parallel Inclusive Scan
 Implementation of a parallel inclusive scan with a given associative binary 

src/MPI_with_SYCL/CMakeLists.txt

@@ -1,11 +1,71 @@
+function(run_mpi_test)
+    cmake_parse_arguments(RUN_MPI_TEST "" "BINARY;BACKEND_SELECTOR" "" ${ARGN})
+    execute_process(
+        COMMAND bash -c "ONEAPI_DEVICE_SELECTOR=${RUN_MPI_TEST_BACKEND_SELECTOR}:gpu mpirun -n 2 ${RUN_MPI_TEST_BINARY} > mpi-test-output.txt && grep 'received status==0' mpi-test-output.txt"
+        RESULT_VARIABLE MPI_TEST_FAILED
+        OUTPUT_QUIET
+        ERROR_QUIET)
+    if (${MPI_TEST_FAILED})
+        set(MPI_TEST_SUCCEEDED FALSE PARENT_SCOPE)
+    else()
+        set(MPI_TEST_SUCCEEDED TRUE PARENT_SCOPE)
+    endif()
+endfunction()
+
+function(test_mpi_gpu_support)
+    set(MPI_TEST_BINARY ${CMAKE_CURRENT_BINARY_DIR}/test-mpi-gpu)
+    try_compile(MPI_TEST_COMPILE_SUCCEEDED
+        ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/send_recv_usm.cpp
+        COMPILE_DEFINITIONS "${SYCL_FLAGS} -I${MPI_CXX_INCLUDE_DIRS}"
+        LINK_OPTIONS ${SYCL_FLAGS}
+        LINK_LIBRARIES ${MPI_LIBRARIES}
+        COPY_FILE ${MPI_TEST_BINARY}
+        OUTPUT_VARIABLE MPI_TEST_COMPILE_OUTPUT)
+    set(MPI_DEVICE_AWARE 0)
+    if (${MPI_TEST_COMPILE_SUCCEEDED})
+        if (${ENABLE_CUDA})
+            run_mpi_test(BINARY ${MPI_TEST_BINARY} BACKEND_SELECTOR "cuda")
+            if (MPI_TEST_SUCCEEDED)
+                set(MPI_DEVICE_AWARE 1)
+                list(APPEND MPI_AVAILABLE_BACKENDS "CUDA")
+            endif()
+        endif()
+
+        if (${ENABLE_HIP})
+            run_mpi_test(BINARY ${MPI_TEST_BINARY} BACKEND_SELECTOR "hip")
+            if (MPI_TEST_SUCCEEDED)
+                set(MPI_DEVICE_AWARE 1)
+                list(APPEND MPI_AVAILABLE_BACKENDS "HIP")
+            endif()
+        endif()
+
+        if (${ENABLE_SPIR})
+            run_mpi_test(BINARY ${MPI_TEST_BINARY} BACKEND_SELECTOR "level_zero")
+            if (MPI_TEST_SUCCEEDED)
+                set(MPI_DEVICE_AWARE 1)
+                list(APPEND MPI_AVAILABLE_BACKENDS "Level Zero")
+            endif()
+        endif()
+    endif()
+
+    set(MPI_DEVICE_AWARE ${MPI_DEVICE_AWARE} CACHE BOOL "Offload device aware MPI implementation is available")
+    set(MPI_AVAILABLE_BACKENDS ${MPI_AVAILABLE_BACKENDS} PARENT_SCOPE)
+endfunction()
+
 find_package(MPI)
 if(NOT MPI_FOUND)
     message(STATUS "MPI not found, skipping the MPI_with_SYCL demo")
 else()
-    message(STATUS "Found MPI, configuring the MPI_with_SYCL demo")
-    foreach(TARGET send_recv_usm send_recv_buff scatter_reduce_gather)
-        add_executable(${TARGET} ${TARGET}.cpp)
-        target_compile_options(${TARGET} PUBLIC ${SYCL_FLAGS} ${MPI_INCLUDE_DIRS})
-        target_link_options(${TARGET} PUBLIC ${SYCL_FLAGS} ${MPI_LIBRARIES})
-    endforeach()
+    test_mpi_gpu_support()
+    if (MPI_DEVICE_AWARE)
+        list(JOIN MPI_AVAILABLE_BACKENDS ", " MPI_AVAILABLE_BACKENDS)
+        message(STATUS "Found offload device aware MPI, configuring the MPI_with_SYCL demo. Available backends: ${MPI_AVAILABLE_BACKENDS}")
+        foreach(TARGET send_recv_usm send_recv_buff scatter_reduce_gather)
+            add_executable(${TARGET} ${TARGET}.cpp)
+            target_compile_options(${TARGET} PUBLIC ${SYCL_FLAGS} -I${MPI_CXX_INCLUDE_DIRS})
+            target_link_options(${TARGET} PUBLIC ${SYCL_FLAGS} ${MPI_LIBRARIES})
+        endforeach()
+    else()
+        message(STATUS "Found MPI which is not offload device aware - skipping the MPI_with_SYCL demo")
+    endif()
 endif()