Add BUILD_DALI_NODEPS to allow building dali_core and dali_kernels wi…

…thout extra third party libraries present in the system (NVIDIA#2321)

Signed-off-by: Joaquin Anton <[email protected]>

CMakeLists.txt

@@ -16,6 +16,7 @@ cmake_minimum_required(VERSION 3.13)
 # allow usage of check_symbol_exists() macro
 include(CheckCXXSymbolExists)
 include(CheckCXXCompilerFlag)
+include(CMakeDependentOption)
 
 include(cmake/Utils.cmake)
 include(cmake/CUDA_utils.cmake)
@@ -34,8 +35,13 @@ check_cxx_compiler_flag(-fopenmp      CXX_HAVE_OMP)
 check_cxx_compiler_flag(-fopenmp-simd CXX_HAVE_OMP_SIMD)
 
 # Build options
-option(BUILD_TEST "Build googletest test suite" ON)
-option(BUILD_BENCHMARK "Build benchmark suite" ON)
+option(BUILD_DALI_NODEPS "Disable components that require extra external libraries to be present in the system. Effectively, it builds only the DALI core and kernel libraries")
+
+# Tests use OpenCV...
+cmake_dependent_option(BUILD_TEST "Build googletest test suite" ON
+                       "NOT BUILD_DALI_NODEPS" OFF)
+cmake_dependent_option(BUILD_BENCHMARK "Build benchmark suite" ON
+                       "NOT BUILD_DALI_NODEPS" OFF)
 option(BUILD_FUZZING "Build fuzzing suite" OFF)
 # if BUILD_NVTX is empty remove it and let is be default
 if ("${BUILD_NVTX}" STREQUAL "")
@@ -47,10 +53,29 @@ if(${CUDA_VERSION} VERSION_GREATER_EQUAL "10.0")
 else()
   option(BUILD_NVTX "Build with NVTX profiling enabled" OFF)
 endif()
-option(BUILD_PYTHON "Build Python bindings" ON)
-option(BUILD_LMDB "Build LMDB readers" OFF)
-option(BUILD_JPEG_TURBO "Build with libjpeg-turbo support" ON)
-option(BUILD_LIBTIFF "Build with libtiff support" ON)
+
+# Third party library dependencies
+cmake_dependent_option(BUILD_PYTHON "Build Python bindings" ON
+                       "NOT BUILD_DALI_NODEPS" OFF)
+cmake_dependent_option(BUILD_LMDB "Build LMDB readers" OFF
+                       "NOT BUILD_DALI_NODEPS" OFF)
+cmake_dependent_option(BUILD_JPEG_TURBO "Build with libjpeg-turbo support" ON
+                       "NOT BUILD_DALI_NODEPS" OFF)
+cmake_dependent_option(BUILD_LIBTIFF "Build with libtiff support" ON
+                       "NOT BUILD_DALI_NODEPS" OFF)
+cmake_dependent_option(BUILD_FFMPEG "Build with ffmpeg support" ON
+                       "NOT BUILD_DALI_NODEPS" OFF)
+cmake_dependent_option(BUILD_LIBSND "Build with suport for libsnd library" ON
+                       "NOT BUILD_DALI_NODEPS" OFF)
+cmake_dependent_option(BUILD_OPENCV "Build with suport for OpenCV library" ON
+                       "NOT BUILD_DALI_NODEPS" OFF)
+cmake_dependent_option(BUILD_PROTOBUF "Build with suport for Protobuf library" ON
+                       "NOT BUILD_DALI_NODEPS" OFF)
+option(BUILD_FFTS "Build with ffts support" ON)  # Built from thirdparty sources
+
+# NVIDIA libraries
+cmake_dependent_option(BUILD_NVDEC "Build with NVIDIA NVDEC support" ON
+                       "BUILD_FFMPEG" OFF)  # NVDEC support requires FFMPEG
 option(BUILD_NVJPEG "Build with nvJPEG support" ON)
 # if BUILD_NVJPEG2K is empty remove it and let is be default
 if ("${BUILD_NVJPEG2K}" STREQUAL "")
@@ -64,19 +89,36 @@ if(${CUDA_VERSION} VERSION_GREATER_EQUAL "11.0")
 endif()
 
 option(BUILD_NVOF "Build with NVIDIA OPTICAL FLOW SDK support" ON)
-option(BUILD_NVDEC "Build with NVIDIA NVDEC support" ON)
-option(BUILD_LIBSND "Build with suport for libsnd library" ON)
 option(BUILD_NVML "Build with NVIDIA Management Library (NVML) support" ON)
-option(BUILD_FFTS "Build with ffts support" ON)
-option(VERBOSE_LOGS "Adds verbose loging to DALI" OFF)
 
+# DALI modules
+# Note dali_core is always enabled
+set(BUILD_DALI_KERNELS ON)
+if (BUILD_DALI_KERNELS AND NOT BUILD_DALI_NODEPS)
+  set(BUILD_DALI_PIPELINE ON)
+  set(BUILD_DALI_OPERATORS ON)
+else()
+  set(BUILD_DALI_PIPELINE OFF)
+  set(BUILD_DALI_OPERATORS OFF)
+endif()
+
+# Experimental, only enabled for BUILD_DALI_NODEPS=ON
+cmake_dependent_option(STATIC_LIBS "Build static libraries instead of shared-object libraries" OFF
+                       "BUILD_DALI_NODEPS" OFF)
+
+option(VERBOSE_LOGS "Adds verbose loging to DALI" OFF)
 option(WERROR "Treat all warnings as errors" OFF)
 
 # ; creates a list here
 set (PYTHON_VERSIONS "3.6;3.7;3.8;3.9")
 
-# FFmpeg is required when we are using NVDEC for video reader
-set(BUILD_FFMPEG ${BUILD_NVDEC})
+if (STATIC_LIBS)
+  message (STATUS "Building static libraries")
+  set(LIBTYPE STATIC)
+else()
+  message (STATUS "Building shared-object libraries")
+  set(LIBTYPE SHARED)
+endif()
 
 propagate_option(BUILD_NVTX)
 propagate_option(BUILD_PYTHON)
@@ -91,6 +133,8 @@ propagate_option(BUILD_LIBSND)
 propagate_option(BUILD_NVML)
 propagate_option(BUILD_FFTS)
 propagate_option(BUILD_FFMPEG)
+propagate_option(BUILD_OPENCV)
+propagate_option(BUILD_PROTOBUF)
 
 get_dali_version(${PROJECT_SOURCE_DIR}/VERSION DALI_VERSION)
 
@@ -228,7 +272,16 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/lint.cmake)
 
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Install.cmake)
 
-add_library(DALI::dali ALIAS dali)
 add_library(DALI::dali_core ALIAS dali_core)
-add_library(DALI::dali_kernels ALIAS dali_kernels)
-add_library(DALI::dali_operators ALIAS dali_operators)
+
+if (BUILD_DALI_KERNELS)
+  add_library(DALI::dali_kernels ALIAS dali_kernels)
+endif()
+
+if (BUILD_DALI_PIPELINE)
+  add_library(DALI::dali ALIAS dali)
+endif()
+
+if (BUILD_DALI_OPERATORS)
+  add_library(DALI::dali_operators ALIAS dali_operators)
+endif()

cmake/Dependencies.cmake

@@ -83,22 +83,25 @@ include(cmake/Dependencies.common.cmake)
 # protobuf
 ##################################################################
 # link statically
-if(NOT DEFINED Protobuf_USE_STATIC_LIBS)
-set(Protobuf_USE_STATIC_LIBS YES)
-endif(NOT DEFINED Protobuf_USE_STATIC_LIBS)
-find_package(Protobuf 2.0 REQUIRED)
-if(${Protobuf_VERSION} VERSION_LESS "3.0")
-  message(STATUS "TensorFlow TFRecord file format support is not available with Protobuf 2")
-else()
-  message(STATUS "Enabling TensorFlow TFRecord file format support")
-  add_definitions(-DDALI_BUILD_PROTO3=1)
-  set(BUILD_PROTO3 ON CACHE STRING "Build proto3")
-endif()
 
-include_directories(SYSTEM ${Protobuf_INCLUDE_DIRS})
-list(APPEND DALI_LIBS ${Protobuf_LIBRARY})
-# hide things from the protobuf, all we export is only is API generated from our proto files
-list(APPEND DALI_EXCLUDES libprotobuf.a)
+if (BUILD_PROTOBUF)
+  if(NOT DEFINED Protobuf_USE_STATIC_LIBS)
+    set(Protobuf_USE_STATIC_LIBS YES)
+  endif(NOT DEFINED Protobuf_USE_STATIC_LIBS)
+  find_package(Protobuf 2.0 REQUIRED)
+  if(${Protobuf_VERSION} VERSION_LESS "3.0")
+    message(STATUS "TensorFlow TFRecord file format support is not available with Protobuf 2")
+  else()
+    message(STATUS "Enabling TensorFlow TFRecord file format support")
+    add_definitions(-DDALI_BUILD_PROTO3=1)
+    set(BUILD_PROTO3 ON CACHE STRING "Build proto3")
+  endif()
+
+  include_directories(SYSTEM ${Protobuf_INCLUDE_DIRS})
+  list(APPEND DALI_LIBS ${Protobuf_LIBRARY})
+  # hide things from the protobuf, all we export is only is API generated from our proto files
+  list(APPEND DALI_EXCLUDES libprotobuf.a)
+endif()
 
 set(DALI_SYSTEM_LIBS rt pthread m dl)
 list(APPEND DALI_LIBS ${CUDART_LIB} ${DALI_SYSTEM_LIBS})

cmake/Dependencies.common.cmake

@@ -15,17 +15,20 @@
 ##################################################################
 # OpenCV
 ##################################################################
-# For OpenCV 3 and later, 'imdecode()' is in the imgcodecs library
-find_package(OpenCV 4.0 QUIET COMPONENTS core imgproc imgcodecs)
-if(NOT OpenCV_FOUND)
-  find_package(OpenCV 3.0 REQUIRED COMPONENTS core imgproc imgcodecs)
-endif()
+if (BUILD_OPENCV)
+  # For OpenCV 3 and later, 'imdecode()' is in the imgcodecs library
+
+  find_package(OpenCV 4.0 QUIET COMPONENTS core imgproc imgcodecs)
+  if(NOT OpenCV_FOUND)
+    find_package(OpenCV 3.0 REQUIRED COMPONENTS core imgproc imgcodecs)
+  endif()
 
-message(STATUS "Found OpenCV: ${OpenCV_INCLUDE_DIRS} (found suitable version \"${OpenCV_VERSION}\", minimum required is \"3.0\")")
-include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
-list(APPEND DALI_LIBS ${OpenCV_LIBRARIES})
-message("OpenCV libraries: ${OpenCV_LIBRARIES}")
-list(APPEND DALI_EXCLUDES libopencv_core.a;libopencv_imgproc.a;libopencv_highgui.a;libopencv_imgcodecs.a;liblibwebp.a;libittnotify.a;libpng.a;liblibtiff.a;liblibjasper.a;libIlmImf.a;liblibjpeg-turbo.a)
+  message(STATUS "Found OpenCV: ${OpenCV_INCLUDE_DIRS} (found suitable version \"${OpenCV_VERSION}\", minimum required is \"3.0\")")
+  include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
+  list(APPEND DALI_LIBS ${OpenCV_LIBRARIES})
+  message("OpenCV libraries: ${OpenCV_LIBRARIES}")
+  list(APPEND DALI_EXCLUDES libopencv_core.a;libopencv_imgproc.a;libopencv_highgui.a;libopencv_imgcodecs.a;liblibwebp.a;libittnotify.a;libpng.a;liblibtiff.a;liblibjasper.a;libIlmImf.a;liblibjpeg-turbo.a)
+endif()
 
 ##################################################################
 #

cmake/Install.cmake

@@ -17,10 +17,24 @@
 #  Installing targets                  #
 ########################################
 
-install(TARGETS dali dali_core dali_operators dali_kernels
-        LIBRARY
-          DESTINATION lib
-          PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ)
+set (DALI_LIBS dali_core)
+
+if (BUILD_DALI_KERNELS)
+  list(APPEND DALI_LIBS dali_kernels)
+endif()
+
+if (BUILD_DALI_PIPELINE)
+  list(APPEND DALI_LIBS dali)
+endif()
+
+if (BUILD_DALI_OPERATORS)
+  list(APPEND DALI_LIBS dali_operators)
+endif()
+
+install(TARGETS ${DALI_LIBS}
+        ARCHIVE DESTINATION lib
+        LIBRARY DESTINATION lib
+        PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ)
 
 # Note the '/' at the end of first path and not present at the end of the other
 install(DIRECTORY ${CMAKE_SOURCE_DIR}/include/ ${CMAKE_SOURCE_DIR}/dali

dali/CMakeLists.txt

@@ -31,42 +31,54 @@ set(TEST_BINARY_DIR "${PROJECT_BINARY_DIR}/${DALI_WHEEL_DIR}/test")
 string(REPLACE ";" ":" exclude_libs "${DALI_EXCLUDES}")
 
 ################################################
-# Build libdali
+# Build DALI libraries
 ################################################
-add_subdirectory(image)
 add_subdirectory(core)
-add_subdirectory(kernels)
-add_subdirectory(pipeline)
-add_subdirectory(operators)
-add_subdirectory(util)
-add_subdirectory(plugin)
-add_subdirectory(c_api)
+
+if (BUILD_DALI_KERNELS)
+  add_subdirectory(kernels)
+endif()
+
+if (BUILD_DALI_PIPELINE)
+  add_subdirectory(pipeline)
+  add_subdirectory(image)
+  add_subdirectory(util)
+  add_subdirectory(plugin)
+  add_subdirectory(c_api)
+endif()
+
+if (BUILD_DALI_OPERATORS)
+  add_subdirectory(operators)
+endif()
 
 # Collect source files for dali
 collect_headers(DALI_INST_HDRS PARENT_SCOPE)
 collect_sources(DALI_SRCS PARENT_SCOPE)
+if (BUILD_PROTOBUF)
+  set(DALI_PROTO_OBJ $<TARGET_OBJECTS:DALI_PROTO>)
+  add_library(dali ${LIBTYPE} ${DALI_SRCS} ${DALI_PROTO_OBJ} ${CUDART_LIB})
+  set_target_properties(dali PROPERTIES
+      LIBRARY_OUTPUT_DIRECTORY "${DALI_LIBRARY_OUTPUT_DIR}")
+endif()
 
-set(DALI_PROTO_OBJ $<TARGET_OBJECTS:DALI_PROTO>)
-add_library(dali SHARED ${DALI_SRCS} ${DALI_PROTO_OBJ} ${CUDART_LIB})
-set_target_properties(dali PROPERTIES
-    LIBRARY_OUTPUT_DIRECTORY "${DALI_LIBRARY_OUTPUT_DIR}")
-
-# Define symbol version script for libdali.so
-set(dali_lib_exports "libdali.map")
-configure_file("${PROJECT_SOURCE_DIR}/cmake/${dali_lib_exports}.in" "${CMAKE_BINARY_DIR}/${dali_lib_exports}")
-target_link_libraries(dali PRIVATE -Wl,--version-script=${CMAKE_BINARY_DIR}/${dali_lib_exports})
-
-# Link in dali's dependencies
-message(STATUS "Adding dependencies to target `dali`: '${DALI_LIBS}'")
-target_link_libraries(dali PUBLIC dali_core dali_kernels)
-target_link_libraries(dali PRIVATE ${DALI_LIBS} dynlink_cuda)
-# Exclude (most) statically linked dali dependencies from the exports of libdali.so
-target_link_libraries(dali PRIVATE "-Wl,--exclude-libs,${exclude_libs}")
+if (BUILD_DALI_PIPELINE)
+  # Define symbol version script for libdali.so
+  set(dali_lib_exports "libdali.map")
+  configure_file("${PROJECT_SOURCE_DIR}/cmake/${dali_lib_exports}.in" "${CMAKE_BINARY_DIR}/${dali_lib_exports}")
+  target_link_libraries(dali PRIVATE -Wl,--version-script=${CMAKE_BINARY_DIR}/${dali_lib_exports})
+
+  # Link in dali's dependencies
+  message(STATUS "Adding dependencies to target `dali`: '${DALI_LIBS}'")
+  target_link_libraries(dali PUBLIC dali_core dali_kernels)
+  target_link_libraries(dali PRIVATE ${DALI_LIBS} dynlink_cuda)
+  # Exclude (most) statically linked dali dependencies from the exports of libdali.so
+  target_link_libraries(dali PRIVATE "-Wl,--exclude-libs,${exclude_libs}")
+endif()
 
 ################################################
 # Build test suite
 ################################################
-if (BUILD_TEST)
+if (BUILD_DALI_PIPELINE AND BUILD_TEST)
   add_subdirectory(test)
   add_executable(dali_test "${DALI_TEST_SRCS}")
 
@@ -132,20 +144,22 @@ endif()
 
 # Copy all headers from DALI_INST_HDRS list to DALI_WHEEL_DIR using install command
 # with `-D` option, that recursively creates missing directories in destination path
-add_custom_target(install_headers ALL
-    DEPENDS dali dali_operators
-)
-
-# Process the DALI_INST_HDRS list
-foreach(INSTALL_HEADER ${DALI_INST_HDRS})
-  file(RELATIVE_PATH HEADER_RELATIVE ${PROJECT_SOURCE_DIR} ${INSTALL_HEADER})
+if (BUILD_PYTHON)
+  add_custom_target(install_headers ALL
+      DEPENDS dali dali_operators
+      )
+
+  # Process the DALI_INST_HDRS list
+  foreach(INSTALL_HEADER ${DALI_INST_HDRS})
+    file(RELATIVE_PATH HEADER_RELATIVE ${PROJECT_SOURCE_DIR} ${INSTALL_HEADER})
+    add_custom_command(
+      TARGET install_headers
+      COMMAND install -D "${INSTALL_HEADER}" "${PROJECT_BINARY_DIR}/${DALI_INCLUDE_DIR}/${HEADER_RELATIVE}")
+  endforeach(INSTALL_HEADER)
+
+  # Copy proper `include` dir
   add_custom_command(
     TARGET install_headers
-    COMMAND install -D "${INSTALL_HEADER}" "${PROJECT_BINARY_DIR}/${DALI_INCLUDE_DIR}/${HEADER_RELATIVE}")
-endforeach(INSTALL_HEADER)
-
-# Copy proper `include` dir
-add_custom_command(
-  TARGET install_headers
-  COMMAND cp -r "${PROJECT_SOURCE_DIR}/include/." "${PROJECT_BINARY_DIR}/${DALI_INCLUDE_DIR}"
-)
+    COMMAND cp -r "${PROJECT_SOURCE_DIR}/include/." "${PROJECT_BINARY_DIR}/${DALI_INCLUDE_DIR}"
+  )
+endif()

dali/core/CMakeLists.txt

@@ -22,7 +22,7 @@ list(REMOVE_ITEM DALI_CORE_SRCS dynlink_cuda.cc)
 
 add_library(dynlink_cuda STATIC dynlink_cuda.cc)
 
-add_library(dali_core SHARED ${DALI_CORE_SRCS})
+add_library(dali_core ${LIBTYPE} ${DALI_CORE_SRCS})
 target_include_directories(dali_core PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
 target_link_libraries(dali_core PRIVATE ${CUDART_LIB})
 target_link_libraries(dali_core PUBLIC ${DALI_SYSTEM_LIBS})

dali/kernels/CMakeLists.txt

@@ -34,7 +34,7 @@ collect_sources(DALI_KERNEL_SRCS)
 collect_test_sources(DALI_KERNEL_TEST_SRCS)
 
 # cuFFT library
-add_library(dali_kernels SHARED ${DALI_KERNEL_SRCS})
+add_library(dali_kernels ${LIBTYPE} ${DALI_KERNEL_SRCS})
 target_link_libraries(dali_kernels PUBLIC dali_core)
 target_link_libraries(dali_kernels PRIVATE ${CUDA_cufft_static_LIBRARY})
 target_link_libraries(dali_kernels PRIVATE ${DALI_LIBS})

dali/operators/CMakeLists.txt

@@ -45,7 +45,7 @@ if (BUILD_PROTO3)
   list(APPEND DALI_OPERATOR_PROTO_OBJ $<TARGET_OBJECTS:TF_PROTO>)
 endif()
 
-add_library(dali_operators SHARED ${DALI_OPERATOR_SRCS} ${DALI_OPERATOR_PROTO_OBJ})
+add_library(dali_operators ${LIBTYPE} ${DALI_OPERATOR_SRCS} ${DALI_OPERATOR_PROTO_OBJ})
 set_target_properties(dali_operators PROPERTIES
     LIBRARY_OUTPUT_DIRECTORY "${DALI_LIBRARY_OUTPUT_DIR}")
 target_link_libraries(dali_operators PUBLIC dali dali_kernels dali_core)

qa/TL1_nodeps_build/CMakeLists_submodule.txt

@@ -0,0 +1,11 @@
+cmake_minimum_required(VERSION 3.13)
+project(nodeps_test CUDA CXX)
+
+set(BUILD_DALI_NODEPS ON)
+set(STATIC_LIBS ON)
+add_subdirectory(dali)
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/dali/include)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/dali/)
+add_executable(nodeps_test main.cc)
+target_link_libraries(nodeps_test dali_core dali_kernels)

qa/TL1_nodeps_build/CMakeLists_system.txt

@@ -0,0 +1,5 @@
+cmake_minimum_required(VERSION 3.13)
+project(nodeps_test CUDA CXX)
+
+add_executable(nodeps_test main.cc)
+target_link_libraries(nodeps_test dali_core dali_kernels)