new(lct): Lightweight Communication Tools

.circleci/config.yml

@@ -37,7 +37,8 @@ jobs:
           command: |
             shopt -s globstar # to activate the ** globbing
             clang-format-11 --version
-            clang-format-11 -i $(find src -type f -name "*.h" -o -name "*.c" -o -name "*.hpp" -o -name "*.cpp")
+            clang-format-11 -i $(find lct -type f -name "*.h" -o -name "*.c" -o -name "*.hpp" -o -name "*.cpp")
+            clang-format-11 -i $(find lci -type f -name "*.h" -o -name "*.c" -o -name "*.hpp" -o -name "*.cpp")
             clang-format-11 -i $(find examples -type f -name "*.h" -o -name "*.c" -o -name "*.hpp" -o -name "*.cpp")
             clang-format-11 -i $(find tests -type f -name "*.h" -o -name "*.c" -o -name "*.hpp" -o -name "*.cpp")
             git diff --exit-code > /tmp/clang_format_results.txt

CMakeLists.txt

@@ -99,9 +99,10 @@ endif()
 # LCI Optimization Options
 # ##############################################################################
 option(LCI_DEBUG "LCI Debug Mode" OFF)
-option(LCI_USE_MUTEX_CQ
-       "Use mutex lock to ensure the thread safety of the completion queue."
-       OFF)
+option(
+  LCI_USE_INLINE_CQ
+  "Use the C version of the completion queue so that it could be compiled inline."
+  OFF)
 option(LCI_ENABLE_MULTITHREAD_PROGRESS
        "LCI_progress can be called by multiple threads simultaneously" OFF)
 option(LCI_IBV_ENABLE_TRY_LOCK_QP
@@ -221,14 +222,19 @@ endif()
 # ##############################################################################
 # Add the actual LCI library
 # ##############################################################################
+add_library(LCT)
+set_target_properties(LCT PROPERTIES CXX_VISIBILITY_PRESET hidden CXX_STANDARD
+                                                                  17)
+target_link_libraries(LCT PUBLIC Threads::Threads)
+
 add_library(LCI)
 set_target_properties(
   LCI
   PROPERTIES C_VISIBILITY_PRESET hidden
              C_STANDARD 11
              C_EXTENSIONS ON)
 target_compile_definitions(LCI PRIVATE _GNU_SOURCE)
-target_link_libraries(LCI PUBLIC Threads::Threads ${FABRIC}::${FABRIC})
+target_link_libraries(LCI PUBLIC Threads::Threads ${FABRIC}::${FABRIC} LCT)
 if(LCI_USE_AVX)
   target_compile_options(LCI PUBLIC -mavx)
 endif()
@@ -237,8 +243,10 @@ if(LCI_USE_PAPI)
 endif()
 
 # set_target_properties(LCI PROPERTIES OUTPUT_NAME lci)
-add_subdirectory(src)
+add_subdirectory(lct)
+add_subdirectory(lci)
 add_subdirectory(dependency)
+target_link_libraries(LCT PRIVATE lci-ucx)
 target_link_libraries(LCI PRIVATE lci-ucx)
 if(LCI_WITH_EXAMPLES)
   add_subdirectory(examples)
@@ -266,12 +274,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   configure_file(liblci.pc.in liblci.pc @ONLY)
 
   install(
-    TARGETS LCI
-    EXPORT LCITargets
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
-  install(
-    TARGETS lci-ucx
+    TARGETS LCI lci-ucx LCT
     EXPORT LCITargets
     ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
@@ -281,12 +284,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     NAMESPACE LCI::
     DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/LCI)
   install(
-    DIRECTORY src/api/
-    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-    FILES_MATCHING
-    PATTERN "*.h")
-  install(
-    DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/src/api/
+    DIRECTORY lci/api/ lct/api/ ${CMAKE_CURRENT_BINARY_DIR}/lct/api/
+              ${CMAKE_CURRENT_BINARY_DIR}/lci/api/
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
     FILES_MATCHING
     PATTERN "*.h")

LCIConfig.cmake.in

@@ -8,6 +8,16 @@ set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 find_dependency(Threads)
 find_dependency(@FABRIC@)
 
+set_and_check(LCT_INCLUDE_DIRS "@PACKAGE_CMAKE_INSTALL_INCLUDEDIR@")
+set_and_check(LCT_SHARED_LIBRARY "@PACKAGE_CMAKE_INSTALL_LIBDIR@/@CMAKE_SHARED_LIBRARY_PREFIX@LCT@CMAKE_SHARED_LIBRARY_SUFFIX@")
+
+add_library(LCI::LCT SHARED IMPORTED)
+set_target_properties(LCI::LCT PROPERTIES
+  IMPORTED_LOCATION ${LCT_SHARED_LIBRARY}
+)
+target_include_directories(LCI::LCT INTERFACE ${LCT_INCLUDE_DIRS})
+target_link_libraries(LCI::LCT INTERFACE Threads::Threads)
+
 set_and_check(LCI_INCLUDE_DIRS "@PACKAGE_CMAKE_INSTALL_INCLUDEDIR@")
 set_and_check(LCI_SHARED_LIBRARY "@PACKAGE_CMAKE_INSTALL_LIBDIR@/@CMAKE_SHARED_LIBRARY_PREFIX@LCI@CMAKE_SHARED_LIBRARY_SUFFIX@")
 # set_and_check(LCI_STATIC_LIBRARY "@PACKAGE_CMAKE_INSTALL_LIBDIR@/@CMAKE_STATIC_LIBRARY_PREFIX@LCI@CMAKE_STATIC_LIBRARY_SUFFIX@")
@@ -17,7 +27,7 @@ set_target_properties(LCI::Shared PROPERTIES
   IMPORTED_LOCATION ${LCI_SHARED_LIBRARY}
 )
 target_include_directories(LCI::Shared INTERFACE ${LCI_INCLUDE_DIRS})
-target_link_libraries(LCI::Shared INTERFACE Threads::Threads @FABRIC@::@FABRIC@)
+target_link_libraries(LCI::Shared INTERFACE Threads::Threads @FABRIC@::@FABRIC@ LCI::LCT)
 
 add_library(LCI::LCI ALIAS LCI::Shared)
 

README.md

@@ -45,7 +45,7 @@ srun -n 2 ./hello_world
 
 See `examples` and `tests` for some example code.
 
-See `src/api/lci.h` for public APIs.
+See `lci/api/lci.h` for public APIs.
 
 `doxygen` for a full [documentation](https://uiuc-hpc.github.io/LC/).
 

cmake_modules/FindPAPI.cmake

@@ -26,7 +26,7 @@ if(NOT TARGET Papi::papi)
   if(NOT PAPI_ROOT AND "$ENV{PAPI_ROOT}")
     set(PAPI_ROOT $ENV{PAPI_ROOT})
   elseif(NOT PAPI_ROOT)
-    string(REPLACE "/src/api" "" PAPI_ROOT "${PAPI_INCLUDE_DIR}")
+    string(REPLACE "/include" "" PAPI_ROOT "${PAPI_INCLUDE_DIR}")
   endif()
 
   # Set PAPI_ROOT in case the other hints are used
@@ -37,7 +37,7 @@ if(NOT TARGET Papi::papi)
     file(TO_CMAKE_PATH $ENV{PAPI_ROOT} PAPI_ROOT)
   else()
     file(TO_CMAKE_PATH "${PAPI_INCLUDE_DIR}" PAPI_INCLUDE_DIR)
-    string(REPLACE "/src/api" "" PAPI_ROOT "${PAPI_INCLUDE_DIR}")
+    string(REPLACE "/include" "" PAPI_ROOT "${PAPI_INCLUDE_DIR}")
   endif()
 
   set(PAPI_LIBRARIES ${PAPI_LIBRARY})

contrib/spack/packages/lci/package.py

@@ -36,8 +36,8 @@ def is_positive_int(val):
             description='Cache line size, in bytes')
     variant('native', default=True, description='Build with -march=native')
 
-    variant('cq', default='aqueue', values=('aqueue', 'mutex-dq'), multi=False,
-            description='Completion queue implementation')
+    variant('inline-cq', default=False,
+            description='Use the inline C completion queue implementation')
     variant('qp-lock', default=True,
             description='Lock queue pairs before access')
 
@@ -96,8 +96,7 @@ def cmake_args(self):
             self.define_from_variant('LCI_USE_AVX', 'vector'),
             self.define_from_variant('LCI_CONFIG_USE_ALIGNED_ALLOC', 'aligned'),
             self.define_from_variant('LCI_OPTIMIZE_FOR_NATIVE', 'native'),
-            self.define('LCI_USE_MUTEX_CQ',
-                        self.spec.variants['cq'].value == 'mutex-dq'),
+            self.define('LCI_USE_INLINE_CQ', 'inline-cq'),
             self.define_from_variant('LCI_IBV_ENABLE_TRY_LOCK_QP', 'qp-lock'),
             self.define('LCI_ENABLE_MULTITHREAD_PROGRESS', 'multithread-progress'),
             self.define('LCI_USE_DREG_DEFAULT',

dependency/ucx/CMakeLists.txt

@@ -120,6 +120,13 @@ if(LCI_UCX_ENABLE_TUNING)
   set(ENABLE_TUNING ON)
 endif()
 
+set(LCI_UCX_ENABLE_PROFILING
+    OFF
+    CACHE STRING "Enable profiling")
+if(LCI_UCX_ENABLE_PROFILING)
+  set(HAVE_PROFILING ON)
+endif()
+
 set(LCI_UCX_MAX_LOG_LEVEL
     debug
     CACHE STRING "Highest log level")

dependency/ucx/lci_ucx_api.h

@@ -5,6 +5,10 @@
 
 #define LCII_API __attribute__((visibility("default")))
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 LCII_API void LCII_ucs_init(void);
 LCII_API void LCII_ucs_cleanup(void);
 
@@ -34,4 +38,8 @@ LCII_API double LCII_ucs_time_to_usec(LCII_ucs_time_t t);
 LCII_API double LCII_ucs_time_to_msec(LCII_ucs_time_t t);
 LCII_API double LCII_ucs_time_to_sec(LCII_ucs_time_t t);
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif  // LCI_LCI_UCX_API_H

format.sh

@@ -3,7 +3,8 @@
 echo "Formating c/c++ files..."
 shopt -s globstar # to activate the ** globbing
 clang-format-11 --version
-clang-format-11 -i $(find src -type f -name "*.h" -o -name "*.c" -o -name "*.hpp" -o -name "*.cpp")
+clang-format-11 -i $(find lct -type f -name "*.h" -o -name "*.c" -o -name "*.hpp" -o -name "*.cpp")
+clang-format-11 -i $(find lci -type f -name "*.h" -o -name "*.c" -o -name "*.hpp" -o -name "*.cpp")
 clang-format-11 -i $(find examples -type f -name "*.h" -o -name "*.c" -o -name "*.hpp" -o -name "*.cpp")
 clang-format-11 -i $(find tests -type f -name "*.h" -o -name "*.c" -o -name "*.hpp" -o -name "*.cpp")
 

src/CMakeLists.txt → lci/CMakeLists.txt

@@ -6,7 +6,7 @@ target_include_directories(LCI PRIVATE .)
 target_sources_relative(
   LCI
   PRIVATE
-  log/lcm_log.c
+  log/logger.c
   profile/papi_wrapper.c
   profile/performance_counter.c
   runtime/1sided_primitive.c
@@ -25,7 +25,6 @@ target_sources_relative(
   runtime/device.c
   runtime/endpoint.c
   runtime/completion/amhandler.c
-  runtime/monitor_thread.c
   runtime/rcache/lcii_rcache.c
   runtime/completion/sync_flag.c)
 

src/api/lci.h → lci/api/lci.h

@@ -14,6 +14,7 @@
 #include <stdbool.h>
 #include <stdio.h>
 #include "lci_config.h"
+#include "lct.h"
 
 #ifdef __cplusplus
 extern "C" {

src/api/lci_config.h.in → lci/api/lci_config.h.in

@@ -1,7 +1,3 @@
-//
-// Created by jiakunyan on 4/13/22.
-//
-
 #ifndef LCI_LCI_CONFIG_H
 #define LCI_LCI_CONFIG_H
 

src/backend/ibv/lcisi_ibv_detail.c → lci/backend/ibv/lcisi_ibv_detail.c

@@ -61,7 +61,7 @@ bool LCISI_ibv_select_best_device_port(struct ibv_device** dev_list,
     struct ibv_context* dev_ctx;
     dev_ctx = ibv_open_device(device);
     if (!dev_ctx) {
-      LCM_Log(LCM_LOG_INFO, "ibv", "Couldn't get context for %s.\n",
+      LCI_Log(LCI_LOG_INFO, "ibv", "Couldn't get context for %s.\n",
               ibv_get_device_name(device));
       continue;
     }
@@ -70,7 +70,7 @@ bool LCISI_ibv_select_best_device_port(struct ibv_device** dev_list,
     struct ibv_device_attr dev_attr;
     int ret = ibv_query_device(dev_ctx, &dev_attr);
     if (ret != 0) {
-      LCM_Log(LCM_LOG_INFO, "ibv", "Unable to query device %s.\n",
+      LCI_Log(LCI_LOG_INFO, "ibv", "Unable to query device %s.\n",
               ibv_get_device_name(device));
       goto close_device;
     }
@@ -81,13 +81,13 @@ bool LCISI_ibv_select_best_device_port(struct ibv_device** dev_list,
     for (uint8_t port_num = 1; port_num <= dev_attr.phys_port_cnt; port_num++) {
       ret = ibv_query_port(dev_ctx, port_num, &port_attr);
       if (ret != 0) {
-        LCM_Log(LCM_LOG_INFO, "ibv", "Unable to query port (%s:%d).\n",
+        LCI_Log(LCI_LOG_INFO, "ibv", "Unable to query port (%s:%d).\n",
                 ibv_get_device_name(device), port_num);
         continue;
       }
       // Check whether the port is active
       if (port_attr.state != IBV_PORT_ACTIVE) {
-        LCM_Log(LCM_LOG_INFO, "ibv", "%s:%d is not active (state: %d).\n",
+        LCI_Log(LCI_LOG_INFO, "ibv", "%s:%d is not active (state: %d).\n",
                 ibv_get_device_name(device), port_num, port_attr.state);
         continue;
       }
@@ -99,20 +99,20 @@ bool LCISI_ibv_select_best_device_port(struct ibv_device** dev_list,
       // Calculate its speed
       int width = translate_width(port_attr.active_width);
       if (width <= 0) {
-        LCM_Log(LCM_LOG_INFO, "ibv", "%s:%d invalid width %d (%d).\n",
+        LCI_Log(LCI_LOG_INFO, "ibv", "%s:%d invalid width %d (%d).\n",
                 ibv_get_device_name(device), port_num, width,
                 port_attr.active_width);
         continue;
       }
       double speed = translate_speed(port_attr.active_speed);
       if (speed <= 0) {
-        LCM_Log(LCM_LOG_INFO, "ibv", "%s:%d invalid speed %f (%d).\n",
+        LCI_Log(LCI_LOG_INFO, "ibv", "%s:%d invalid speed %f (%d).\n",
                 ibv_get_device_name(device), port_num, speed,
                 port_attr.active_width);
         continue;
       }
       double total_speed = speed * width;
-      LCM_Log(LCM_LOG_INFO, "ibv", "%s:%d speed is %.f (%d x %f).\n",
+      LCI_Log(LCI_LOG_INFO, "ibv", "%s:%d speed is %.f (%d x %f).\n",
               ibv_get_device_name(device), port_num, total_speed, width, speed);
       // Update the record if it is better.
       if (total_speed > best_speed) {
@@ -129,11 +129,11 @@ bool LCISI_ibv_select_best_device_port(struct ibv_device** dev_list,
   if (best_speed > 0) {
     *device_o = best_device;
     *port_o = best_port;
-    LCM_Log(LCM_LOG_INFO, "ibv", "Select the best device %s:%d.\n",
+    LCI_Log(LCI_LOG_INFO, "ibv", "Select the best device %s:%d.\n",
             ibv_get_device_name(best_device), best_port);
     return true;
   } else {
-    LCM_Log(LCM_LOG_INFO, "ibv", "No device is available!\n");
+    LCI_Log(LCI_LOG_INFO, "ibv", "No device is available!\n");
     return false;
   }
 }