enhance: enable asan for cpp unittest

Signed-off-by: chyezh <[email protected]>

.github/workflows/main.yaml

@@ -112,6 +112,8 @@ jobs:
           kind: 'cpp'
 
   UT-Cpp:
+    # skip the UT-Cpp job in github workflow, it run in jenkins now see: UT-CPP.groovy
+    if: false 
     name: UT for Cpp
     needs: Build
     runs-on: ubuntu-latest

Makefile

@@ -30,8 +30,8 @@ endif
 use_asan = OFF
 ifdef USE_ASAN
 	use_asan =${USE_ASAN}
-	CGO_LDFLAGS := $(shell go env CGO_LDFLAGS) -fsanitize=address -fno-omit-frame-pointer
-	CGO_CFLAGS := $(shell go env CGO_CFLAGS) -fsanitize=address -fno-omit-frame-pointer
+	CGO_LDFLAGS := $(shell go env CGO_LDFLAGS) -fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address
+	CGO_CFLAGS := $(shell go env CGO_CFLAGS) -fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address
 	MILVUS_GO_BUILD_TAGS := $(MILVUS_GO_BUILD_TAGS),use_asan
 endif
 

ci/jenkins/UT-CPP.groovy

@@ -1,4 +1,4 @@
-@Library('jenkins-shared-library@tekton') _
+@Library('jenkins-shared-library@v0.62.0') _
 
 def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml'
 def milvus_helm_chart_version = '4.2.8'
@@ -38,7 +38,7 @@ pipeline {
                                               gitMode: gitMode ,
                                               gitBaseRef: gitBaseRef,
                                               pullRequestNumber: "$env.CHANGE_ID",
-                                              make_cmd: "make clean && make USE_ASAN=OFF build-cpp-with-coverage",
+                                              make_cmd: "make clean && make USE_ASAN=ON build-cpp-with-coverage",
                                               test_entrypoint: "./scripts/run_cpp_codecov.sh",
                                               codecov_files: "./lcov_output.info,./it_coverage.txt"
                     }

internal/core/CMakeLists.txt

@@ -131,8 +131,8 @@ if (LINUX OR MSYS)
                   )
     if (USE_ASAN STREQUAL "ON") 
         message( STATUS "Building Milvus Core Using AddressSanitizer")
-        add_compile_options(-fno-omit-frame-pointer -fsanitize=address)
-        add_link_options(-fno-omit-frame-pointer -fsanitize=address)
+        add_compile_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
+        add_link_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
     endif()
     if (CMAKE_BUILD_TYPE STREQUAL "Release")
         append_flags( CMAKE_CXX_FLAGS

internal/core/src/segcore/vector_index_c.cpp

@@ -27,7 +27,7 @@ GetIndexListSize() {
 
 void
 GetIndexFeatures(void* index_key_list, uint64_t* index_feature_list) {
-    auto features = knowhere::IndexFactory::Instance().GetIndexFeatures();
+    auto& features = knowhere::IndexFactory::Instance().GetIndexFeatures();
     int idx = 0;
 
     const char** index_keys = (const char**)index_key_list;

internal/core/unittest/test_storage.cpp

@@ -38,21 +38,24 @@ get_azure_storage_config() {
         "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/"
         "K1SZFPTOtr/KBHBeksoGMGw==";
 
-    return CStorageConfig{endpoint,
-                          bucketName.c_str(),
-                          accessKey,
-                          accessValue,
-                          rootPath.c_str(),
-                          "remote",
-                          "azure",
-                          "",
-                          "error",
-                          "",
-                          false,
-                          "",
-                          false,
-                          false,
-                          30000};
+    return CStorageConfig{
+        endpoint,
+        bucketName.c_str(),
+        accessKey,
+        accessValue,
+        rootPath.c_str(),
+        "remote",
+        "azure",
+        "",
+        "error",
+        "",
+        false,
+        "",
+        false,
+        false,
+        30000,
+        "",
+    };
 }
 
 class StorageTest : public testing::Test {
@@ -93,7 +96,9 @@ TEST_F(StorageTest, GetLocalUsedSize) {
 
 TEST_F(StorageTest, InitRemoteChunkManagerSingleton) {
     CStorageConfig storageConfig = get_azure_storage_config();
-    InitRemoteChunkManagerSingleton(storageConfig);
+    auto status = InitRemoteChunkManagerSingleton(storageConfig);
+    EXPECT_STREQ(status.error_msg, "");
+    EXPECT_EQ(status.error_code, Success);
     auto rcm =
         RemoteChunkManagerSingleton::GetInstance().GetRemoteChunkManager();
     EXPECT_EQ(rcm->GetRootPath(), "/tmp/milvus/remote_data");

scripts/install_milvus.sh

@@ -24,8 +24,6 @@ cp $PWD"/internal/core/output/lib/"*.dylib* "$LIBRARY_PATH" 2>/dev/null || true
 cp $PWD"/internal/core/output/lib/"*.so* "$LIBRARY_PATH" || true
 cp $PWD"/internal/core/output/lib64/"*.so* "$LIBRARY_PATH" 2>/dev/null || true
 
-if [ "$USE_ASAN" == "ON" ]; then
-    for LIB_PATH in $(ldconfig -p | grep -E '(asan|atomic)' | awk '{print $NF}'); do
-        cp "$LIB_PATH" "$LIBRARY_PATH" 2>/dev/null
-    done
-fi
+for LIB_PATH in $(ldd ./bin/milvus | grep -E '(asan|atomic)' | awk '{print $3}'); do
+    cp "$LIB_PATH" "$LIBRARY_PATH" 2>/dev/null
+done

scripts/run_cpp_codecov.sh

@@ -21,11 +21,11 @@ set -e
 
 SOURCE="${BASH_SOURCE[0]}"
 while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
-  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-  SOURCE="$(readlink "$SOURCE")"
-  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+    DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)"
+    SOURCE="$(readlink "$SOURCE")"
+    [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
 done
-ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
+ROOT_DIR="$(cd -P "$(dirname "$SOURCE")/.." && pwd)"
 source ${ROOT_DIR}/scripts/setenv.sh
 
 MILVUS_CORE_DIR="${ROOT_DIR}/internal/core"
@@ -59,13 +59,18 @@ if [ $? -ne 0 ]; then
     exit -1
 fi
 # starting the timer
-beginTime=`date +%s`
+beginTime=$(date +%s)
 
 # run unittest
-for test in `ls ${MILVUS_CORE_UNITTEST_DIR}`; do
+for test in $(ls ${MILVUS_CORE_UNITTEST_DIR}); do
     echo "Running cpp unittest: ${MILVUS_CORE_UNITTEST_DIR}/$test"
     # run unittest
-    ${MILVUS_CORE_UNITTEST_DIR}/${test}
+    if [ -n "$MILVUS_ENABLE_ASAN_LIB" ]; then
+        echo "ASAN is enabled with env MILVUS_ENABLE_ASAN_LIB, set {$MILVUS_ENABLE_ASAN_LIB} at the front of LD_PRELOAD"
+        LD_PRELOAD="$MILVUS_ENABLE_ASAN_LIB:$LD_PRELOAD" ${MILVUS_CORE_UNITTEST_DIR}/${test}
+    else
+        ${MILVUS_CORE_UNITTEST_DIR}/${test}
+    fi
     if [ $? -ne 0 ]; then
         echo ${args}
         echo "${MILVUS_CORE_UNITTEST_DIR}/${test} run failed"
@@ -94,7 +99,6 @@ ${LCOV_CMD} -r "${FILE_INFO_COMBINE}" -o "${FILE_INFO_OUTPUT}" \
 ${LCOV_GEN_CMD} ${FILE_INFO_OUTPUT} --output-directory ${DIR_LCOV_OUTPUT}/
 echo "Generate cpp code coverage report to ${DIR_LCOV_OUTPUT}"
 
+endTime=$(date +%s)
 
-endTime=`date +%s`
-
-echo "Total time for cpp unittest:" $(($endTime-$beginTime)) "s"
+echo "Total time for cpp unittest:" $(($endTime - $beginTime)) "s"

scripts/setenv.sh

@@ -36,6 +36,13 @@ unameOut="$(uname -s)"
 
 case "${unameOut}" in
     Linux*)
+      # check if use asan.
+      MILVUS_ENABLE_ASAN_LIB=$(ldd $ROOT_DIR/internal/core/output/lib/libmilvus_core.so | grep asan | awk '{print $3}')
+      if [ -n "$MILVUS_ENABLE_ASAN_LIB" ]; then
+          echo "Enable ASAN With ${MILVUS_ENABLE_ASAN_LIB}"
+          export MILVUS_ENABLE_ASAN_LIB="$MILVUS_ENABLE_ASAN_LIB"
+      fi
+
       LIBJEMALLOC=$PWD/internal/core/output/lib/libjemalloc.so
       if test -f "$LIBJEMALLOC"; then
         export LD_PRELOAD="$LIBJEMALLOC"