ci: adaptation for windows (#232)

Signed-off-by: Ji Bin <[email protected]>

.github/workflows/main.yaml

@@ -152,3 +152,40 @@ jobs:
       - name: Uint Testing
         run: |
           make test
+  windows:
+    name: Build and test windows
+    runs-on: windows-2022
+    timeout-minutes: 45
+    env:
+      CCACHE_DIR: ${{ github.workspace }}/.ccache
+      CCACHE_COMPILERCHECK: content
+      CCACHE_COMPRESS: 1
+      CCACHE_COMPRESSLEVEL: 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Cache ccache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ github.workspace }}/.ccache
+          key: windows-ccache-${{ github.sha }}
+          restore-keys: windows-ccache-
+      - name: Install dependencies on windows
+        shell: cmd
+        run: |
+          choco install cmake ninja ccache
+          cmake --version
+          ninja --version
+      - name: Build
+        shell: cmd
+        run: |
+          call "%ProgramFiles%\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
+          cmake -S . -B build -DMILVUS_BUILD_TEST=YES -G Ninja
+          cmake --build build
+      - name: Uint Testing
+        shell: cmd
+        run: |
+          build\test\testing-ut
+          build\test\testing-it
+

CMakeLists.txt

@@ -32,25 +32,21 @@ find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
     set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
     set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
+    message(STATUS "using ccache")
 endif(CCACHE_FOUND)
 
-include(FindPython)
 include(FindClangTools)
 include(GoogleTest)
 include(CTest)
 
 include(DefineOptions)
-include(ThirdPartyPackages)
-include(MilvusProtoGen)
 
 # options
 
 set_option_category("Build")
 
-define_option(BUILD_TEST "Build with testing" OFF)
-define_option(BUILD_COVERAGE "Build with coverage" OFF)
-define_option(BUILD_SHARED "Build with shared" ON)
-define_option(BUILD_STATIC "Build with static" OFF)
+define_option(MILVUS_BUILD_TEST "Build with testing" OFF)
+define_option(MILVUS_BUILD_COVERAGE "Build with coverage" OFF)
 define_option_string(MILVUS_SDK_VERSION
     "Version for sdk"
     "2.0.0")
@@ -69,14 +65,19 @@ define_option_string(MILVUS_WITH_NLOHMANN_JSON  "nlohmann json from"    "module"
 define_option_string(MILVUS_WITH_GTEST          "Using GTest from"      "module" "package" "module")
 
 
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED on)
 set(BUILD_SCRIPTS_DIR ${PROJECT_SOURCE_DIR}/scripts)
-
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-if (BUILD_TEST)
-    if ( CMAKE_BUILD_TYPE STREQUAL "Debug" )
+
+# load third packages and milvus-proto
+include(ThirdPartyPackages)
+include(MilvusProtoGen)
+
+# add testing
+if (MILVUS_BUILD_TEST)
+    if ( "${CMAKE_BUILD_TYPE}" STREQUAL "Debug" AND MILVUS_BUILD_COVERAGE )
         # Set compile flag for code coverage.
         # Note: Only do this in Debug/unittest mode. Must do this before add_subdirectory(src).
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
@@ -88,16 +89,10 @@ endif ()
 add_subdirectory(src)
 add_subdirectory(examples)
 
-
-# lint verbose options
-if (NOT MILVUS_VERBOSE_LINT)
-    set(MILVUS_LINT_QUIET "--quiet")
-endif ()
-
 #
 # make lint
 #
-find_package(Python)
+find_package(Python2)
 
 if (NOT LINT_EXCLUSIONS_FILE)
     set(LINT_EXCLUSIONS_FILE ${BUILD_SCRIPTS_DIR}/lint_exclusions.txt)
@@ -108,7 +103,7 @@ if (NOT FORMAT_EXCLUSIONS_FILE)
 endif ()
 
 add_custom_target(lint
-        ${Python_EXECUTABLE}
+        ${Python2_EXECUTABLE}
         ${BUILD_SCRIPTS_DIR}/run_cpplint.py
         --cpplint_binary
         ${BUILD_SCRIPTS_DIR}/cpplint.py
@@ -125,7 +120,7 @@ add_custom_target(lint
 if (${CLANG_FORMAT_FOUND})
     # runs clang format and updates files in place.
     add_custom_target(clang-format
-            ${Python_EXECUTABLE}
+            ${Python2_EXECUTABLE}
             ${BUILD_SCRIPTS_DIR}/run_clang_format.py
             --clang_format_binary
             ${CLANG_FORMAT_BIN}
@@ -138,7 +133,7 @@ if (${CLANG_FORMAT_FOUND})
 
     # runs clang format and exits with a non-zero exit code if any files need to be reformatted
     add_custom_target(check-clang-format
-            ${Python_EXECUTABLE}
+            ${Python2_EXECUTABLE}
             ${BUILD_SCRIPTS_DIR}/run_clang_format.py
             --clang_format_binary
             ${CLANG_FORMAT_BIN}
@@ -155,7 +150,7 @@ endif ()
 if (${CLANG_TIDY_FOUND})
     # runs clang-tidy and attempts to fix any warning automatically
     add_custom_target(clang-tidy
-            ${Python_EXECUTABLE}
+            ${Python2_EXECUTABLE}
             ${BUILD_SCRIPTS_DIR}/run_clang_tidy.py
             --clang_tidy_binary
             ${CLANG_TIDY_BIN}
@@ -170,7 +165,7 @@ if (${CLANG_TIDY_FOUND})
 
     # runs clang-tidy and exits with a non-zero exit code if any errors are found.
     add_custom_target(check-clang-tidy
-            ${Python_EXECUTABLE}
+            ${Python2_EXECUTABLE}
             ${BUILD_SCRIPTS_DIR}/run_clang_tidy.py
             --clang_tidy_binary
             ${CLANG_TIDY_BIN}
@@ -184,4 +179,4 @@ if (${CLANG_TIDY_FOUND})
 endif ()
 
 # output config summary
-config_summary()
+config_summary()

Makefile

@@ -41,8 +41,9 @@ st:
 	@echo "System Testing with Milvus SDK"
 	@(env bash $(PWD)/scripts/build.sh -s)
 
-coverage: test st
+coverage:
 	@echo "Run code coverage ..."
+	@(env bash $(PWD)/scripts/build.sh -u -s -c)
 	@(env bash $(PWD)/scripts/coverage.sh)
 
 documentation:

cmake/ThirdPartyPackages.cmake

@@ -16,27 +16,29 @@
 
 include_guard(GLOBAL)
 
-
-# ----------------------------------------------------------------------
-# Needs threads
-set(THREADS_PREFER_PTHREAD_FLAG ON)
-find_package(Threads REQUIRED)
 include(CPM)
 
 # grpc
 if ("${MILVUS_WITH_GRPC}" STREQUAL "pakcage")
     find_package(grpc REQUIRED)
 else ()
+    if (WIN32)
+        set(OPENSSL_NO_ASM_TXT "YES")
+    else ()
+        set(OPENSSL_NO_ASM_TXT "NO")
+    endif ()
     CPMAddPackage(
         NAME grpc
         VERSION 1.49.1
         GITHUB_REPOSITORY grpc/grpc
+        EXCLUDE_FROM_ALL YES
         OPTIONS
             "gRPC_SSL_PROVIDER module"
             "gRPC_PROTOBUF_PROVIDER module"
             "gRPC_BUILD_TESTS OFF"
             "RE2_BUILD_TESTING OFF"
             "ABSL_PROPAGATE_CXX_STD ON"
+            "OPENSSL_NO_ASM ${OPENSSL_NO_ASM_TXT}"
     )
     if (grpc_ADDED)
         add_library(gRPC::grpc++ ALIAS grpc++)

examples/simple/main.cpp

@@ -81,7 +81,7 @@ main(int argc, char* argv[]) {
     std::vector<int8_t> insert_ages;
     std::vector<std::vector<float>> insert_vectors;
     std::default_random_engine ran(time(nullptr));
-    std::uniform_int_distribution<int8_t> int_gen(1, 100);
+    std::uniform_int_distribution<int> int_gen(1, 100);
     std::uniform_real_distribution<float> float_gen(0.0, 1.0);
     for (auto i = 0; i < row_count; ++i) {
         insert_ids.push_back(i);

scripts/build.sh

@@ -23,14 +23,15 @@ SYS_TEST="OFF"
 BUILD_TEST="OFF"
 MAKE_CLEAN="OFF"
 RUN_CPPLINT="OFF"
+BUILD_COVERAGE="OFF"
 MILVUS_SDK_VERSION=${MILVUS_SDK_VERSION:-2.0.0}
 
 JOBS="$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || echo 3)"
 if [ ${JOBS} -lt 3 ] ; then
     JOBS=3
 fi
 
-while getopts "t:v:ulrsph" arg; do
+while getopts "t:v:ulrcsph" arg; do
   case $arg in
   t)
     BUILD_TYPE=$OPTARG # BUILD_TYPE
@@ -52,6 +53,9 @@ while getopts "t:v:ulrsph" arg; do
     UNIT_TEST="ON"
     BUILD_TEST="ON"
     ;;
+  c)
+    BUILD_COVERAGE="ON"
+    ;;
   s)
     SYS_TEST="ON"
     BUILD_TEST="ON"
@@ -73,6 +77,7 @@ parameter:
 -u: build with unit testing(default: OFF)
 -r: clean before build
 -s: build with system testing(default: OFF)
+-c: build with coverage
 -p: build with production(-t RelWithDebInfo -r)
 -h: help
 
@@ -104,8 +109,8 @@ fi
 
 CMAKE_CMD="cmake \
 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
--DBUILD_TEST=${BUILD_TEST} \
--DMAKE_BUILD_ARGS=-j${JOBS}
+-DMILVUS_BUILD_TEST=${BUILD_TEST} \
+-DMILVUS_BUILD_COVERAGE=${BUILD_COVERAGE} \
 -DMILVUS_SDK_VERSION=${MILVUS_SDK_VERSION} \
 ../"
 echo ${CMAKE_CMD}
@@ -147,10 +152,11 @@ fi
 
 if [[ "${UNIT_TEST}" == "ON" ]]; then
   make -j ${JOBS}  || exit 1
-  make CTEST_OUTPUT_ON_FAILURE=1 test || exit 1
+  ./test/testing-ut || exit 1
+  ./test/testing-it || exit 1
 fi
 
 if [[ "${SYS_TEST}" == "ON" ]]; then
   make -j ${JOBS}  || exit 1
-  make CTEST_OUTPUT_ON_FAILURE=1 system-test || exit 1
+  ./test/testing-st || exit 1
 fi

src/impl/TypeUtils.cpp

@@ -23,11 +23,11 @@ operator==(const proto::schema::FieldData& lhs, const BoolFieldData& rhs) {
     if (lhs.field_name() != rhs.Name()) {
         return false;
     }
-    if (not lhs.has_scalars()) {
+    if (!lhs.has_scalars()) {
         return false;
     }
     const auto& scalars = lhs.scalars();
-    if (not scalars.has_bool_data()) {
+    if (!scalars.has_bool_data()) {
         return false;
     }
     const auto& scalars_data = scalars.bool_data().data();
@@ -42,11 +42,11 @@ operator==(const proto::schema::FieldData& lhs, const Int8FieldData& rhs) {
     if (lhs.field_name() != rhs.Name()) {
         return false;
     }
-    if (not lhs.has_scalars()) {
+    if (!lhs.has_scalars()) {
         return false;
     }
     const auto& scalars = lhs.scalars();
-    if (not scalars.has_int_data()) {
+    if (!scalars.has_int_data()) {
         return false;
     }
     const auto& scalars_data = scalars.int_data().data();
@@ -61,11 +61,11 @@ operator==(const proto::schema::FieldData& lhs, const Int16FieldData& rhs) {
     if (lhs.field_name() != rhs.Name()) {
         return false;
     }
-    if (not lhs.has_scalars()) {
+    if (!lhs.has_scalars()) {
         return false;
     }
     const auto& scalars = lhs.scalars();
-    if (not scalars.has_int_data()) {
+    if (!scalars.has_int_data()) {
         return false;
     }
     const auto& scalars_data = scalars.int_data().data();
@@ -80,11 +80,11 @@ operator==(const proto::schema::FieldData& lhs, const Int32FieldData& rhs) {
     if (lhs.field_name() != rhs.Name()) {
         return false;
     }
-    if (not lhs.has_scalars()) {
+    if (!lhs.has_scalars()) {
         return false;
     }
     const auto& scalars = lhs.scalars();
-    if (not scalars.has_int_data()) {
+    if (!scalars.has_int_data()) {
         return false;
     }
     const auto& scalars_data = scalars.int_data().data();
@@ -99,11 +99,11 @@ operator==(const proto::schema::FieldData& lhs, const Int64FieldData& rhs) {
     if (lhs.field_name() != rhs.Name()) {
         return false;
     }
-    if (not lhs.has_scalars()) {
+    if (!lhs.has_scalars()) {
         return false;
     }
     const auto& scalars = lhs.scalars();
-    if (not scalars.has_long_data()) {
+    if (!scalars.has_long_data()) {
         return false;
     }
     const auto& scalars_data = scalars.long_data().data();
@@ -118,11 +118,11 @@ operator==(const proto::schema::FieldData& lhs, const FloatFieldData& rhs) {
     if (lhs.field_name() != rhs.Name()) {
         return false;
     }
-    if (not lhs.has_scalars()) {
+    if (!lhs.has_scalars()) {
         return false;
     }
     const auto& scalars = lhs.scalars();
-    if (not scalars.has_float_data()) {
+    if (!scalars.has_float_data()) {
         return false;
     }
     const auto& scalars_data = scalars.float_data().data();
@@ -137,11 +137,11 @@ operator==(const proto::schema::FieldData& lhs, const DoubleFieldData& rhs) {
     if (lhs.field_name() != rhs.Name()) {
         return false;
     }
-    if (not lhs.has_scalars()) {
+    if (!lhs.has_scalars()) {
         return false;
     }
     const auto& scalars = lhs.scalars();
-    if (not scalars.has_double_data()) {
+    if (!scalars.has_double_data()) {
         return false;
     }
     const auto& scalars_data = scalars.double_data().data();
@@ -156,11 +156,11 @@ operator==(const proto::schema::FieldData& lhs, const VarCharFieldData& rhs) {
     if (lhs.field_name() != rhs.Name()) {
         return false;
     }
-    if (not lhs.has_scalars()) {
+    if (!lhs.has_scalars()) {
         return false;
     }
     const auto& scalars = lhs.scalars();
-    if (not scalars.has_string_data()) {
+    if (!scalars.has_string_data()) {
         return false;
     }
     const auto& scalars_data = scalars.string_data().data();
@@ -204,7 +204,7 @@ operator==(const proto::schema::FieldData& lhs, const FloatVecFieldData& rhs) {
     if (lhs.field_name() != rhs.Name()) {
         return false;
     }
-    if (not lhs.has_vectors()) {
+    if (!lhs.has_vectors()) {
         return false;
     }
     size_t dim = 0;