V2.6.0 (#165)

Author: cwlacewe

CMakeLists.txt

@@ -1,7 +1,9 @@
 cmake_minimum_required (VERSION 3.17)
+
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
 set(CMAKE_CXX_STANDARD 17)
 
+
 IF(CODE_COVERAGE)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -Wall -coverage -fprofile-arcs -ftest-coverage")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -Wall -coverage -fprofile-arcs -ftest-coverage")
@@ -12,16 +14,32 @@ project(vdms_application)
 add_compile_options(-g -fPIC -std=c++17)
 
 find_package( OpenCV REQUIRED )
-find_package(Protobuf REQUIRED)
+find_package(Protobuf CONFIG REQUIRED)
 find_package( CURL REQUIRED )
 find_package(AWSSDK REQUIRED COMPONENTS core s3)
 
 include_directories(${Protobuf_INCLUDE_DIRS})
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
-execute_process(COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/createApiString.py ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/api_schema.json ${CMAKE_CURRENT_BINARY_DIR}/APISchema.h)
-protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS utils/src/protobuf/partitionerMessages.proto utils/src/protobuf/pmgdMessages.proto utils/src/protobuf/queryMessage.proto)
-add_library(vdms_protobuf SHARED ${PROTO_SRCS} ${PROTO_HDRS})
+execute_process(COMMAND python3
+  ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/createApiString.py
+  ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/api_schema/api_schema.json
+  ${CMAKE_CURRENT_BINARY_DIR}/APISchema.h
+)
+add_library(vdms_protobuf OBJECT
+  ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf/partitionerMessages.proto
+  ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf/pmgdMessages.proto
+  ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf/queryMessage.proto
+)
+target_link_libraries(vdms_protobuf PUBLIC protobuf::libprotobuf)
+set(PROTO_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
+target_include_directories(vdms_protobuf PUBLIC "$<BUILD_INTERFACE:${PROTO_BINARY_DIR}>")
+protobuf_generate(
+  LANGUAGE cpp
+  TARGET vdms_protobuf
+  IMPORT_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf"
+  PROTOC_OUT_DIR "${PROTO_BINARY_DIR}"
+)
 
 option(CLIENT "Built client library." OFF)
 if (CLIENT)
@@ -54,7 +72,9 @@ else()
     src/PMGDIterators.cc
     src/PMGDQuery.cc
     src/PMGDQueryHandler.cc
-    src/QueryHandler.cc
+    src/QueryHandlerExample.cc
+    src/QueryHandlerBase.cc
+    src/QueryHandlerPMGD.cc
     src/QueryMessage.cc
     src/RSCommand.cc
     src/SearchExpression.cc
@@ -63,10 +83,9 @@ else()
     src/VideoCommand.cc
     src/AutoDeleteNode.cc
     src/ImageLoop.cc
-    ${PROTO_SRCS} ${PROTO_HDRS}
-)
+    src/VideoLoop.cc
+  )
   target_link_libraries(dms vcl pmgd pmgd-util protobuf tbb tiledb vdms-utils pthread -lcurl -lzmq ${AWSSDK_LINK_LIBRARIES})
-
   add_executable(vdms src/vdms.cc)
   target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS} ${AWSSDK_LINK_LIBRARIES})
 endif ()

INSTALL.md

@@ -2,20 +2,20 @@
 Here is the detailed process of installation of VDMS dependencies.
 
 ## Dependencies
-To install VDMS, we must install the necessary dependencies via apt, github, and pip.
+To install VDMS, we must install the necessary dependencies via apt, github, and pip (Python 3.9+).
 
-### Install Debian Packages
-Here we will install the Debian and Python3 packages.
+### Install Debian/Ubuntu Packages
+Here we will install the Debian/Ubuntu packages.
 ```bash
 sudo apt-get update
 sudo apt-get install -y --no-install-suggests --no-install-recommends \
     apt-transport-https autoconf automake bison build-essential bzip2 ca-certificates \
     curl ed flex g++-9 gcc-9 git gnupg-agent javacc libarchive-tools libatlas-base-dev \
     libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev libc-ares-dev libcurl4-openssl-dev \
-    libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev libgtk-3-dev libgtk2.0-dev \
-    libhdf5-dev libjpeg-dev libjpeg62-turbo-dev libjsoncpp-dev libleveldb-dev liblmdb-dev \
+    libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtk-3-dev libgtk2.0-dev \
+    libhdf5-dev libjpeg-dev libjsoncpp-dev libleveldb-dev liblmdb-dev \
     liblz4-dev libopenblas-dev libopenmpi-dev libpng-dev librdkafka-dev libsnappy-dev libssl-dev \
-    libswscale-dev libtbb-dev libtbb2 libtiff-dev libtiff5-dev libtool libzmq3-dev mpich \
+    libswscale-dev libtbb-dev libtbb2 libtiff-dev libtiff5-dev libtool libzmq3-dev linux-libc-dev mpich \
     openjdk-11-jdk-headless pkg-config procps python3-dev python3-pip software-properties-common \
     swig unzip uuid-dev
 ```
@@ -25,81 +25,110 @@ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 1
 update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 1
 ```
 
+#### **Install JPEG package**
+Please install the JPEG package based on the OS platform being used:
+* ***Debian 10+:*** `sudo apt-get install -y libjpeg62-turbo-dev`
+* ***Ubuntu 20.04+:*** `sudo apt-get install -y libjpeg8-dev`
+<br>
+
 ### Install Remaining Dependencies
 Here we assume `$VDMS_DEP_DIR` is the directory for installing additional dependencies.
 This directory is user-defined but here we use `/dependencies`.
 These instructions assume you have full permissions to your system.
-If not running as root, add `sudo` where necessary.
+***NOTE:*** If running as ***root***, remove `sudo` where applicable.
 ```bash
 VDMS_DEP_DIR=/dependencies  # Set to any directory
 BUILD_THREADS="-j`nproc`"
 mkdir -p $VDMS_DEP_DIR
 ```
 
+
 #### Python3 Packages
-Here we will install the necessary Python3 packages Numpy and Protobuf 3.20.3.
+Here we will install the necessary Python 3.9+ packages Numpy and Protobuf v24.2.
+It is expected that you have Python3.9 or higher installed on your system.
+All python calls will use Python3.9+; therefore you may find it convenient to set alias for python.
+```bash
+alias python=/usr/bin/python3
+```
+***NOTE:*** If multiple versions of Python 3 are present on your system, verify you are using Python3.9 or higher. You can specify the specific verison in above command and also set the following with your specific version: `alias python3=/usr/bin/python3.x`.
+
 You can also install the coverage package if interested in running the Python unit tests.
 ```bash
-PROTOBUF_VERSION="3.20.3"
-pip3 install --no-cache-dir "numpy>=1.25.1" "protobuf==${PROTOBUF_VERSION}" "coverage>=7.2.7"
+python3 -m pip install --upgrade pip
+python3 -m pip install --no-cache-dir "numpy>=1.25.1" "coverage>=7.2.7"
 ```
 
 
-#### CMAKE v3.26.4
-VDMS requires CMake v3.21+.  Here we install CMake v3.26.4.
+#### **CMAKE v3.27.2**
+VDMS requires CMake v3.21+.  Here we install CMake v3.27.2.
 ```bash
-CMAKE_VERSION="v3.26.4"
+CMAKE_VERSION="v3.27.2"
 git clone --branch ${CMAKE_VERSION} https://github.com/Kitware/CMake.git $VDMS_DEP_DIR/CMake
 cd $VDMS_DEP_DIR/CMake
 ./bootstrap
 make ${BUILD_THREADS}
-make install
+sudo make install
 ```
 
-### gtest
-Unfortunately apt doesn't build gtest so you need to do the following:
-```bash
-cd /usr/src/gtest/
-cmake .
-make ${BUILD_THREADS}
-mv lib/libgtest* /usr/lib
-```
 
-### Faiss v1.7.3
+#### **Faiss v1.7.3**
+Install the Faiss library for similarity search.
 ```bash
 FAISS_VERSION="v1.7.3"
 git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git $VDMS_DEP_DIR/faiss
 cd $VDMS_DEP_DIR/faiss
 mkdir build && cd build
-cmake -DFAISS_ENABLE_GPU=OFF ..
+cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=/usr/bin/python3 ..
 make ${BUILD_THREADS}
-make install
+sudo make install
 ```
 
-### FLINNG
+
+#### **FLINNG**
+Install the Filters to Identify Near-Neighbor Groups (FLINNG) library for similarity search.
 ```bash
 git clone https://github.com/tonyzhang617/FLINNG.git $VDMS_DEP_DIR/FLINNG
 cd $VDMS_DEP_DIR/FLINNG
 mkdir build && cd build
 cmake ..
 make ${BUILD_THREADS}
-make install
+sudo make install
 ```
 
-### Protobuf 3.20.3
+
+#### **Protobuf v24.2 (4.24.2)**
+Install Protobuf (C++ and Python) which requires GoogleTest and Abseil C++ as dependencies.
 ```bash
-PROTOBUF_VERSION="3.20.3"
-curl -L -o ${VDMS_DEP_DIR}/${PROTOBUF_VERSION}.tar.gz https://github.com/protocolbuffers/protobuf/archive/refs/tags/v${PROTOBUF_VERSION}.tar.gz
-cd ${VDMS_DEP_DIR} && tar -xvf ${PROTOBUF_VERSION}.tar.gz
-cd protobuf-${PROTOBUF_VERSION}
-./autogen.sh
-./configure
+PROTOBUF_VERSION="24.2"
+git clone -b v${PROTOBUF_VERSION} --recursive https://github.com/protocolbuffers/protobuf.git $VDMS_DEP_DIR/protobuf
+
+cd $VDMS_DEP_DIR/protobuf/third_party/googletest
+mkdir build && cd build
+cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \
+    -DBUILD_GMOCK=ON -DCMAKE_CXX_STANDARD=17 ..
+make ${BUILD_THREADS}
+sudo make install
+sudo ldconfig
+
+cd $VDMS_DEP_DIR/protobuf/third_party/abseil-cpp
+mkdir build && cd build
+cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ \
+    -DABSL_BUILD_TESTING=ON -DABSL_ENABLE_INSTALL=ON -DABSL_USE_EXTERNAL_GOOGLETEST=ON \
+    -DABSL_FIND_GOOGLETEST=ON -DCMAKE_CXX_STANDARD=17 ..
+make ${BUILD_THREADS}
+sudo make install
+
+cd $VDMS_DEP_DIR/protobuf
+cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_CXX_STANDARD=17 \
+    -Dprotobuf_ABSL_PROVIDER=package -DCMAKE_PREFIX_PATH=/usr/local .
 make ${BUILD_THREADS}
-make install
-ldconfig
+sudo make install
+
+python3 -m pip install --no-cache-dir "protobuf==4.${PROTOBUF_VERSION}"
 ```
 
-### [OpenCV](https://opencv.org/) 4.5.5
+
+#### **[OpenCV](https://opencv.org/) 4.5.5**
 Below are instructions for installing ***OpenCV v4.5.5***.
 ```bash
 OPENCV_VERSION="4.5.5"
@@ -108,7 +137,7 @@ cd $VDMS_DEP_DIR/opencv
 mkdir build && cd build
 cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF ..
 make ${BUILD_THREADS}
-make install
+sudo make install
 ```
 
 **Note**: When using videos, and getting the following error: "Unable to stop the stream: Inappropriate ioctl for device", you may need to include more flags when compiling OpenCV. Follow these instructions ([source](https://stackoverflow.com/questions/41200201/opencv-unable-to-stop-the-stream-inappropriate-ioctl-for-device)):
@@ -121,20 +150,21 @@ cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF -D CMAKE_BUILD_TYPE=RELEASE -D
     -D WITH_V4L=ON -D WITH_OPENGL=ON -D WITH_CUBLAS=ON \
     -DWITH_QT=OFF -DCUDA_NVCC_FLAGS="-D_FORCE_INLINES" ..
 make ${BUILD_THREADS}
-make install
+sudo make install
 ```
 
-### Valijson v0.6
-This is a headers-only library, no compilation/installation necessary
+
+#### **Valijson v0.6**
+This is a headers-only library, no compilation/installation necessary.
 ```bash
 VALIJSON_VERSION="v0.6"
 git clone --branch ${VALIJSON_VERSION} https://github.com/tristanpenman/valijson.git $VDMS_DEP_DIR/valijson
 cd $VDMS_DEP_DIR/valijson
-cp -r include/* /usr/local/include/
+sudo cp -r include/* /usr/local/include/
 ```
 
 
-### [TileDB](https://tiledb.io/) 2.14.1
+#### **[TileDB](https://tiledb.io/) 2.14.1**
 The directions below will help you install TileDB v2.14.1 from the source.
 You can also follow the directions listed [here](https://docs.tiledb.io/en/latest/installation.html).
 ```bash
@@ -146,19 +176,22 @@ cd TileDB-${TILEDB_VERSION}
 mkdir build && cd build
 ../bootstrap --prefix=/usr/local/
 make ${BUILD_THREADS}
-make install-tiledb
+sudo make install-tiledb
 ```
 
-### AWS SDK CPP 1.11.0
+
+#### **AWS SDK CPP 1.11.0**
+Use the following instructions to install AWS SDK for C++.
 ```bash
 AWS_SDK_VERSION="1.11.0"
 git clone -b ${AWS_SDK_VERSION} --recurse-submodules https://github.com/aws/aws-sdk-cpp ${VDMS_DEP_DIR}/aws-sdk-cpp
 mkdir -p ${VDMS_DEP_DIR}/aws-sdk-cpp/build
 cd ${VDMS_DEP_DIR}/aws-sdk-cpp/build
 cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ -DBUILD_ONLY="s3" -DCUSTOM_MEMORY_MANAGEMENT=OFF
 make ${BUILD_THREADS}
-make install
+sudo make install
 ```
+<br>
 
 ## Install VDMS
 This version of VDMS treats PMGD as a submodule so both libraries are compiled at one time. After entering the vdms directory, the command `git submodule update --init --recursive` will pull pmgd into the appropriate directory. Furthermore, Cmake is used to compile all directories.