Release v2.4.0 (#156)

* Add generic AddBlob command
* Correct zlib and tiledb paths
* Add CSV CPP Client Plugin
* Add files to update pypi package
Co-authored-by: Ragaad <[email protected]>

Author: cwlacewe

.gitignore

@@ -1,6 +1,7 @@
 build/
 db/
 *.gcov
+**/__pycache__/
 
 # VS Code Specific
 .devcontainer

CMakeLists.txt

@@ -40,10 +40,30 @@ else()
 
   link_directories(/usr/local/lib /usr/lib/x86_64-linux-gnu/)
   include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl /usr/include ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf)
-  add_library(dms SHARED src/BoundingBoxCommand.cc src/CommunicationManager.cc src/DescriptorsCommand.cc src/DescriptorsManager.cc src/ExceptionsCommand.cc src/ImageCommand.cc src/PMGDIterators.cc src/PMGDQuery.cc src/PMGDQueryHandler.cc src/QueryHandler.cc src/QueryMessage.cc src/RSCommand.cc src/SearchExpression.cc src/Server.cc src/VDMSConfig.cc src/VideoCommand.cc src/AutoDeleteNode.cc ${PROTO_SRCS} ${PROTO_HDRS})
+  add_library(dms SHARED
+                        src/BoundingBoxCommand.cc
+                        src/BlobCommand.cc
+                        src/CommunicationManager.cc
+                        src/DescriptorsCommand.cc
+                        src/DescriptorsManager.cc
+                        src/ExceptionsCommand.cc
+                        src/ImageCommand.cc
+                        src/PMGDIterators.cc
+                        src/PMGDQuery.cc
+                        src/PMGDQueryHandler.cc
+                        src/QueryHandler.cc
+                        src/QueryMessage.cc
+                        src/RSCommand.cc
+                        src/SearchExpression.cc
+                        src/Server.cc
+                        src/VDMSConfig.cc
+                        src/VideoCommand.cc
+                        src/AutoDeleteNode.cc
+                        ${PROTO_SRCS} ${PROTO_HDRS}
+                        )
   target_link_libraries(dms vcl pmgd pmgd-util protobuf vdms-utils pthread)
 
 
   add_executable(vdms src/vdms.cc)
   target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS})
-endif ()  
+endif ()

INSTALL.md

@@ -2,126 +2,117 @@
 Here is the detailed process of installation of VDMS dependencies.
 
 ## Dependencies
-Here we will install the Ubuntu 20.04 packages.
+Here we will install the Ubuntu 20.04 and Python3 packages. We assume `python`/`pip` is an alias for `python3`/`pip3`. If your system has both  Python 2 and Python 3, please replace all pip and python commands with pip3 and python3, respectively.
 ```bash
-apt-get update
-apt-get -y install software-properties-common
-add-apt-repository "deb http://security.ubuntu.com/ubuntu focal-security main"
-apt-get -y install apt-transport-https autoconf automake bison build-essential \
-    bzip2 ca-certificates curl ed flex g++ git gnupg-agent javacc libarchive-tools \
+sudo apt-get update
+sudo apt-get -y install --no-install-recommends software-properties-common
+sudo add-apt-repository "deb http://security.ubuntu.com/ubuntu focal-security main"
+sudo apt-get -y install --no-install-recommends apt-transport-https autoconf automake bison build-essential \
+    bzip2 ca-certificates curl=7.68.0-1ubuntu2.18 ed flex g++ git gnupg-agent javacc libarchive-tools \
     libatlas-base-dev libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev \
     libc-ares-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtest-dev \
     libgtk-3-dev libgtk2.0-dev libhdf5-serial-dev libjpeg-dev libjpeg8-dev libjsoncpp-dev \
     libleveldb-dev liblmdb-dev liblz4-dev libopenblas-dev libopenmpi-dev \
     libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev \
-    libtbb2 libtiff-dev libtiff5-dev libtool maven mpich openjdk-11-jdk-headless \
-    pkg-config python python-dev python3-pip unzip wget
-pip3 install numpy
+    libtbb2 libtiff-dev libtiff5-dev libtool mpich openjdk-11-jdk-headless \
+    pkg-config python3-dev python3-pip unzip
+pip install --no-cache-dir "numpy>=1.23.2" "setuptools>=65.5.1"
 ```
 ### Clone/Download Dependencies
-Here we clone the repositories for grpc v1.40.0, libpng12, Swig v4.0.2, OpenCV 4.5.3, Valijson v0.6, CMake v3.21.2, Faiss v1.7.1, and FLINNG. Then download necesarry files for zlib v1.2.12, Json-simple v1.1.1, and TileDB v1.3.1.
-Here we assume `/` is the working directory. This is important when installing the dependencies.
+Here we clone the repositories for grpc v1.40.0, libpng12, Swig v4.0.2, OpenCV 4.5.3, Valijson v0.6, CMake v3.21.2, Faiss v1.7.1, and FLINNG. Then download necesarry files for zlib v1.2.13, Json-simple v1.1.1, and TileDB v1.3.1.
+Here we assume `$VDMS_DEP_DIR` is the working directory for installing dependencies and `python` is Python 3.
 ```bash
-git clone --branch v1.40.0 https://github.com/grpc/grpc.git && \
-git clone --branch v4.0.2 https://github.com/swig/swig.git && \
-git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \
-git clone --branch v0.6 https://github.com/tristanpenman/valijson.git && \
+cd $VDMS_DEP_DIR
 git clone --branch v3.21.2 https://github.com/Kitware/CMake.git && \
+git clone --branch v4.0.2 https://github.com/swig/swig.git && \
 git clone --branch v1.7.1 https://github.com/facebookresearch/faiss.git && \
-git clone https://github.com/tonyzhang617/FLINNG.git
+git clone https://github.com/tonyzhang617/FLINNG.git && \
+git clone --recurse-submodules -b v1.40.0 https://github.com/grpc/grpc.git && \
+git clone --branch 4.5.3 https://github.com/opencv/opencv.git && \
+git clone --branch v0.6 https://github.com/tristanpenman/valijson.git
 
-curl http://zlib.net/zlib-1.2.12.tar.gz -o zlib-1.2.12.tar.gz && \
-curl https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar \
-    -o /usr/share/java/json-simple-1.1.1.jar && \
-wget https://github.com/TileDB-Inc/TileDB/archive/1.3.1.tar.gz
+sudo curl -L -o /usr/share/java/json-simple-1.1.1.jar https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/json-simple/json-simple-1.1.1.jar && \
+sudo curl -L -o 1.3.1.tar.gz https://github.com/TileDB-Inc/TileDB/archive/refs/tags/1.3.1.tar.gz && \
+sudo curl -L -o zlib-1.2.13.tar.gz http://zlib.net/zlib-1.2.13.tar.gz
 ```
 
 ### Install Dependencies
 These instructions assume you have full permissions to your system.
-If needed, use `sudo` where necessary.
+If running as root, remove `sudo` where necessary.
+
 #### CMAKE
 ```bash
-cd /CMake && ./bootstrap
-make -j && make install
+cd $VDMS_DEP_DIR/CMake && ./bootstrap
+make -j && sudo make install
 ```
 
 ### Swig
 ```bash
-cd /swig
+cd $VDMS_DEP_DIR/swig
 ./autogen.sh && ./configure
-make -j && make install
+make -j && sudo make install
 ```
 
 ### Faiss
 ```bash
-cd /faiss
+cd $VDMS_DEP_DIR/faiss
 mkdir build && cd build
 cmake -DFAISS_ENABLE_GPU=OFF ..
-make -j && make install
+make -j && sudo make install
 ```
 
 ### FLINNG
 ```bash
-cd /FLINNG
+cd $VDMS_DEP_DIR/FLINNG
 mkdir build && cd build
 cmake ..
-make -j && make install
+make -j && sudo make install
 ```
 
 ### grpc
 ```bash
-cd /grpc && git submodule update --init --recursive
-cd third_party/protobuf/cmake && mkdir build && cd build
+cd $VDMS_DEP_DIR/grpc
+pip install --no-cache-dir -r requirements.txt
+GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install --no-cache-dir .
+
+cd tools/distrib/python/grpcio_tools
+python ../make_grpcio_tools.py
+GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install --no-cache-dir .
+
+cd $VDMS_DEP_DIR/grpc/third_party/zlib/ && mkdir build && cd build
 cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ..
-make -j && make install
+make -j && sudo make install
+
+cd $VDMS_DEP_DIR/grpc/third_party/protobuf/cmake
+mkdir build && cd build
+cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE -Dprotobuf_BUILD_TESTS=OFF ..
+make -j && sudo make install
 
 cd ../../../abseil-cpp && mkdir build && cd build
 cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ..
-make -j && make install
+make -j && sudo make install
 
 cd ../../re2/ && mkdir build && cd build
 cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ..
-make -j && make install
+make -j && sudo make install
 
-cd ../../zlib/ && mkdir build && cd build
-cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ..
-make -j && make install
-
-cd /grpc/cmake && mkdir build && cd build
+cd $VDMS_DEP_DIR/grpc/cmake && mkdir build && cd build
 cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_ABSL_PROVIDER=package \
     -DgRPC_CARES_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package \
     -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package \
     -DgRPC_ZLIB_PROVIDER=package -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../..
-make -j && make install
-```
-
-### Zlib
-```bash
-cd / && gunzip zlib-1.2.12.tar.gz && tar -xvf zlib-1.2.12.tar
-cd zlib-1.2.12 && ./configure
-make -j && make install
-cd / && rm -rf zlib-1.2.12.tar zlib-1.2.12
-```
-
-### gtest
-Unfortunately apt doesn't build gtest;
-you need to do the following steps to get it to work correctly:
-```bash
-cd /usr/src/gtest/
-cmake .
-make -j
-mv lib/libgtest* /usr/lib
+make -j && sudo make install
 ```
 
 ### [OpenCV](https://opencv.org/)
 
 Below are instructions for installing ***OpenCV v4.5.3***. It may also work with newer versions of OpenCV.
 ```bash
-cd /opencv
+cd $VDMS_DEP_DIR/opencv
 mkdir build && cd build
-cmake -DBUILD_PERF_TESTS=OFF -DBUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local ..
+cmake -DBUILD_PERF_TESTS=OFF -DBUILD_TESTS=OFF ..
 make -j
-make install
+sudo make install
 ```
 
 **Note**: When using videos, and getting the following error: "Unable to stop the stream: Inappropriate ioctl for device", you may need to include more flags when compiling OpenCV. Follow these instructions ([source](https://stackoverflow.com/questions/41200201/opencv-unable-to-stop-the-stream-inappropriate-ioctl-for-device)):
@@ -137,76 +128,61 @@ make -j
 make install
 ```
 
+### Zlib
+```bash
+cd $VDMS_DEP_DIR && tar -xvzf zlib-1.2.13.tar.gz
+cd zlib-1.2.13 && ./configure
+make -j && sudo make install
+```
+
 ### [TileDB](https://tiledb.io/)
 VDMS works with ***TileDB v1.3.1.***<br>
 The directions below will help you install TileDB v1.3.1 from the source.
 You can also follow the directions listed
 [here](https://docs.tiledb.io/en/latest/installation.html).
 ```bash
-cd / && tar xf 1.3.1.tar.gz && rm 1.3.1.tar.gz
+cd $VDMS_DEP_DIR && tar -xvf 1.3.1.tar.gz
 cd TileDB-1.3.1 && mkdir build && cd build
 ../bootstrap --prefix=/usr/local/
-make -j && make install-tiledb
-rm -rf /TileDB-1.3.1
+make -j && sudo make install-tiledb
 ```
 
-### Maven
+### gtest
+Unfortunately apt doesn't build gtest;
+you need to do the following steps to get it to work correctly:
 ```bash
-ln -s /grpc/third_party/protobuf/cmake/build/protoc grpc/third_party/protobuf/src/protoc
-cd /grpc/third_party/protobuf/java/core
-mvn package
-cp target/protobuf-java-3.13.0.jar /usr/share/java/protobuf.jar
-```
-
-You may need to change proxy setting for Maven if you are behind a proxy like this example.
-Add setting.xml file to ~/.m2 folder
-```
-<proxies>
-    <proxy>
-    <id>optional</id>
-        <!-- <active>true</active> -->
-        <protocol>https</protocol>
-        <!--<username>proxyuser</username>
-        <password>proxypass</password>-->
-        <host>prox-address</host>
-        <port>proxy-port</port>
-        <nonProxyHosts></nonProxyHosts>
-    </proxy>
-</proxies>
+cd /usr/src/gtest/
+sudo cmake .
+sudo make -j
+sudo mv lib/libgtest* /usr/lib
 ```
 
 ### Valijson
 This is a headers-only library, no compilation/installation necessary
 ```bash
-cd /valijson
-cp -r include/* /usr/local/include
+cd $VDMS_DEP_DIR/valijson
+sudo cp -r include/* /usr/local/include
 ```
 
 ## Install VDMS
+This version of VDMS treats PMGD as a submodule so both libraries are compiled at one time. After entering the vdms directory, the command `git submodule update --init --recursive` will pull pmgd into the appropriate directory. Furthermore, Cmake is used to compile all directories.
 ```bash
 git clone https://github.com/IntelLabs/vdms.git
 cd vdms && git checkout develop
 git submodule update --init --recursive
+```
+
+When compiling on a target without Optane persistent memory, use the following:
+```bash
 mkdir build && cd build
 cmake ..
 make -j
 cp ../config-vdms.json .
 ```
 
-
-### Compilation
-This version of VDMS treats PMGD as a submodule so both libraries are compiled at one time. After entering the vdms directory, the command `git submodule update --init --recursive` will pull pmgd into the appropriate directory. Furthermore, Cmake is used to compile all directories.
-
 When compiling on a target with Optane persistent memory, use the command set:
 ```bash
 mkdir build && cd build
 cmake -DCMAKE_CXX_FLAGS='-DPM' ..
 make -j
 ```
-
-For systems without Optane, use the command set:
-```bash
-mkdir build && cd build
-cmake ..
-make -j<number of threads to use for compiling>
-```

Security.md

@@ -1,6 +1,5 @@
-# Security Policy
-Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. 
-
-## Reporting a Vulnerability
-Please report any security vulnerabilities in this project [utilizing the guidelines here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html).
-
+# Security Policy
+Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation.
+
+## Reporting a Vulnerability
+Please report any security vulnerabilities in this project [utilizing the guidelines here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html).