VDMS 2.7.0 Release (#168)

Author: cwlacewe

INSTALL.md

@@ -7,7 +7,8 @@ To install VDMS, we must install the necessary dependencies via apt, github, and
 ### Install Debian/Ubuntu Packages
 Here we will install the Debian/Ubuntu packages.
 ```bash
-sudo apt-get update
+sudo apt-get update -y  --fix-missing
+sudo apt-get upgrade -y
 sudo apt-get install -y --no-install-suggests --no-install-recommends \
     apt-transport-https autoconf automake bison build-essential bzip2 ca-certificates \
     curl ed flex g++-9 gcc-9 git gnupg-agent javacc libarchive-tools libatlas-base-dev \
@@ -55,7 +56,17 @@ alias python=/usr/bin/python3
 You can also install the coverage package if interested in running the Python unit tests.
 ```bash
 python3 -m pip install --upgrade pip
-python3 -m pip install --no-cache-dir "numpy>=1.25.1" "coverage>=7.2.7"
+python3 -m pip install --no-cache-dir "numpy>=1.26.0" "coverage>=7.3.1"
+```
+
+
+#### **Valijson v0.6**
+This is a headers-only library, no compilation/installation necessary.
+```bash
+VALIJSON_VERSION="v0.6"
+git clone --branch ${VALIJSON_VERSION} https://github.com/tristanpenman/valijson.git $VDMS_DEP_DIR/valijson
+cd $VDMS_DEP_DIR/valijson
+sudo cp -r include/* /usr/local/include/
 ```
 
 
@@ -71,36 +82,11 @@ sudo make install
 ```
 
 
-#### **Faiss v1.7.3**
-Install the Faiss library for similarity search.
-```bash
-FAISS_VERSION="v1.7.3"
-git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git $VDMS_DEP_DIR/faiss
-cd $VDMS_DEP_DIR/faiss
-mkdir build && cd build
-cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=/usr/bin/python3 ..
-make ${BUILD_THREADS}
-sudo make install
-```
-
-
-#### **FLINNG**
-Install the Filters to Identify Near-Neighbor Groups (FLINNG) library for similarity search.
-```bash
-git clone https://github.com/tonyzhang617/FLINNG.git $VDMS_DEP_DIR/FLINNG
-cd $VDMS_DEP_DIR/FLINNG
-mkdir build && cd build
-cmake ..
-make ${BUILD_THREADS}
-sudo make install
-```
-
-
 #### **Protobuf v24.2 (4.24.2)**
 Install Protobuf (C++ and Python) which requires GoogleTest and Abseil C++ as dependencies.
 ```bash
 PROTOBUF_VERSION="24.2"
-git clone -b v${PROTOBUF_VERSION} --recursive https://github.com/protocolbuffers/protobuf.git $VDMS_DEP_DIR/protobuf
+git clone -b v${PROTOBUF_VERSION} --recurse-submodules https://github.com/protocolbuffers/protobuf.git $VDMS_DEP_DIR/protobuf
 
 cd $VDMS_DEP_DIR/protobuf/third_party/googletest
 mkdir build && cd build
@@ -128,42 +114,31 @@ python3 -m pip install --no-cache-dir "protobuf==4.${PROTOBUF_VERSION}"
 ```
 
 
-#### **[OpenCV](https://opencv.org/) 4.5.5**
-Below are instructions for installing ***OpenCV v4.5.5***.
+#### **Faiss v1.7.3**
+Install the Faiss library for similarity search.
 ```bash
-OPENCV_VERSION="4.5.5"
-git clone --branch ${OPENCV_VERSION} https://github.com/opencv/opencv.git $VDMS_DEP_DIR/opencv
-cd $VDMS_DEP_DIR/opencv
+FAISS_VERSION="v1.7.3"
+git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git $VDMS_DEP_DIR/faiss
+cd $VDMS_DEP_DIR/faiss
 mkdir build && cd build
-cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF ..
+cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=/usr/bin/python3 ..
 make ${BUILD_THREADS}
 sudo make install
 ```
 
-**Note**: When using videos, and getting the following error: "Unable to stop the stream: Inappropriate ioctl for device", you may need to include more flags when compiling OpenCV. Follow these instructions ([source](https://stackoverflow.com/questions/41200201/opencv-unable-to-stop-the-stream-inappropriate-ioctl-for-device)):
-```bash
-apt-get install ffmpeg
-apt-get install libavcodec-dev libavformat-dev libavdevice-dev
 
-cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local \
-    -D WITH_FFMPEG=ON -D WITH_TBB=ON -D WITH_GTK=ON \
-    -D WITH_V4L=ON -D WITH_OPENGL=ON -D WITH_CUBLAS=ON \
-    -DWITH_QT=OFF -DCUDA_NVCC_FLAGS="-D_FORCE_INLINES" ..
+#### **FLINNG**
+Install the Filters to Identify Near-Neighbor Groups (FLINNG) library for similarity search.
+```bash
+git clone https://github.com/tonyzhang617/FLINNG.git $VDMS_DEP_DIR/FLINNG
+cd $VDMS_DEP_DIR/FLINNG
+mkdir build && cd build
+cmake ..
 make ${BUILD_THREADS}
 sudo make install
 ```
 
 
-#### **Valijson v0.6**
-This is a headers-only library, no compilation/installation necessary.
-```bash
-VALIJSON_VERSION="v0.6"
-git clone --branch ${VALIJSON_VERSION} https://github.com/tristanpenman/valijson.git $VDMS_DEP_DIR/valijson
-cd $VDMS_DEP_DIR/valijson
-sudo cp -r include/* /usr/local/include/
-```
-
-
 #### **[TileDB](https://tiledb.io/) 2.14.1**
 The directions below will help you install TileDB v2.14.1 from the source.
 You can also follow the directions listed [here](https://docs.tiledb.io/en/latest/installation.html).
@@ -191,28 +166,53 @@ cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTAL
 make ${BUILD_THREADS}
 sudo make install
 ```
+
+
+#### **[OpenCV](https://opencv.org/) 4.5.5**
+Below are instructions for installing ***OpenCV v4.5.5***.
+```bash
+OPENCV_VERSION="4.5.5"
+git clone --branch ${OPENCV_VERSION} https://github.com/opencv/opencv.git $VDMS_DEP_DIR/opencv
+cd $VDMS_DEP_DIR/opencv
+mkdir build && cd build
+cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF ..
+make ${BUILD_THREADS}
+sudo make install
+```
+
+**Note**: When using videos, and getting the following error: "Unable to stop the stream: Inappropriate ioctl for device", you may need to include more flags when compiling OpenCV. Follow these instructions ([source](https://stackoverflow.com/questions/41200201/opencv-unable-to-stop-the-stream-inappropriate-ioctl-for-device)):
+```bash
+sudo apt-get install -y ffmpeg
+sudo apt-get install -y libavdevice-dev
+
+cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local \
+    -D WITH_FFMPEG=ON -D WITH_TBB=ON -D WITH_GTK=ON \
+    -D WITH_V4L=ON -D WITH_OPENGL=ON -D WITH_CUBLAS=ON \
+    -DWITH_QT=OFF -DCUDA_NVCC_FLAGS="-D_FORCE_INLINES" ..
+make ${BUILD_THREADS}
+sudo make install
+```
 <br>
 
 ## Install VDMS
 This version of VDMS treats PMGD as a submodule so both libraries are compiled at one time. After entering the vdms directory, the command `git submodule update --init --recursive` will pull pmgd into the appropriate directory. Furthermore, Cmake is used to compile all directories.
 ```bash
-git clone -b develop https://github.com/IntelLabs/vdms.git
+git clone -b develop --recurse-submodules https://github.com/IntelLabs/vdms.git
 cd vdms
-git submodule update --init --recursive
 ```
 
 When compiling on a target without Optane persistent memory, use the following:
 ```bash
 mkdir build && cd build
 cmake ..
-make -j
+make ${BUILD_THREADS}
 cp ../config-vdms.json .
 ```
 
 When compiling on a target with Optane persistent memory, use the command set:
 ```bash
 mkdir build && cd build
 cmake -DCMAKE_CXX_FLAGS='-DPM' ..
-make -j
+make ${BUILD_THREADS}
 ```
 

client/python/setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="vdms",
-    version="0.0.19",
+    version="0.0.20",
     author="Chaunté W. Lacewell",
     author_email="[email protected]",
     description="VDMS Client Module",

client/python/vdms/vdms.py

@@ -42,6 +42,17 @@
 class vdms(object):
     def __init__(self):
         self.dataNotUsed = []
+        self.init_connection()
+        self.last_response = ""
+
+    def __del__(self):
+        self.conn.close()
+        self.connected = False
+
+    def init_connection(self):
+        if hasattr(self, "conn") and self.conn is not None:
+            self.conn.close()
+
         self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         self.conn.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
 
@@ -51,20 +62,29 @@ def __init__(self):
         # https://docs.python.org/dev/library/sys.html#sys.platform
         if sys.platform.startswith("linux"):
             self.conn.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
-
         self.connected = False
-        self.last_response = ""
-
-    def __del__(self):
-        self.conn.close()
 
     def connect(self, host="localhost", port=55555):
-        self.conn.connect((host, port))
-        self.connected = True
+        if self.connected is False:
+            self.init_connection()
+            self.conn.connect((host, port))
+            self.connected = True
+            return True
+        else:
+            print("Connection is already active")
+            return False
 
     def disconnect(self):
-        self.conn.close()
-        self.connected = False
+        if self.connected is True:
+            self.conn.close()
+            self.connected = False
+            return True
+        else:
+            print("There is not an active connection")
+            return False
+
+    def is_connected(self):
+        return self.connected
 
     # Recieves a json struct as a string
     def query(self, query, blob_array=[]):

config-vdms.json

@@ -6,7 +6,9 @@
     // "backup_path":"backups_test", // set this if you want different path to store the back up file
     "db_root_path": "db",
     "backup_flag" : "false",
-    "storage_type": "local", //local, aws, etc
+    "storage_type": "local", //local, aws
+    // use_endpoint: [true|false] in case of "storage_type" is equals to "aws", this key is used to specify whether it is going to use a "mocked" AWS connection
+    "use_endpoint": false,
     "bucket_name": "minio-bucket",
     "more-info": "github.com/IntelLabs/vdms"
 }