Release v2.11.0 (#280)

Author: cwlacewe

.clang-format-ignore

@@ -0,0 +1,2 @@
+./build/*
+./src/pmgd/*

.github/CONTRIBUTING.md

@@ -0,0 +1,5 @@
+# Contributing to VDMS
+Thank you for even being interested in contributing to VDMS.
+As an open-source project in a rapidly developing field, we are extremely open to contributions, whether they involve new features, improved infrastructure, better documentation, or bug fixes.
+
+To learn how to contribute to VDMS, please follow the [Developer Guide](https://github.com/IntelLabs/vdms/wiki/Developer-Guide).

.github/coverage/cpp.develop.coverage_report.txt

@@ -1 +1 @@
-63.8504
+68.9262

.github/coverage/cpp.develop.coverage_value.txt

@@ -1,6 +1,6 @@
 Name                                   Stmts   Miss  Cover   Missing
 --------------------------------------------------------------------
 /vdms/client/python/vdms/__init__.py       2      0   100%
-/vdms/client/python/vdms/vdms.py          99      2    98%   152, 167
+/vdms/client/python/vdms/vdms.py          95      2    98%   148, 163
 --------------------------------------------------------------------
-TOTAL                                    101      2    98%
+TOTAL                                     97      2    98%

.github/coverage/python.develop.coverage_report.txt

@@ -1 +1 @@
-98.02
+97.94

.github/coverage/python.develop.coverage_value.txt

@@ -1,22 +1,24 @@
-blinker==1.8.2
+blinker==1.9.0
 cffi==1.17.1
-click==8.1.7
-colorlog==6.8.2
-coverage==7.6.2
-cryptography==43.0.1
-Flask==3.0.3
-gcovr==8.0
+click==8.1.8
+colorlog==6.9.0
+coverage==7.8.0
+cryptography==44.0.2
+Flask==3.1.0
+gcovr==8.3
 imutils==0.5.4
 itsdangerous==2.2.0
-Jinja2==3.1.4
-lxml==5.3.0
-MarkupSafe==3.0.1
+Jinja2==3.1.6
+lxml==5.3.2
+MarkupSafe==3.0.2
 numpy==1.26.4
-opencv-python-headless==4.9.0.80
+opencv-python-headless==4.11.0.86
+pillow==11.1.0
 protobuf==4.24.2
 pycparser==2.22
-Pygments==2.18.0
-pyzmq==26.0.3
-scipy==1.14.1
+Pygments==2.19.1
+pyzmq==26.4.0
+scipy==1.15.2
 sk-video==1.1.10
-Werkzeug==3.0.4
+Werkzeug==3.1.3
+zmq==0.0.0

.github/requirements.txt

@@ -15,6 +15,7 @@ ARG NEO4J_PASS=""
 ARG NEO4J_ENDPOINT=""
 ARG AWS_API_PORT=9000
 ARG AWS_CONSOLE_PORT=9001
+ARG USE_K8S="OFF"
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV DEBCONF_NOWARNINGS="yes"
@@ -32,6 +33,12 @@ ENV NEO4J_ENDPOINT="${NEO4J_ENDPOINT}"
 ENV AWS_API_PORT="${AWS_API_PORT}"
 ENV AWS_CONSOLE_PORT="${AWS_CONSOLE_PORT}"
 
+# Convert ARG value to uppercase and set ENV
+# hadolint ignore=SC3059
+RUN export USE_K8S=$(echo "${USE_K8S^^}") && \
+    echo "export USE_K8S=${USE_K8S}" >> /etc/profile.d/envvars.sh
+ENV USE_K8S="${USE_K8S}"
+
 ############################################################
 # BUILD DEPENDENCIES
 FROM base AS build
@@ -41,15 +48,15 @@ FROM base AS build
 RUN apt-get update -y && apt-get upgrade -y && \
     apt-get install -o 'Acquire::Retries=3' -y --no-install-suggests \
         --no-install-recommends --fix-broken --fix-missing \
-        apt-transport-https automake bison build-essential bzip2 ca-certificates \
+        apt-transport-https automake bazel-bootstrap bison build-essential bzip2 ca-certificates \
         cppzmq-dev curl ed flex g++ gcc git gnupg-agent javacc libarchive-tools libatlas-base-dev \
-        libavcodec-dev libavformat-dev libavutil-dev libboost-all-dev libbz2-dev libc-ares-dev \
+        libavcodec-dev libavformat-dev libavutil-dev libbison-dev libboost-all-dev libbz2-dev libc-ares-dev \
         libcurl4-openssl-dev libdc1394-dev libgflags-dev libgoogle-glog-dev \
         libgtk-3-dev libgtk2.0-dev libhdf5-dev libjpeg-dev libjpeg62-turbo-dev libjsoncpp-dev \
         libleveldb-dev liblmdb-dev liblz4-dev libncurses5-dev libopenblas-dev libopenmpi-dev \
         libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev libtbbmalloc2 \
-        libtiff-dev libtiff5-dev libtool linux-libc-dev mpich openjdk-17-jdk-headless \
-        pkg-config procps software-properties-common swig unzip uuid-dev && \
+        libtiff-dev libtiff5-dev libtool libwebsockets-dev linux-libc-dev mpich openjdk-17-jdk-headless \
+        pkg-config procps software-properties-common swig uncrustify unzip uuid-dev && \
     apt-get --purge remove -y python3.11 && apt-get autoremove -y && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
@@ -69,7 +76,7 @@ WORKDIR /dependencies
 ENV AUTOCONF_VERSION="2.71" \
     AWS_SDK_VERSION="1.11.336" \
     CMAKE_VERSION="v3.28.5"  \
-    FAISS_VERSION="v1.7.4" \
+    FAISS_VERSION="v1.9.0" \
     LIBEDIT_VERSION="20230828-3.1" \
     OPENCV_VERSION="4.9.0" \
     PEG_VERSION="0.1.19" \
@@ -163,6 +170,31 @@ RUN curl -L -O https://github.com/gpakosz/peg/releases/download/${PEG_VERSION}/p
         ./configure --disable-werror --prefix=/opt/dist/usr && \
         make clean check && make install -w --debug
 
+# FOR KUBERNETES ORCHESTRATION
+# hadolint ignore=DL3003,SC2086
+RUN if [ "${USE_K8S}" = "ON" ]; then \
+        git clone --depth 1 https://github.com/yaml/libyaml.git /dependencies/libyaml && \
+        cd /dependencies/libyaml && \
+        mkdir build && \
+        cd build && \
+        cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=ON .. && \
+        make ${BUILD_THREADS} && \
+        make install; \
+    fi
+
+# hadolint ignore=DL3003,SC2086
+RUN if [ "${USE_K8S}" = "ON" ]; then \
+        git clone https://github.com/kubernetes-client/c.git /dependencies/k8s && \
+        ls /dependencies/k8s && \
+        CLIENT_REPO_ROOT=/dependencies/k8s && \
+        cd ${CLIENT_REPO_ROOT}/kubernetes && \
+        mkdir build && \
+        cd build && \
+        cmake -DCMAKE_PREFIX_PATH=/usr/local -DCMAKE_INSTALL_PREFIX=/usr/local .. && \
+        make ${BUILD_THREADS} && \
+        make install; \
+    fi
+
 # CLEANUP
 RUN rm -rf /dependencies /usr/local/share/doc /usr/local/share/man && \
     mkdir -p /opt/dist/usr/include/x86_64-linux-gnu && \
@@ -172,7 +204,12 @@ RUN rm -rf /dependencies /usr/local/share/doc /usr/local/share/man && \
 ############################################################
 # FINAL IMAGE
 FROM base
-ARG BUILD_COVERAGE="on"
+ARG BUILD_COVERAGE="ON"
+
+# Convert ARG value to uppercase and set ENV
+# hadolint ignore=SC3059
+RUN export BUILD_COVERAGE=$(echo "${BUILD_COVERAGE^^}") && \
+    echo "export BUILD_COVERAGE=${BUILD_COVERAGE}" >> /etc/profile.d/envvars.sh
 ENV BUILD_COVERAGE="${BUILD_COVERAGE}"
 
 # COPY FILES
@@ -183,38 +220,46 @@ COPY client /vdms/client
 COPY distributed /vdms/distributed
 COPY include /vdms/include
 COPY remote_function /vdms/remote_function
+COPY resources /vdms/resources
 COPY src /vdms/src
 COPY tests /vdms/tests
 COPY user_defined_operations /vdms/user_defined_operations
 COPY utils /vdms/utils
 COPY CMakeLists.txt /vdms/CMakeLists.txt
 COPY config-vdms.json /vdms/config-vdms.json
+COPY kubernetes/global_vdms_setup_script.sh  /vdms/kubernetes/global_vdms_setup_script.sh
+COPY kubernetes/kubeConfig.json /vdms/kubernetes/kubeConfig.json
 COPY docker/override_default_config.py /vdms/override_default_config.py
 COPY --from=build /opt/dist /
 COPY --from=build /usr/local/bin/python${PYTHON_BASE} /usr/local/bin/python${PYTHON_BASE}
 COPY --from=build /usr/local/lib/python${PYTHON_BASE} /usr/local/lib/python${PYTHON_BASE}
 COPY --from=build ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+COPY --from=build /usr/local/include/kubernete[s] /usr/local/include/kubernetes
+COPY --from=build /usr/include/libwebsocket[s] /usr/include/libwebsockets
+COPY --from=build /usr/local/lib/libkubernetes.s[o] /usr/local/lib/libkubernetes.so
+COPY --from=build /usr/local/lib/libyaml.s[o] /usr/local/lib/libyaml.so
+COPY --from=build /usr/lib/x86_64-linux-gnu/libwebsockets.s[o] /usr/lib/x86_64-linux-gnu/libwebsockets.so
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 # hadolint ignore=DL3008,SC2086
 RUN apt-get update -y && apt-get upgrade -y && \
     apt-get install -o 'Acquire::Retries=3' -y --no-install-suggests \
         --no-install-recommends --fix-broken --fix-missing \
-        build-essential bzip2 cppzmq-dev curl g++ gcc git javacc libarchive-tools libavcodec-dev \
+        build-essential bzip2 cppzmq-dev curl g++ gcc git javacc libarchive-tools libavcodec-dev bazel-bootstrap \
         libavformat-dev libcurl4-openssl-dev libdc1394-dev libgoogle-glog-dev libgtk-3-dev \
         libhdf5-dev libjpeg62-turbo-dev libjsoncpp-dev libopenblas-dev libpng-dev librdkafka-dev \
-        libssl-dev libswscale-dev libtbb-dev libtbbmalloc2 libtiff5-dev libzip-dev openjdk-17-jdk-headless \
-        procps && \
+        libssl-dev libswscale-dev libtbb-dev libtbbmalloc2 libtiff5-dev libwebsockets-dev libzip-dev \
+        openjdk-17-jdk-headless procps uncrustify && \
     apt-get --purge remove -y python3.11 && apt-get autoremove -y && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
     echo "/usr/local/lib" >> /etc/ld.so.conf.d/all-libs.conf && ldconfig && \
     python3 -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" "protobuf==4.${PROTOBUF_VERSION}" \
-        "coverage>=7.3.1" "cryptography>=42.0.7"
+        "coverage>=7.3.1" "cryptography>=44.0.1"
 
 # COVERAGE TESTING
 WORKDIR /vdms
 # hadolint ignore=DL3008,SC2086
-RUN if [ "${BUILD_COVERAGE}" = "on" ]; then \
+RUN if [ "${BUILD_COVERAGE}" = "ON" ]; then \
         apt-get update -y ; \
         apt-get install -y --no-install-suggests --no-install-recommends gdb ; \
         apt-get clean ; \
@@ -239,11 +284,12 @@ RUN git submodule update --init --recursive && \
     sed -i "s|#include <stdio.h>|#include <stdio.h>\n#include <stdexcept>|" /vdms/src/pmgd/test/neighbortest.cc && \
     sed -i "s|#include <stdio.h>|#include <stdio.h>\n#include <stdexcept>|" /vdms/src/pmgd/tools/mkgraph.cc && \
     mkdir -p /vdms/build && cd /vdms/build && \
-    cmake -DCODE_COVERAGE="${BUILD_COVERAGE}" .. && make ${BUILD_THREADS} && \
+    cmake -DUSE_K8S="${USE_K8S}" -DCODE_COVERAGE="${BUILD_COVERAGE}" .. && make ${BUILD_THREADS} VERBOSE=1 && \
     echo '#!/bin/bash' > /start.sh && echo 'cd /vdms/build' >> /start.sh && \
-    echo 'python /vdms/override_default_config.py -i /vdms/config-vdms.json -o /vdms/build/config-vdms.json' >> /start.sh && \
+    echo 'python3 /vdms/override_default_config.py -i /vdms/config-vdms.json -o /vdms/build/config-vdms.json' >> /start.sh && \
     echo './vdms' >> /start.sh && chmod 755 /start.sh
 
-ENV PYTHONPATH=/vdms/client/python:${PYTHONPATH}
+ENV PYTHONPATH=/vdms/client/python
+
 HEALTHCHECK CMD echo "This is a healthcheck test." || exit 1
 CMD ["/start.sh"]

.github/scripts/Dockerfile.checkin

@@ -4,7 +4,8 @@ services:
       dockerfile: .github/scripts/Dockerfile.checkin
       context: ../..
       args:
-        - BUILD_COVERAGE=on
+        - BUILD_COVERAGE=ON
+        - USE_K8S=ON
     image: vdms:${SOURCE_CONTAINER_NAME}
     container_name: ${SOURCE_CONTAINER_NAME}
     networks: [backend, frontend]

.github/scripts/auto-formatter.sh

@@ -2,18 +2,19 @@
 
 cd /vdms/tests
 
-# Run Local C++ PMGD Based Tests
-chmod +x run_tests.sh
-echo 'Running run_tests.sh script'
-./run_tests.sh
-
 # Run S3 C++ PMGD Based Tests
 echo 'Checking for the available disk space due MinIO requires at least 1gb free'
 df -h
-chmod +x run_aws_tests.sh
-echo 'Running run_aws_tests.sh script'
-./run_aws_tests.sh -u ${AWS_ACCESS_KEY_ID} -p ${AWS_SECRET_ACCESS_KEY}
 
+chmod +x ./run_all_tests.py
+
+echo 'Running run_all_tests.py script for remote C++ tests (-t=ru)'
+python ./run_all_tests.py -t=ru -k -u ${AWS_ACCESS_KEY_ID} -p ${AWS_SECRET_ACCESS_KEY}
+
+echo 'Running run_all_tests script for C++ tests (-t=ut)'
+python ./run_all_tests.py -t=ut -k
+
+echo 'Getting the coverage for C++'
 # Obtain Coverage
 gcovr --root /vdms \
     -e /vdms/src/pmgd -e /vdms/build -e /vdms/distributed -e /vdms/tests \

.github/scripts/docker-compose.yml

@@ -1,11 +1,28 @@
 #!/bin/bash -e
 
-cd /vdms/tests/python
+cd /vdms/tests
+chmod +x ./run_all_tests.py
+chmod +x ./TestScript.py
 
-./run_python_tests.sh
-./run_python_aws_tests.sh -u ${AWS_ACCESS_KEY_ID} -p ${AWS_SECRET_ACCESS_KEY}
-python -m coverage report -m 2>&1 | tee /vdms/tests/coverage_report/python.new.coverage_report.txt
-python -m coverage xml -o /vdms/tests/coverage_report/python.new.coverage_report.xml
+# Check .coverage file doesn't exist
+if [ -f './.coverage' ]; then
+    echo ".coverage exists."
+    rm ./.coverage
+    echo ".coverage deleted."
+fi
+
+echo 'Running run_all_tests script for Python tests (-t=pt)'
+python3 ./run_all_tests.py -t=pt -k
+
+echo 'Running run_all_tests script for Remote Python tests (-t=rp)'
+python3 ./run_all_tests.py -t=rp -k -u=${AWS_ACCESS_KEY_ID} -p=${AWS_SECRET_ACCESS_KEY}
+
+echo 'Running the tests included in TestScript.py file'
+# Append the coverage results to the ones obtained from running run_all_tests.py
+python3 -m coverage run -a --omit="./run_all_tests.py,./TestScript.py" -m unittest discover --pattern=TestScript.py -v
+
+python3 -m coverage report -m 2>&1 | tee /vdms/tests/coverage_report/python.new.coverage_report.txt
+python3 -m coverage xml -o /vdms/tests/coverage_report/python.new.coverage_report.xml
 
 echo "DONE"