Orchestration in VDMS with Kubernetes

CMakeLists.txt

@@ -59,7 +59,7 @@ else()
   add_subdirectory(distributed)
 
   link_directories(/usr/local/lib /usr/lib/x86_64-linux-gnu/)
-  include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl /usr/include ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf)
+  include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl /usr/include ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf /usr/local/include/kubernetes)
   add_library(dms SHARED
     src/BackendNeo4j.cc
     src/BoundingBoxCommand.cc

Dockerfile.txt

@@ -0,0 +1,237 @@
+#Copyright (C) 2023 Intel Corporation
+#SPDX-License-Identifier: MIT
+
+ARG BASE_VERSION=12-slim
+ARG BUILD_THREADS="-j16"
+############################################################
+# BASE IMAGE W/ ENV VARS
+FROM debian:${BASE_VERSION} AS base
+# Dockerfile limitations force a repetition of global args
+ARG BUILD_THREADS
+ARG AWS_ACCESS_KEY_ID=""
+ARG AWS_SECRET_ACCESS_KEY=""
+ARG NEO4J_USER=""
+ARG NEO4J_PASS=""
+ARG NEO4J_ENDPOINT=""
+ARG AWS_API_PORT=9000
+ARG AWS_CONSOLE_PORT=9001
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBCONF_NOWARNINGS="yes"
+ENV PYTHON_BASE="3.12"
+ENV PYTHON_VERSION="${PYTHON_BASE}.3"
+ENV PROTOBUF_VERSION="24.2"
+ENV NUMPY_MIN_VERSION="1.26.0"
+ENV VIRTUAL_ENV=/opt/venv
+
+ENV AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}"
+ENV AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}"
+ENV NEO4J_USER="${NEO4J_USER}"
+ENV NEO4J_PASS="${NEO4J_PASS}"
+ENV NEO4J_ENDPOINT="${NEO4J_ENDPOINT}"
+ENV AWS_API_PORT="${AWS_API_PORT}"
+ENV AWS_CONSOLE_PORT="${AWS_CONSOLE_PORT}"
+
+############################################################
+# BUILD DEPENDENCIES
+FROM base AS build
+
+# Install Packages
+# hadolint ignore=DL3008
+RUN apt-get update -y && apt-get upgrade -y && \
+    apt-get install -o 'Acquire::Retries=3' -y --no-install-suggests \
+        --no-install-recommends --fix-broken --fix-missing \
+        apt-transport-https automake bison build-essential bzip2 ca-certificates \
+        cppzmq-dev curl ed flex g++ gcc git gnupg-agent javacc libarchive-tools libatlas-base-dev \
+        libavcodec-dev libavformat-dev libavutil-dev libboost-all-dev libbz2-dev libc-ares-dev \
+        libcurl4-openssl-dev libdc1394-dev libgflags-dev libgoogle-glog-dev \
+        libgtk-3-dev libgtk2.0-dev libhdf5-dev libjpeg-dev libjpeg62-turbo-dev libjsoncpp-dev \
+        libwebsockets-dev uncrustify \
+        libleveldb-dev liblmdb-dev liblz4-dev libncurses5-dev libopenblas-dev libopenmpi-dev \
+        libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev libtbbmalloc2 \
+        libtiff-dev libtiff5-dev libtool linux-libc-dev mpich openjdk-17-jdk-headless \
+        pkg-config procps software-properties-common swig unzip uuid-dev && \
+    apt-get --purge remove -y python3.11 && apt-get autoremove -y && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /usr/src
+# hadolint ignore=DL3003,DL3008,SC2086
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends libffi-dev libgdbm-dev libnss3-dev libreadline-dev libsqlite3-dev zlib1g-dev && \
+    curl -O https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \
+    tar -xzf Python-${PYTHON_VERSION}.tgz && cd Python-${PYTHON_VERSION} && \
+    ./configure --enable-optimizations && make ${BUILD_THREADS} && make altinstall && \
+    update-alternatives --install /usr/bin/python python /usr/local/bin/python3.12 1 && \
+    python${PYTHON_BASE} -m venv ${VIRTUAL_ENV}
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Pull and Install Dependencies
+WORKDIR /dependencies
+ENV AUTOCONF_VERSION="2.71" \
+    AWS_SDK_VERSION="1.11.336" \
+    CMAKE_VERSION="v3.28.5"  \
+    FAISS_VERSION="v1.7.4" \
+    LIBEDIT_VERSION="20230828-3.1" \
+    OPENCV_VERSION="4.9.0" \
+    PEG_VERSION="0.1.19" \
+    TILEDB_VERSION="2.14.1" \
+    VALIJSON_VERSION="v0.6"
+
+# CMAKE
+# hadolint ignore=DL3003,SC2086
+RUN git clone --branch ${CMAKE_VERSION} https://github.com/Kitware/CMake.git /dependencies/CMake && \
+        cd /dependencies/CMake && ./bootstrap && make ${BUILD_THREADS} && \
+            make install DESTDIR=/opt/dist && make install
+
+# PROTOBUF & ITS DEPENDENCIES
+# hadolint ignore=DL3003,SC2086
+RUN git clone -b "v${PROTOBUF_VERSION}" --recurse-submodules https://github.com/protocolbuffers/protobuf.git /dependencies/protobuf && \
+        cd /dependencies/protobuf/third_party/googletest && mkdir build && cd build/ && \
+            cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release \
+                -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/opt/dist/usr/local \
+                -DBUILD_GMOCK=ON -DCMAKE_CXX_STANDARD=17 .. && \
+            make ${BUILD_THREADS} && make install && \
+        cd /dependencies/protobuf/third_party/abseil-cpp && mkdir build && cd build && \
+            cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON \
+                -DCMAKE_INSTALL_PREFIX=/opt/dist/usr/local -DABSL_BUILD_TESTING=ON \
+                -DABSL_USE_EXTERNAL_GOOGLETEST=ON \
+                -DABSL_FIND_GOOGLETEST=ON -DCMAKE_CXX_STANDARD=17 .. && \
+            make ${BUILD_THREADS} && make install && ldconfig /opt/dist/usr/local/lib && \
+        cd /dependencies/protobuf && \
+            cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX=/opt/dist/usr/local \
+                -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_SHARED_LIBS=ON \
+                -Dprotobuf_ABSL_PROVIDER=package \
+                -Dprotobuf_BUILD_TESTS=ON \
+                -Dabsl_DIR=/opt/dist/usr/local/lib/cmake/absl . && \
+            make ${BUILD_THREADS} && make install
+
+# OPENCV
+# hadolint ignore=DL3003,SC2086
+RUN git clone https://github.com/opencv/opencv.git /dependencies/opencv && \
+        cd /dependencies/opencv && git checkout tags/${OPENCV_VERSION} && \
+        mkdir build && cd build && cmake -DBUILD_PERF_TESTS=OFF -DBUILD_TESTS=OFF .. && \
+            make ${BUILD_THREADS} && make install DESTDIR=/opt/dist && make install
+
+# VALIJSON
+# hadolint ignore=DL3003
+RUN python -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" && \
+    git clone --branch ${VALIJSON_VERSION} https://github.com/tristanpenman/valijson.git /dependencies/valijson && \
+        cd /dependencies/valijson && cp -r include/* /usr/local/include/ && \
+        mkdir -p /opt/dist/usr/local/include/ && cp -r include/* /opt/dist/usr/local/include/
+
+# FAISS & FLINNG LIBRARIES
+# hadolint ignore=DL3003,SC2086
+RUN git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git /dependencies/faiss && \
+        cd /dependencies/faiss && mkdir build && cd build && \
+            cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=$(which python) \
+            -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release .. && \
+            make ${BUILD_THREADS} && make install DESTDIR=/opt/dist && make install && \
+    git clone https://github.com/tonyzhang617/FLINNG.git /dependencies/FLINNG && \
+        cd /dependencies/FLINNG && mkdir build && cd build && cmake .. && \
+            make ${BUILD_THREADS} && make install DESTDIR=/opt/dist && make install
+
+# TILEDB & AWS S3 SDK
+# hadolint ignore=DL3003,SC2086
+RUN curl -L -O https://github.com/TileDB-Inc/TileDB/archive/refs/tags/${TILEDB_VERSION}.tar.gz && \
+        tar -xvf ${TILEDB_VERSION}.tar.gz && cd TileDB-${TILEDB_VERSION} && \
+        mkdir build && cd build && ../bootstrap --prefix=/usr/local/ && make ${BUILD_THREADS} && \
+        make install-tiledb DESTDIR=/opt/dist && make install-tiledb && \
+    git clone -b ${AWS_SDK_VERSION} --recurse-submodules https://github.com/aws/aws-sdk-cpp /dependencies/aws-sdk-cpp && \
+        mkdir -p /dependencies/aws-sdk-cpp/build && cd /dependencies/aws-sdk-cpp/build && \
+        cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ \
+            -DBUILD_ONLY="s3" -DCUSTOM_MEMORY_MANAGEMENT=OFF -DENABLE_TESTING=OFF && \
+        make ${BUILD_THREADS} && make install DESTDIR=/opt/dist && make install
+
+# AUTOCONF VERSION FOR NEO4J
+# hadolint ignore=DL3003,SC2086
+RUN curl -O  https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.xz && \
+        tar -xf autoconf-${AUTOCONF_VERSION}.tar.xz && cd autoconf-${AUTOCONF_VERSION} && \
+        ./configure && make ${BUILD_THREADS} && make install DESTDIR=/opt/dist && make install
+
+# LIB-OMNI FOR NEO4J QUERY HANDLER
+# hadolint ignore=DL3003,SC2086
+RUN curl -L -O https://github.com/gpakosz/peg/releases/download/${PEG_VERSION}/peg-${PEG_VERSION}.tar.gz && \
+        tar -xf peg-${PEG_VERSION}.tar.gz && cd peg-${PEG_VERSION} && \
+        make ${BUILD_THREADS} && make install DESTDIR=/opt/dist && make install && \
+    git clone https://github.com/cleishm/libcypher-parser.git /dependencies/libcypher && \
+        cd /dependencies/libcypher && ./autogen.sh && ./configure && \
+        make install DESTDIR=/opt/dist && make install && \
+    curl -L -o /dependencies/libedit-${LIBEDIT_VERSION}.tar.gz https://thrysoee.dk/editline/libedit-${LIBEDIT_VERSION}.tar.gz && \
+        cd /dependencies && tar -xzf libedit-${LIBEDIT_VERSION}.tar.gz && cd libedit-${LIBEDIT_VERSION} && \
+        ./configure && make ${BUILD_THREADS} && make install DESTDIR=/opt/dist && make install && \
+    git clone https://github.com/majensen/libneo4j-omni.git /dependencies/libomni && \
+        cd /dependencies/libomni && ./autogen.sh && \
+        ./configure --disable-werror --prefix=/opt/dist/usr && \
+        make clean check && make install -w --debug
+
+# CLEANUP
+RUN rm -rf /dependencies /usr/local/share/doc /usr/local/share/man && \
+    mkdir -p /opt/dist/usr/include/x86_64-linux-gnu && \
+    cp -rp /usr/include/x86_64-linux-gnu /opt/dist/usr/include/x86_64-linux-gnu
+
+
+############################################################
+# FINAL IMAGE
+FROM base
+COPY --from=build /opt/dist /
+COPY --from=build /usr/local/bin/python${PYTHON_BASE} /usr/local/bin/python${PYTHON_BASE}
+COPY --from=build /usr/local/lib/python${PYTHON_BASE} /usr/local/lib/python${PYTHON_BASE}
+COPY --from=build ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# hadolint ignore=DL3008,SC2086
+RUN apt-get update -y && apt-get upgrade -y && \
+    apt-get install -o 'Acquire::Retries=3' -y --no-install-suggests \
+        --no-install-recommends --fix-broken --fix-missing \
+        build-essential bzip2 cppzmq-dev curl g++ gcc git javacc libarchive-tools libavcodec-dev \
+        libavformat-dev libcurl4-openssl-dev libdc1394-dev libgoogle-glog-dev libgtk-3-dev \
+        libhdf5-dev libjpeg62-turbo-dev libjsoncpp-dev libopenblas-dev libpng-dev librdkafka-dev \
+        libwebsockets-dev uncrustify \
+        libssl-dev libswscale-dev libtbb-dev libtbbmalloc2 libtiff5-dev libzip-dev openjdk-17-jdk-headless \
+        procps && \
+    apt-get --purge remove -y python3.11 && apt-get autoremove -y && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && \  
+    echo "/usr/local/lib" >> /etc/ld.so.conf.d/all-libs.conf && ldconfig && \
+    python3 -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" "protobuf==4.${PROTOBUF_VERSION}" \
+        "coverage>=7.3.1" "cryptography>=42.0.7"
+
+RUN git clone --depth 1 https://github.com/yaml/libyaml.git /libyaml && \
+    cd /libyaml && \
+    mkdir build && \
+    cd build &&\
+    cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_TESTING=OFF  -DBUILD_SHARED_LIBS=ON && \
+    make && \
+    make install
+
+#Kubernetes
+RUN git clone https://github.com/kubernetes-client/c.git /k8s && \
+    CLIENT_REPO_ROOT=/k8s && \
+    cd /k8s/kubernetes && \
+    mkdir build && \
+    cd build && \
+    cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
+    make && make install
+# VDMS
+WORKDIR /vdms
+# hadolint ignore=DL3003,SC2086
+COPY . .
+COPY custom_bash.sh /
+RUN chmod +x /custom_bash.sh
+RUN sed -i "s|java-11-openjdk|java-17-openjdk|g" /vdms/src/pmgd/java/CMakeLists.txt && \
+sed -i "s|#include <stdio.h>|#include <stdio.h>\n#include <stdexcept>|" /vdms/src/pmgd/test/neighbortest.cc && \
+sed -i "s|#include <stdio.h>|#include <stdio.h>\n#include <stdexcept>|" /vdms/src/pmgd/tools/mkgraph.cc && \
+cp /vdms/docker/override_default_config.py /vdms/override_default_config.py && \  
+mkdir -p /vdms/build && cd /vdms/build && \
+cmake .. && make ${BUILD_THREADS} VERBOSE=1 && \
+echo '#!/bin/bash' > /start.sh && echo 'cd /vdms/build' >> /start.sh && \
+cp /vdms/docker/override_default_config.py /vdms/ && \
+echo 'python3 /vdms/override_default_config.py -i /vdms/config-vdms.json -o /vdms/build/config-vdms.json' >> /start.sh && \
+echo './vdms' >> /start.sh && chmod 755 /start.sh
+ENV PYTHONPATH=/vdms/client/python:${PYTHONPATH}
+
+RUN /custom_bash.sh
+RUN pip install Flask
+RUN apt-get update && apt-get install vim -y
+
+HEALTHCHECK CMD echo "This is a healthcheck test." || exit 1
+CMD ["/start.sh"]

custom_bash.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+cd /vdms/user_defined_operations/
+python3 -m pip install -r requirements.txt
+python3 -m pip install sk-video
+python3 -m pip install imutils
+python3 -m pip install zmq

docker/base/Dockerfile

@@ -47,6 +47,7 @@ RUN apt-get update -y && apt-get upgrade -y && \
         libcurl4-openssl-dev libdc1394-dev libgflags-dev libgoogle-glog-dev \
         libgtk-3-dev libgtk2.0-dev libhdf5-dev libjpeg-dev libjpeg62-turbo-dev libjsoncpp-dev \
         libleveldb-dev liblmdb-dev liblz4-dev libncurses5-dev libopenblas-dev libopenmpi-dev \
+        libwebsockets-dev uncrustify \
         libpng-dev librdkafka-dev libsnappy-dev libssl-dev libswscale-dev libtbb-dev libtbbmalloc2 \
         libtiff-dev libtiff5-dev libtool linux-libc-dev mpich openjdk-17-jdk-headless \
         pkg-config procps software-properties-common swig unzip uuid-dev && \
@@ -163,6 +164,22 @@ RUN curl -L -O https://github.com/gpakosz/peg/releases/download/${PEG_VERSION}/p
         ./configure --disable-werror --prefix=/opt/dist/usr && \
         make clean check && make install -w --debug
 
+RUN git clone --depth 1 https://github.com/yaml/libyaml.git /libyaml && \
+cd /libyaml && \
+mkdir build && \
+cd build &&\
+cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_TESTING=OFF  -DBUILD_SHARED_LIBS=ON && \
+make && \
+make install
+
+#Kubernetes
+RUN git clone https://github.com/kubernetes-client/c.git /k8s && \
+    CLIENT_REPO_ROOT=/k8s && \
+    cd /k8s/kubernetes && \
+    mkdir build && \
+    cd build && \
+    cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local/ && \
+    make && make install
 # CLEANUP
 RUN rm -rf /dependencies /usr/local/share/doc /usr/local/share/man && \
     mkdir -p /opt/dist/usr/include/x86_64-linux-gnu && \
@@ -185,6 +202,7 @@ RUN apt-get update -y && apt-get upgrade -y && \
         build-essential bzip2 cppzmq-dev curl g++ gcc git javacc libarchive-tools libavcodec-dev \
         libavformat-dev libcurl4-openssl-dev libdc1394-dev libgoogle-glog-dev libgtk-3-dev \
         libhdf5-dev libjpeg62-turbo-dev libjsoncpp-dev libopenblas-dev libpng-dev librdkafka-dev \
+        libwebsockets-dev uncrustify \
         libssl-dev libswscale-dev libtbb-dev libtbbmalloc2 libtiff5-dev libzip-dev openjdk-17-jdk-headless \
         procps && \
     apt-get --purge remove -y python3.11 && apt-get autoremove -y && \
@@ -196,7 +214,7 @@ RUN apt-get update -y && apt-get upgrade -y && \
 # VDMS
 WORKDIR /vdms
 # hadolint ignore=DL3003,SC2086
-RUN git clone -b master --recurse-submodules https://github.com/IntelLabs/vdms.git /vdms && \
+RUN git clone -b kubernetes_support --recurse-submodules https://github.com/IntelLabs/vdms.git /vdms && \
     sed -i "s|java-11-openjdk|java-17-openjdk|g" /vdms/src/pmgd/java/CMakeLists.txt && \
     sed -i "s|#include <stdio.h>|#include <stdio.h>\n#include <stdexcept>|" /vdms/src/pmgd/test/neighbortest.cc && \
     sed -i "s|#include <stdio.h>|#include <stdio.h>\n#include <stdexcept>|" /vdms/src/pmgd/tools/mkgraph.cc && \