Upgrade grpc/protobuf for PR#301 (gRPC and Protobuf support for remote user-defined operations)

.github/requirements.txt

@@ -1,24 +1,26 @@
-blinker==1.9.0
-cffi==1.17.1
-click==8.1.8
+cffi==2.0.0
 colorlog==6.9.0
-coverage==7.8.0
-cryptography==44.0.2
-Flask==3.1.1
+coverage==7.10.7
+cryptography==46.0.2
 gcovr==8.4
+grpcio==1.75.1
+grpcio-tools==1.75.1
+imageio==2.31.1
 imutils==0.5.4
-itsdangerous==2.2.0
 Jinja2==3.1.6
-lxml==5.3.2
-MarkupSafe==3.0.2
+lxml==6.0.2
+MarkupSafe==3.0.3
 numpy==1.26.4
+opencv-contrib-python==4.6.0.66
+opencv-python==4.10.0.82
 opencv-python-headless==4.11.0.86
-pillow==11.1.0
-protobuf==4.25.8
-pycparser==2.22
-Pygments==2.19.1
-pyzmq==26.4.0
-scipy==1.15.2
+pillow==11.3.0
+protobuf==6.31.1
+pycparser==2.23
+Pygments==2.19.2
+pyzmq==27.1.0
+scipy==1.16.2
+setuptools==80.9.0
 sk-video==1.1.10
-Werkzeug==3.1.3
+typing_extensions==4.15.0
 zmq==0.0.0

.github/scripts/Dockerfile.checkin

@@ -21,7 +21,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 ENV DEBCONF_NOWARNINGS="yes"
 ENV PYTHON_BASE="3.12"
 ENV PYTHON_VERSION="${PYTHON_BASE}.3"
-ENV PROTOBUF_VERSION="25.8"
+ENV PROTOBUF_VERSION="6.31.1"
 ENV NUMPY_MIN_VERSION="1.26.0"
 ENV VIRTUAL_ENV=/opt/venv
 
@@ -73,13 +73,17 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 # Pull and Install Dependencies
 WORKDIR /dependencies
-ENV AUTOCONF_VERSION="2.71" \
+ENV ABSEIL_VERSION="20250512.1" \
+    AUTOCONF_VERSION="2.71" \
     AWS_SDK_VERSION="1.11.336" \
     CMAKE_VERSION="v3.28.5"  \
     FAISS_VERSION="v1.9.0" \
+    GRPC_VERSION="v1.75.1" \
+    GTEST_VERSION="52eb8108c5bdec04579160ae17225d66034bd723" \
     LIBEDIT_VERSION="20230828-3.1" \
     OPENCV_VERSION="4.9.0" \
     PEG_VERSION="0.1.19" \
+    PROTOBUF_VERSION_COMMIT="74211c0dfc2777318ab53c2cd2c317a2ef9012de" \
     TILEDB_VERSION="2.14.1" \
     VALIJSON_VERSION="v0.6"
 
@@ -89,27 +93,49 @@ RUN git clone --branch ${CMAKE_VERSION} https://github.com/Kitware/CMake.git /de
         cd /dependencies/CMake && ./bootstrap && make ${BUILD_THREADS} && \
             make install DESTDIR=/opt/dist && make install
 
-# PROTOBUF & ITS DEPENDENCIES
+# PROTOBUF & ITS DEPENDENCIES (GOOGLETEST, ABSEIL-CPP)
 # hadolint ignore=DL3003,SC2086
-RUN git clone -b "v${PROTOBUF_VERSION}" --recurse-submodules https://github.com/protocolbuffers/protobuf.git /dependencies/protobuf && \
-        cd /dependencies/protobuf/third_party/googletest && mkdir build && cd build/ && \
-            cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release \
-                -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/opt/dist/usr/local \
-                -DBUILD_GMOCK=ON -DCMAKE_CXX_STANDARD=17 .. && \
-            make ${BUILD_THREADS} && make install && \
-        cd /dependencies/protobuf/third_party/abseil-cpp && mkdir build && cd build && \
-            cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON \
-                -DCMAKE_INSTALL_PREFIX=/opt/dist/usr/local -DABSL_BUILD_TESTING=ON \
-                -DABSL_USE_EXTERNAL_GOOGLETEST=ON \
-                -DABSL_FIND_GOOGLETEST=ON -DCMAKE_CXX_STANDARD=17 .. && \
-            make ${BUILD_THREADS} && make install && ldconfig /opt/dist/usr/local/lib && \
-        cd /dependencies/protobuf && \
-            cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX=/opt/dist/usr/local \
-                -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_SHARED_LIBS=ON \
-                -Dprotobuf_ABSL_PROVIDER=package \
-                -Dprotobuf_BUILD_TESTS=ON \
-                -Dabsl_DIR=/opt/dist/usr/local/lib/cmake/absl . && \
-            make ${BUILD_THREADS} && make install
+RUN git clone https://github.com/google/googletest.git /dependencies/googletest && \
+		cd /dependencies/googletest && git checkout ${GTEST_VERSION} && \
+            mkdir build && cd build && \
+			cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release \
+				-DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/opt/dist/usr/local \
+				-DBUILD_GMOCK=ON -DCMAKE_CXX_STANDARD=17 .. && \
+			make ${BUILD_THREADS} && make install && \
+	git clone -b ${ABSEIL_VERSION} https://github.com/abseil/abseil-cpp.git /dependencies/abseil && \
+		cd /dependencies/abseil && mkdir build && cd build && \
+		cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON \
+			-DCMAKE_INSTALL_PREFIX=/opt/dist/usr/local -DABSL_BUILD_TESTING=ON \
+			-DABSL_USE_EXTERNAL_GOOGLETEST=ON \
+			-DABSL_FIND_GOOGLETEST=ON -DCMAKE_CXX_STANDARD=17 .. && \
+		make ${BUILD_THREADS} && make install && ldconfig /opt/dist/usr/local/lib && \
+	git clone --recurse-submodules https://github.com/protocolbuffers/protobuf.git /dependencies/protobuf && \
+		cd /dependencies/protobuf && git checkout ${PROTOBUF_VERSION_COMMIT} && \
+		cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX=/opt/dist/usr/local \
+			-DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_SHARED_LIBS=ON \
+			-Dprotobuf_ABSL_PROVIDER=package \
+			-Dprotobuf_GTEST_PROVIDER=package \
+			-Dprotobuf_BUILD_TESTS=ON \
+			-Dabsl_DIR=/opt/dist/usr/local/lib/cmake/absl . && \
+		make ${BUILD_THREADS} && make install
+
+# AUTOCONF VERSION FOR NEO4J
+# hadolint ignore=DL3003,SC2086
+RUN curl -L -O http://ftpmirror.gnu.org/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz && \
+        tar -xzf autoconf-${AUTOCONF_VERSION}.tar.gz && cd autoconf-${AUTOCONF_VERSION} && \
+        ./configure && make ${BUILD_THREADS} && make install DESTDIR=/opt/dist && make install
+
+# gRPC
+RUN ldconfig && git clone -b ${GRPC_VERSION} --depth 1 --recursive https://github.com/grpc/grpc /dependencies/grpc && \
+        cd /dependencies/grpc && \
+        mkdir -p cmake/build && cd cmake/build && \
+        cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON \
+            -DCMAKE_CXX_STANDARD=17 -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF \
+            -DCMAKE_INSTALL_PREFIX=/opt/dist/usr/local \
+            -DgRPC_ABSL_PROVIDER=package \
+            -DgRPC_PROTOBUF_PROVIDER=package \
+            ../.. && \
+        cmake --build . -- -j && cmake --install .
 
 # OPENCV
 # hadolint ignore=DL3003,SC2086
@@ -148,12 +174,6 @@ RUN curl -L -O https://github.com/TileDB-Inc/TileDB/archive/refs/tags/${TILEDB_V
             -DBUILD_ONLY="s3" -DCUSTOM_MEMORY_MANAGEMENT=OFF -DENABLE_TESTING=OFF && \
         make ${BUILD_THREADS} && make install DESTDIR=/opt/dist && make install
 
-# AUTOCONF VERSION FOR NEO4J
-# hadolint ignore=DL3003,SC2086
-RUN curl -O  https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.xz && \
-        tar -xf autoconf-${AUTOCONF_VERSION}.tar.xz && cd autoconf-${AUTOCONF_VERSION} && \
-        ./configure && make ${BUILD_THREADS} && make install DESTDIR=/opt/dist && make install
-
 # LIB-OMNI FOR NEO4J QUERY HANDLER
 # hadolint ignore=DL3003,SC2086
 RUN curl -L -O https://github.com/gpakosz/peg/releases/download/${PEG_VERSION}/peg-${PEG_VERSION}.tar.gz && \
@@ -239,6 +259,9 @@ COPY --from=build /usr/include/libwebsocket[s] /usr/include/libwebsockets
 COPY --from=build /usr/local/lib/libkubernetes.s[o] /usr/local/lib/libkubernetes.so
 COPY --from=build /usr/local/lib/libyaml.s[o] /usr/local/lib/libyaml.so
 COPY --from=build /usr/lib/x86_64-linux-gnu/libwebsockets.s[o] /usr/lib/x86_64-linux-gnu/libwebsockets.so
+COPY --from=build /usr/local/include/ /usr/local/include/
+COPY --from=build /usr/local/bin /usr/local/bin
+COPY --from=build /usr/local/lib/ /usr/local/lib/
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 # hadolint ignore=DL3008,SC2086
@@ -253,11 +276,12 @@ RUN apt-get update -y && apt-get upgrade -y && \
     apt-get --purge remove -y python3.11 && apt-get autoremove -y && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
     echo "/usr/local/lib" >> /etc/ld.so.conf.d/all-libs.conf && ldconfig && \
-    python3 -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" "protobuf==4.${PROTOBUF_VERSION}" \
+    python3 -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" "protobuf==${PROTOBUF_VERSION}" \
         "coverage>=7.3.1" "cryptography>=44.0.1"
 
-# COVERAGE TESTING
 WORKDIR /vdms
+
+# COVERAGE TESTING
 # hadolint ignore=DL3008,SC2086
 RUN if [ "${BUILD_COVERAGE}" = "ON" ]; then \
         apt-get update -y ; \

.github/scripts/setup_vdms.sh

@@ -190,15 +190,19 @@ fi
 #######################################################################################################################
 # INSTALL DEPENDENCIES
 #######################################################################################################################
+ABSEIL_VERSION="20250512.1"
 AUTOCONF_VERSION="2.71"
 AWS_SDK_VERSION="1.11.336"
 CMAKE_VERSION="v3.28.5"
 FAISS_VERSION="v1.9.0"
+GRPC_VERSION="v1.75.1"
+GTEST_VERSION="52eb8108c5bdec04579160ae17225d66034bd723"
 LIBEDIT_VERSION="20230828-3.1"
 NUMPY_MIN_VERSION="1.26.0"
 OPENCV_VERSION="4.9.0"
 PEG_VERSION="0.1.19"
-PROTOBUF_VERSION="25.8"
+PROTOBUF_VERSION="6.31.1"
+PROTOBUF_VERSION_COMMIT="74211c0dfc2777318ab53c2cd2c317a2ef9012de"
 TILEDB_VERSION="2.14.1"
 VALIJSON_VERSION="v0.6"
 
@@ -213,17 +217,18 @@ make ${BUILD_THREADS}
 make install
 
 
-# INSTALL PROTOBUF & ITS DEPENDENCIES
-git clone -b "v${PROTOBUF_VERSION}" --recurse-submodules https://github.com/protocolbuffers/protobuf.git $VDMS_DEP_DIR/protobuf
-cd $VDMS_DEP_DIR/protobuf/third_party/googletest
+# INSTALL PROTOBUF & ITS DEPENDENCIES (GOOGLETEST, ABSEIL-CPP)
+git clone https://github.com/google/googletest.git $VDMS_DEP_DIR/googletest
+cd $VDMS_DEP_DIR/googletest && git checkout ${GTEST_VERSION}
 mkdir build && cd build/
 cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release \
     -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local \
     -DBUILD_GMOCK=ON -DCMAKE_CXX_STANDARD=17 ..
 make ${BUILD_THREADS}
 make install
 
-cd $VDMS_DEP_DIR/protobuf/third_party/abseil-cpp
+git clone -b ${ABSEIL_VERSION} https://github.com/abseil/abseil-cpp.git $VDMS_DEP_DIR/abseil-cpp
+cd $VDMS_DEP_DIR/abseil-cpp
 mkdir build && cd build
 cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON \
     -DCMAKE_INSTALL_PREFIX=/usr/local -DABSL_BUILD_TESTING=ON \
@@ -233,16 +238,43 @@ make ${BUILD_THREADS}
 make install
 ldconfig /usr/local/lib
 
-cd $VDMS_DEP_DIR/protobuf
+git clone --recurse-submodules https://github.com/protocolbuffers/protobuf.git $VDMS_DEP_DIR/protobuf
+cd $VDMS_DEP_DIR/protobuf && git checkout ${PROTOBUF_VERSION_COMMIT}
 cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX=/usr/local \
     -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_SHARED_LIBS=ON \
     -Dprotobuf_ABSL_PROVIDER=package \
+    -Dprotobuf_GTEST_PROVIDER=package \
     -Dprotobuf_BUILD_TESTS=ON \
     -Dabsl_DIR=/usr/local/lib/cmake/absl .
 make ${BUILD_THREADS}
 make install
 
 
+# INSTALL AUTOCONF
+curl -L -o $VDMS_DEP_DIR/autoconf-${AUTOCONF_VERSION}.tar.gz http://ftpmirror.gnu.org/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz
+cd $VDMS_DEP_DIR
+tar -xzf autoconf-${AUTOCONF_VERSION}.tar.gz
+cd autoconf-${AUTOCONF_VERSION}
+./configure
+make ${BUILD_THREADS}
+make install
+
+
+# INSTALL gRPC
+ldconfig
+git clone -b ${GRPC_VERSION} --depth 1 --recursive https://github.com/grpc/grpc $VDMS_DEP_DIR/grpc
+cd $VDMS_DEP_DIR/grpc
+mkdir -p cmake/build && cd cmake/build
+cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON \
+    -DCMAKE_CXX_STANDARD=17 -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF \
+    -DCMAKE_INSTALL_PREFIX=/usr/local \
+    -DgRPC_ABSL_PROVIDER=package \
+    -DgRPC_PROTOBUF_PROVIDER=package \
+    ../..
+cmake --build . -- -j
+cmake --install .
+
+
 # INSTALL OPENCV
 git clone https://github.com/opencv/opencv.git $VDMS_DEP_DIR/opencv
 cd $VDMS_DEP_DIR/opencv
@@ -255,7 +287,7 @@ make install
 
 # INSTALL PYTHON PACKAGES
 python -m pip install --no-cache-dir "numpy>=${NUMPY_MIN_VERSION},<2.0.0" "coverage>=7.3.1" \
-    "protobuf==4.${PROTOBUF_VERSION}" "cryptography>=44.0.1"
+    "protobuf==${PROTOBUF_VERSION}" "cryptography>=44.0.1"
 
 
 # INSTALL VALIJSON
@@ -302,16 +334,6 @@ make ${BUILD_THREADS}
 make install
 
 
-# INSTALL AUTOCONF
-curl -L -o $VDMS_DEP_DIR/autoconf-${AUTOCONF_VERSION}.tar.xz https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.xz
-cd $VDMS_DEP_DIR
-tar -xf autoconf-${AUTOCONF_VERSION}.tar.xz
-cd autoconf-${AUTOCONF_VERSION}
-./configure
-make ${BUILD_THREADS}
-make install
-
-
 # INSTALL NEO4J CLIENTS
 curl -L -o $VDMS_DEP_DIR/peg-${PEG_VERSION}.tar.gz https://github.com/gpakosz/peg/releases/download/${PEG_VERSION}/peg-${PEG_VERSION}.tar.gz
 cd $VDMS_DEP_DIR/

.gitignore

@@ -13,3 +13,6 @@ tmp
 
 # Jetbrains Specific
 .idea
+
+# Remote UDF Temps
+remote_function/entity_pb2*

CMakeLists.txt

@@ -22,6 +22,12 @@ find_package( OpenCV REQUIRED )
 find_package(Protobuf CONFIG REQUIRED)
 find_package( CURL REQUIRED )
 find_package(AWSSDK REQUIRED COMPONENTS core s3)
+find_package(gRPC CONFIG REQUIRED)
+find_package(absl CONFIG REQUIRED)
+
+set(_PROTOBUF_PROTOC $<TARGET_FILE:protobuf::protoc>)
+set(_GRPC_GRPCPP gRPC::grpc++)
+set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
 
 include_directories(${Protobuf_INCLUDE_DIRS})
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
@@ -35,8 +41,9 @@ add_library(vdms_protobuf OBJECT
   ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf/partitionerMessages.proto
   ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf/pmgdMessages.proto
   ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf/queryMessage.proto
+  ${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf/entity.proto
 )
-target_link_libraries(vdms_protobuf PUBLIC protobuf::libprotobuf)
+target_link_libraries(vdms_protobuf PUBLIC protobuf::libprotobuf gRPC::grpc++)
 set(PROTO_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
 target_include_directories(vdms_protobuf PUBLIC "$<BUILD_INTERFACE:${PROTO_BINARY_DIR}>")
 protobuf_generate(
@@ -46,6 +53,14 @@ protobuf_generate(
   PROTOC_OUT_DIR "${PROTO_BINARY_DIR}"
 )
 
+protobuf_generate(
+    TARGET vdms_protobuf
+    LANGUAGE grpc
+    GENERATE_EXTENSIONS .grpc.pb.h .grpc.pb.cc
+    PLUGIN "protoc-gen-grpc=\$<TARGET_FILE:gRPC::grpc_cpp_plugin>"
+    IMPORT_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/utils/src/protobuf"
+    PROTOC_OUT_DIR "${PROTO_BINARY_DIR}")
+
 if (CLIENT)
   add_definitions("-D CLIENT")
 
@@ -103,11 +118,41 @@ else()
     src/VideoLoop.cc
   )
 
-  target_link_libraries(dms vcl pmgd pmgd-util protobuf tbb tiledb vdms-utils pthread -lcurl -lzmq -lzip ${AWSSDK_LINK_LIBRARIES} neo4j-client)
+  target_link_libraries(dms
+    vcl
+    pmgd
+    pmgd-util
+    protobuf
+    tbb
+    tiledb
+    vdms-utils
+    pthread
+    -lcurl
+    -lzmq
+    -lzip
+    ${AWSSDK_LINK_LIBRARIES}
+    neo4j-client
+    gRPC::grpc++
+  )
 
   add_executable(vdms src/vdms.cc)
 
-  target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS} ${AWSSDK_LINK_LIBRARIES})
+  target_link_libraries(vdms
+    dms
+    vdms_protobuf
+    vcl
+    tiledb
+    faiss
+    flinng
+    jsoncpp
+    ${OpenCV_LIBS}
+    ${AWSSDK_LINK_LIBRARIES}
+    ${Protobuf_LIBRARIES}
+    ${_REFLECTION}
+    ${_GRPC_GRPCPP}
+    ${_PROTOBUF_LIBPROTOBUF}
+  )
+
 endif ()
 
 message("Coverage:" ${CODE_COVERAGE})

client/python/setup.py

@@ -5,11 +5,11 @@
 
 setuptools.setup(
     name="vdms",
-    version="0.0.21",
+    version="0.0.22",
     author="Chaunté W. Lacewell",
     author_email="[email protected]",
     description="VDMS Client Module",
-    install_requires=["protobuf==4.25.8"],
+    install_requires=["protobuf==6.31.1"],
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/IntelLabs/vdms",