Merge branch 'develop' into cwl-283_rv355_grpc_proto

Signed-off-by: Lacewell, Chaunte W <[email protected]>

Author: cwlacewe

.github/coverage/cpp.develop.coverage_report.txt

@@ -1 +1 @@
-71.332
+68.113

.github/coverage/cpp.develop.coverage_value.txt

@@ -5,7 +5,7 @@ colorlog==6.9.0
 coverage==7.8.0
 cryptography==44.0.2
 Flask==3.1.1
-gcovr==8.3
+gcovr==8.4
 imutils==0.5.4
 itsdangerous==2.2.0
 Jinja2==3.1.6
@@ -14,7 +14,7 @@ MarkupSafe==3.0.2
 numpy==1.26.4
 opencv-python-headless==4.11.0.86
 pillow==11.1.0
-protobuf==4.24.2
+protobuf==4.25.8
 pycparser==2.22
 Pygments==2.19.1
 pyzmq==26.4.0

.github/requirements.txt

@@ -307,7 +307,7 @@ RUN if [ "${BUILD_COVERAGE}" = "ON" ]; then \
         apt-get install -y --no-install-suggests --no-install-recommends gdb ; \
         apt-get clean ; \
         rm -rf /var/lib/apt/lists/* ; \
-        python3 -m pip install --no-cache-dir "gcovr>=7.0" ; \
+        python3 -m pip install --no-cache-dir "gcovr==8.4" ; \
         curl -L -o /vdms/minio https://dl.min.io/server/minio/release/linux-amd64/minio ; \
         chmod +x /vdms/minio ; \
         mkdir -p /vdms/minio_files/minio-bucket ; \

.github/scripts/Dockerfile.checkin

@@ -198,7 +198,7 @@ LIBEDIT_VERSION="20230828-3.1"
 NUMPY_MIN_VERSION="1.26.0"
 OPENCV_VERSION="4.9.0"
 PEG_VERSION="0.1.19"
-PROTOBUF_VERSION="24.2"
+PROTOBUF_VERSION="25.8"
 TILEDB_VERSION="2.14.1"
 VALIJSON_VERSION="v0.6"
 

.github/scripts/setup_vdms.sh

@@ -0,0 +1,100 @@
+name: "CodeQL Analysis"
+
+on:
+  push:
+    branches: [ "develop" ]
+  workflow_dispatch:
+
+jobs:
+  analyze-python:
+    name: Analyze Python
+    runs-on: ubuntu-latest
+
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL (Python)
+        uses: github/codeql-action/init@v3
+        with:
+          languages: python
+
+      - name: Autobuild Python
+        uses: github/codeql-action/autobuild@v3
+
+      - name: Perform CodeQL Analysis (Python)
+        uses: github/codeql-action/analyze@v3
+
+  analyze-cpp:
+    name: Analyze CPP
+    runs-on: ubuntu-latest
+
+    # Run everything inside your Docker image
+    container:
+      image: intellabs/vdms:latest
+
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'cpp' ]   # VDMS is C++ based
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+
+      - name: Install latest CMake
+        run: |
+          apt-get update
+          apt-get remove --purge -y cmake cmake-data || true
+          apt-get install -y wget tar xz-utils build-essential
+
+          CMAKE_VERSION=3.30.4
+          wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz
+          tar -xzf cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz
+          mv cmake-${CMAKE_VERSION}-linux-x86_64 /opt/cmake
+          ln -sf /opt/cmake/bin/* /usr/local/bin/
+          cmake --version
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: cpp
+          queries: security-extended,security-and-quality
+          build-mode: manual
+
+      - name: Configure CMake (disable linking)
+        run: |
+          sed -i "s|#include <stdio.h>|#include <stdio.h>\n#include <stdexcept>|" src/pmgd/test/neighbortest.cc
+          sed -i "s|#include <stdio.h>|#include <stdio.h>\n#include <stdexcept>|" src/pmgd/tools/mkgraph.cc
+          sed -i "s|java-11-openjdk|java-17-openjdk|g" src/pmgd/java/CMakeLists.txt
+          cmake -S . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+                -DCMAKE_C_LINK_EXECUTABLE="/bin/true" \
+                -DCMAKE_CXX_LINK_EXECUTABLE="/bin/true"
+
+      - name: Build (compile only, no linking)
+        run: cmake --build build -j$(nproc)
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: '/language:cpp'
+          output: ./codeql-results
+
+      - name: Upload SARIF File
+        uses: actions/upload-artifact@v4
+        with:
+          name : codeql-results
+          path: ./codeql-results/**/*.sarif

.github/workflows/codeql_analysis.yml

@@ -0,0 +1,17 @@
+name: Intel Security Scan
+
+permissions: read-all
+
+on:
+  workflow_dispatch: {}
+  schedule:
+    - cron: "5 4 * * sun" # 4:05 AM Sunday
+
+jobs:
+  Call-Workflow:
+    uses: intel/.github/.github/workflows/intel_scan.yaml@main
+    permissions:
+      contents: read
+      packages: read
+      actions: read
+      security-events: write

.github/workflows/intel_security_scan.yaml

@@ -676,6 +676,8 @@ void PMGDQuery::QueryNode(int ref, const std::string &tag,
                           const Json::Value &constraints,
                           const Json::Value &results, bool unique,
                           bool intermediate_query) {
+  _readonly = false;
+
   PMGDCmd *cmdquery = new PMGDCmd();
   cmdquery->set_cmd_id(PMGDCmd::QueryNode);
   cmdquery->set_cmd_grp_id(_current_group_id);
@@ -713,6 +715,8 @@ void PMGDQuery::QueryEdge(int ref, int src_ref, int dest_ref,
                           const std::string &tag,
                           const Json::Value &constraints,
                           const Json::Value &results, bool unique) {
+  _readonly = false;
+
   PMGDCmd *cmdquery = new PMGDCmd();
   cmdquery->set_cmd_id(PMGDCmd::QueryEdge);
   cmdquery->set_cmd_grp_id(_current_group_id);

src/PMGDQuery.cc

@@ -186,63 +186,75 @@ void QueryHandlerNeo4j::process_query(protobufs::queryMessage &proto_query,
   Json::Value json_responses;
   Json::Value cmd_result;
 
+  std::vector<std::string> images_log;
+
   Json::Value root;
   int blob_count = 0;
   bool error = false;
 
-  rc = parse_commands(proto_query, root);
-  // begin neo4j transaction
-  tx = neoconn_pool->open_tx(conn, 10000, "w");
-  for (int j = 0; j < root.size(); j++) {
-    Json::Value neo4j_resp;
-    std::string cypher;
+  try {
+    rc = parse_commands(proto_query, root);
+    // begin neo4j transaction
+    tx = neoconn_pool->open_tx(conn, 10000, "w");
+    for (int j = 0; j < root.size(); j++) {
+      Json::Value neo4j_resp;
+      std::string cypher;
 
-    Json::Value &query = root[j];
-    std::string cmd = query.getMemberNames()[0];
+      Json::Value &query = root[j];
+      std::string cmd = query.getMemberNames()[0];
 
 
-    if (_rs_cmds.count(cmd) == 0) {
-        std::cout<<"Command: " << cmd << "Does not exist!" << std::endl;
-    }
+      if (_rs_cmds.count(cmd) == 0) {
+          std::cout<<"Command: " << cmd << "Does not exist!" << std::endl;
+      }
 
-    Neo4jCommand *rscmd = _rs_cmds[cmd];
-    cypher = query[cmd]["cypher"].asString();
+      Neo4jCommand *rscmd = _rs_cmds[cmd];
+      cypher = query[cmd]["cypher"].asString();
 
-    const std::string &blob =
-        rscmd->need_blob(query) ? proto_query.blobs(blob_count++) : "";
+      const std::string &blob =
+          rscmd->need_blob(query) ? proto_query.blobs(blob_count++) : "";
 
-    rc = rscmd->data_processing(cypher, query, blob, 0, cmd_result);
-    if (rc != 0) {
-      error = true;
-      proto_res.set_json(fastWriter.write(cmd_result));
-      break;
-    }
-    res_stream = neoconn_pool->run_in_tx((char *)cypher.c_str(), tx);
-    neo4j_resp = neoconn_pool->results_to_json(res_stream);
-    query["cp_result"] = cmd_result;
-    Json::Value resp_retval = rscmd->construct_responses(neo4j_resp, query, proto_res, blob);
-    //THIS IS VERY CLUNKY and confusing, NEEDS TO BE REFACTORED
-    if (neo4j_resp.isMember("metadata_res") && (cmd == "NeoAdd" || cmd == "NeoFind")) {
-        resp_retval["metadata_res"] = neo4j_resp["metadata_res"];
-    }
-    json_responses.append(resp_retval);
+      rc = rscmd->data_processing(cypher, query, blob, 0, cmd_result);
+      if (rc != 0) {
+        error = true;
+        proto_res.set_json(fastWriter.write(cmd_result));
+        break;
+      }
+      if (cmd_result.isMember("image_added")) {
+        images_log.push_back(cmd_result["image_added"].asString());
+      }
+      res_stream = neoconn_pool->run_in_tx((char *)cypher.c_str(), tx);
+      neo4j_resp = neoconn_pool->results_to_json(res_stream);
+      query["cp_result"] = cmd_result;
+      Json::Value resp_retval = rscmd->construct_responses(neo4j_resp, query, proto_res, blob);
+      //THIS IS VERY CLUNKY and confusing, NEEDS TO BE REFACTORED
+      if (neo4j_resp.isMember("metadata_res") && (cmd == "NeoAdd" || cmd == "NeoFind")) {
+          resp_retval["metadata_res"] = neo4j_resp["metadata_res"];
+      }
+      json_responses.append(resp_retval);
 
 
-  }
-    proto_res.set_json(fastWriter.write(json_responses));
-  // commit neo4j transaction, needs to be updated in future to account for
-  // errors on response construction
-  if (error == false) {
-    rc = neoconn_pool->commit_tx(tx);
-
-    if(rc != 0){
-        printf("Warning! Transaction Error: %d\n", rc);
-        exit(1);
+    }
+      proto_res.set_json(fastWriter.write(json_responses));
+    // commit neo4j transaction, needs to be updated in future to account for
+    // errors on response construction
+    if (error == false) {
+      rc = neoconn_pool->commit_tx(tx);
+
+      if(rc != 0){
+          printf("Warning! Transaction Error: %d\n", rc);
+          exit(1);
+      }
+
     }
 
+    neoconn_pool->put_conn(conn);
+  } catch(...) {
+    VCL::RemoteConnection *connection = get_existing_connection();
+    for (const std::string image : images_log) {
+      connection->Remove_Object(image);
+    }
   }
-
-  neoconn_pool->put_conn(conn);
 }
 
 int QueryHandlerNeo4j::parse_commands(

src/QueryHandlerNeo4j.cc

@@ -47,6 +47,7 @@
 #include "pmgd.h"
 #include "util.h"
 #include "PmgdFilter.h"
+#include "OpsIOCoordinator.h"
 
 #include "APISchema.h"
 #include <jsoncpp/json/writer.h>
@@ -276,6 +277,16 @@ void QueryHandlerPMGD::process_query(
     Json::Value response;
     response.append(exception_error);
     proto_res.set_json(fastWriter.write(response));
+
+    if (VDMSConfig::instance()->get_aws_flag()) {
+      VCL::RemoteConnection *connection = get_existing_connection();
+      for (const std::string image : images_log) {
+        connection->Remove_Object(image);
+      }
+      for (const std::string video : videos_log) {
+        connection->Remove_Object(video);
+      }
+    }
   };
 
   try {