Feat[bmq, mqb]: Support TLS listeners

.github/workflows/build-ubuntu.yaml

@@ -101,6 +101,7 @@ jobs:
             libfl-dev \
             libbenchmark-dev \
             libgmock-dev \
+            libgtest-dev \
             libz-dev
       - name: Install cached non packaged dependencies
         if: steps.build-cache-restore-step.outputs.cache-hit != 'true' # Variable type is string, thus using quotes

.github/workflows/build.yaml

@@ -279,6 +279,7 @@ jobs:
             libfl-dev \
             libbenchmark-dev \
             libgmock-dev \
+            libgtest-dev \
             libz-dev \
             autoconf \
             libtool

.github/workflows/dependencies.yaml

@@ -53,6 +53,7 @@ jobs:
             libfl-dev \
             libbenchmark-dev \
             libgmock-dev \
+            libgtest-dev \
             libz-dev
 
       - name: Fetch & build non packaged dependencies

.gitignore

@@ -41,4 +41,4 @@ settings.json
 src/applications/bmqbrkr/etc/etc
 
 # 'sim_cpp11_features.pl' backups
-*.bak
+*.bak 

bin/build-darwin.sh

@@ -91,7 +91,7 @@ mkdir -p "${DIR_THIRDPARTY}"
 DIR_BUILD="${DIR_BUILD:-${DIR_ROOT}/build}"
 mkdir -p "${DIR_BUILD}"
 
-DIR_INSTALL="${DIR_INSTALL:-${DIR_ROOT}}"
+DIR_INSTALL="${DIR_INSTALL:-${DIR_ROOT}/install}"
 mkdir -p "${DIR_INSTALL}"
 
 
@@ -103,7 +103,7 @@ if [ ! -d "${DIR_THIRDPARTY}/bde" ]; then
     git clone --depth 1 --branch 4.28.0.0 https://github.com/bloomberg/bde.git "${DIR_THIRDPARTY}/bde"
 fi
 if [ ! -d "${DIR_THIRDPARTY}/ntf-core" ]; then
-    git clone --depth 1 --branch 2.4.2 https://github.com/bloomberg/ntf-core.git "${DIR_THIRDPARTY}/ntf-core"
+    git clone --depth 1 --branch 2.5.4 https://github.com/bloomberg/ntf-core.git "${DIR_THIRDPARTY}/ntf-core"
 fi
 
 

bin/build-ubuntu.sh

@@ -6,7 +6,7 @@ echo -e "Before running this script, install the following prerequisites, if not
         "by executing the following commands:\n"                                               \
         "sudo apt update && sudo apt -y install ca-certificates\n"                             \
         "sudo apt install -y --no-install-recommends"                                          \
-        "autoconf automake build-essential gdb cmake ninja-build pkg-config bison libfl-dev libbenchmark-dev libgmock-dev libtool libz-dev"
+        "autoconf automake build-essential gdb cmake ninja-build pkg-config bison libfl-dev libbenchmark-dev libgmock-dev libgtest-dev libtool libz-dev"
 
 # :: Parse and validate arguments :::::::::::::::::::::::::::::::::::::::::::::
 print_usage_and_exit_with_error() {
@@ -59,7 +59,7 @@ mkdir -p "${DIR_THIRDPARTY}"
 DIR_BUILD="${DIR_BUILD:-${DIR_ROOT}/build}"
 mkdir -p "${DIR_BUILD}"
 
-DIR_INSTALL="${DIR_INSTALL:-${DIR_ROOT}}"
+DIR_INSTALL="${DIR_INSTALL:-${DIR_ROOT}/install}"
 mkdir -p "${DIR_INSTALL}"
 
 # :: Clone dependencies :::::::::::::::::::::::::::::::::::::::::::::::::::::::
@@ -71,7 +71,7 @@ if [ ! -d "${DIR_THIRDPARTY}/bde" ]; then
     git clone --depth 1 --branch 4.28.0.0 https://github.com/bloomberg/bde.git "${DIR_THIRDPARTY}/bde"
 fi
 if [ ! -d "${DIR_THIRDPARTY}/ntf-core" ]; then
-    git clone --depth 1 --branch 2.4.2 https://github.com/bloomberg/ntf-core.git "${DIR_THIRDPARTY}/ntf-core"
+    git clone --depth 1 --branch 2.5.4 https://github.com/bloomberg/ntf-core.git "${DIR_THIRDPARTY}/ntf-core"
 fi
 # prometheus-cpp and its dependency for the plugin
 if [ "${BUILD_PROMETHEUS}" == true ]; then

bin/gen-tls-certs.sh

@@ -0,0 +1,127 @@
+#!/bin/bash
+#
+#
+# This scripts generates:
+#  - root CA certificate
+#  - server certificate and keystore
+#  - client keys
+#
+# Based off of
+#   https://github.com/confluentinc/librdkafka/blob/master/tests/gen-ssl-certs.sh
+
+OP="$1"
+CA_CERT="$2"
+PFX="$3"
+HOST="$4"
+
+C=NN
+ST=NN
+L=NN
+O=NN
+OU=NN
+CN="$HOST"
+
+
+# Password
+PASS="secret"
+
+# Cert validity, in days
+VALIDITY=10000
+
+set -e
+
+export LC_ALL=C
+
+if [[ $OP == "ca" && -n "$CA_CERT" && -n "$3" ]]; then
+    CN="$3"
+    openssl req -new -x509 -keyout "${CA_CERT}.key" -out "${CA_CERT}" -days $VALIDITY -passin "pass:$PASS" -passout "pass:$PASS" <<EOF
+${C}
+${ST}
+${L}
+${O}
+${OU}
+${CN}
+$USER@${CN}
+.
+.
+EOF
+
+
+
+elif [[ $OP == "server" && -n "$CA_CERT" && -n "$PFX" && -n "$CN" ]]; then
+    HOST_CERT_CONFIG_PATH="${PFX}host_cert.cnf"
+    HOST_PRIVATE_RSA_KEY_PATH="${PFX}host_private_key_rsa.pem"
+    HOST_PRIVATE_KEY_PATH="${PFX}private_key.pem"
+    HOST_CSR_PATH="${PFX}host_csr.pem"
+    HOST_CERT_PATH="${PFX}host_cert.pem"
+    HOST_CERT_CHAIN_PATH="${PFX}client_${CN}.pem"
+
+    # Create the CA cert config file
+    echo "Setting up host certs..."
+
+    cat <<EOF > "${HOST_CERT_CONFIG_PATH}"
+[req]
+default_bits        = 2048
+distinguished_name  = req_distinguished_name
+req_extensions      = v3_req
+prompt              = no
+[req_distinguished_name]
+C   = ${C}
+ST  = ${ST}
+L   = ${L}
+O   = ${O}
+CN  = ${CN}
+[v3_req]
+subjectAltName = @alt_names
+[alt_names]
+DNS.1  = ${CN}
+DNS.2  = localhost.
+EOF
+
+    #Step 1
+    echo "############ Generating key"
+    openssl genrsa -out "${HOST_PRIVATE_RSA_KEY_PATH}" 2048
+    openssl pkcs8 -nocrypt -topk8 -v1 PBE-SHA1-RC4-128 -inform pem -outform pem -in "${HOST_PRIVATE_RSA_KEY_PATH}" -out "${HOST_PRIVATE_KEY_PATH}"
+
+    #Step 2
+    echo "############ Generate the CSR"
+    openssl req -nodes -new -extensions v3_req -sha256 -config "${HOST_CERT_CONFIG_PATH}" -key "${HOST_PRIVATE_KEY_PATH}" -out "${HOST_CSR_PATH}"
+
+    #Step 3
+    echo "############ Generate the cert"
+    openssl x509 -req -in "${HOST_CSR_PATH}" -CA "${CA_CERT}" -CAkey "${CA_CERT}.key" -CAcreateserial -out "${HOST_CERT_PATH}" -days ${VALIDITY} -sha256 -extensions v3_req -extfile "${HOST_CERT_CONFIG_PATH}" -passin "pass:${PASS}"
+
+    cat "${HOST_CERT_PATH}" > "${HOST_CERT_CHAIN_PATH}"
+
+
+elif [[ $OP == "client" && -n "$CA_CERT" && -n "$PFX" && -n "$CN" ]]; then
+
+# Standard OpenSSL keys
+	echo "############ Generating key"
+	openssl genrsa -nodes -passout "pass:${PASS}" -out "${PFX}client.key" 2048 
+
+	echo "############ Generating request"
+	openssl req -passin "pass:${PASS}" -passout "pass:${PASS}" -key "${PFX}client.key" -new -out "${PFX}client.req" \
+		<<EOF
+$C
+$ST
+$L
+$O
+$OU
+$CN
+.
+$PASS
+.
+EOF
+
+	echo "########### Signing key"
+	openssl x509 -req -passin "pass:${PASS}" -in "${PFX}client.req" -CA "${CA_CERT}" -CAkey "${CA_CERT}.key" -CAcreateserial -out "${PFX}client.pem" -days ${VALIDITY}
+
+
+else
+    echo "Usage: $0 ca <ca-cert-file> <CN>"
+    echo "       $0 server|client <ca-cert-file> <file_prefix> <hostname>"
+    echo ""
+    exit 1
+fi
+

docker/Dockerfile

@@ -15,6 +15,7 @@ RUN apt-get update && \
     libfl-dev \
     libbenchmark-dev \
     libgmock-dev \
+    libgtest-dev \
     libz-dev \
     && apt clean \
     && rm -rf /var/lib/apt/lists/*

etc/cmake/BMQTest.cmake

@@ -1,89 +1,10 @@
 # This module provides functions to support generating test targets compatible
 # with BlazingMQ CI.
 #
-# add_bmq_test( TARGET )
+# bmq_add_application_test( TARGET )
 
 include_guard()
 
-# :: bmq_add_test :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-# This function searches for the test drivers of a UOR-style TARGET within the
-# `tests` directory of each package. For each component, it generates a target
-# named ${UOR_component}.t for each component found.
-function(bmq_add_test target)
-  cmake_parse_arguments(PARSE_ARGV 1
-    ""
-    "SKIP_TESTS;NO_GEN_BDE_METADATA;NO_EMIT_PKG_CONFIG_FILE;COMPAT"
-    "SOURCE_DIR"
-    "CUSTOM_PACKAGES")
-
-  find_package(BdeBuildSystem REQUIRED)
-
-  # Get the name of the unit from the target
-  get_target_property(uor_name ${target} NAME)
-
-  # Use the current source directory if none is specified
-  if(NOT _SOURCE_DIR)
-    set(_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
-  endif()
-
-  # Check that BDE metadata exists and load it
-  if(NOT DEFINED ${uor_name}_PACKAGES)
-    if(EXISTS ${_SOURCE_DIR}/group)
-      bbs_read_metadata(GROUP ${uor_name}
-        SOURCE_DIR ${_SOURCE_DIR}
-        CUSTOM_PACKAGES "${_CUSTOM_PACKAGES}")
-    else()
-      if(EXISTS ${_SOURCE_DIR}/package)
-        bbs_read_metadata(PACKAGE ${uor_name}
-          SOURCE_DIR ${_SOURCE_DIR})
-      endif()
-    endif()
-  endif()
-
-  # Each package in the groups
-  if(${uor_name}_PACKAGES)
-    foreach(pkg ${${uor_name}_PACKAGES})
-      bbs_configure_target_tests(${pkg}
-        SOURCES ${${pkg}_TEST_SOURCES}
-        TEST_DEPS ${${pkg}_DEPENDS}
-        ${${pkg}_TEST_DEPENDS}
-        ${${uor_name}_PCDEPS}
-        ${${uor_name}_TEST_PCDEPS}
-        LABELS "unit;all" ${target} ${pkg})
-    endforeach()
-
-    set(import_test_deps ON)
-
-    foreach(pkg ${${uor_name}_PACKAGES})
-      if(${pkg}_TEST_TARGETS)
-        if(NOT TARGET ${target}.t)
-          add_custom_target(${target}.t)
-        endif()
-
-        add_dependencies(${target}.t ${${pkg}_TEST_TARGETS})
-
-        if(import_test_deps)
-          # Import UOR test dependencies only once and only if we have at least
-          # one generated test target
-          bbs_import_target_dependencies(${target} ${${uor_name}_TEST_PCDEPS})
-          set(import_test_deps OFF)
-        endif()
-      endif()
-    endforeach()
-  else()
-    # Configure standalone library ( no packages ) and tests from BDE metadata
-    bbs_configure_target_tests(${target}
-      SOURCES ${${uor_name}_TEST_SOURCES}
-      TEST_DEPS ${${uor_name}_PCDEPS}
-      ${${uor_name}_TEST_PCDEPS}
-      LABELS "unit;all" ${target})
-
-    if(${target}_TEST_TARGETS)
-      bbs_import_target_dependencies(${target} ${${uor_name}_TEST_PCDEPS})
-    endif()
-  endif()
-endfunction()
-
 # :: bmq_add_application_test :::::::::::::::::::::::::::::::::::::::::::::::::
 # This function searches for the test drivers of an 'application' TARGET.  It
 # expects existence of intermediate library '${uor_name}_lib' which is created

etc/cmake/BmqPackageProvider.cmake

@@ -22,5 +22,6 @@ macro(setup_package_provider)
 
         find_package(GTest CONFIG REQUIRED)
         add_library(gmock ALIAS GTest::gmock)
+        add_library(gtest ALIAS GTest::gtest)
     endif()
 endmacro()

etc/cmake/TargetBMQStyleUor.cmake

@@ -59,7 +59,6 @@ function(target_bmq_style_uor TARGET)
   add_library(${TARGET}-flags INTERFACE IMPORTED)
 
   bbs_setup_target_uor(${TARGET}
-    SKIP_TESTS
     PRIVATE_PACKAGES "${_PRIVATE_PACKAGES}")
 
   get_target_property(uor_name ${TARGET} NAME)
@@ -84,9 +83,6 @@ function(target_bmq_style_uor TARGET)
       target_link_libraries(${pkg}-iface PUBLIC ${TARGET}-flags)
     endforeach()
   endif()
-
-  include(BMQTest)
-  bmq_add_test(${TARGET} COMPAT)
 endfunction()
 
 # :: bmq_install_target_headers ::::::::::::::::::::::::::::::::::::::::::::::::

src/applications/bmqbrkr/etc/bmqbrkrcfg.json

@@ -86,11 +86,28 @@
                 "highWatermark": 1073741824,
                 "nodeLowWatermark": 5242880,
                 "nodeHighWatermark": 10485760,
-                "heartbeatIntervalMs": 3000
+                "heartbeatIntervalMs": 3000,
+                "listeners": [
+                    {
+                        "name": "TCPListener",
+                        "port": 30114,
+                        "tls": false
+                    },
+                    {
+                        "name": "TLSListener",
+                        "port": 30115,
+                        "tls": true
+                    }
+                ]
             }
         },
         "bmqconfConfig": {
             "cacheTTLSeconds": 30
+        },
+        "tlsConfig": {
+            "certificateAuthority": "/blazingmq/certs/ca-cert",
+            "certificate": "/blazingmq/certs/broker_host_cert.pem",
+            "key": "/blazingmq/certs/broker_private_key.pem"
         }
     }
 }

src/applications/bmqtool/bmqtool.m.cpp

@@ -204,6 +204,18 @@ static bool parseArgs(Parameters* parameters, int argc, const char* argv[])
          "address and port of the broker",
          balcl::TypeInfo(&params.broker()),
          balcl::OccurrenceInfo(params.broker())},
+        {"tls-authority",
+         "tlsAuthority",
+         "Path to the certificate authority FILE for TLS mode."
+         "The empty string value means that TLS is disabled, "
+         "non-empty string value means that TLS is enabled",
+         balcl::TypeInfo(&params.tlsAuthority()),
+         balcl::OccurrenceInfo(params.tlsAuthority())},
+        {"tls-versions",
+         "tlsVersions",
+         "TLS protocol versions, has effect only in TLS mode",
+         balcl::TypeInfo(&params.tlsVersions()),
+         balcl::OccurrenceInfo(params.tlsVersions())},
         {"q|queueuri",
          "uri",
          "URI of the queue (for auto/syschk modes)",

src/applications/bmqtool/bmqtoolcmd.xsd

@@ -223,6 +223,8 @@
     <sequence>
       <element name='mode'                     type='string'  default="cli"/>
       <element name='broker'                   type='string'  default="tcp://localhost:30114"/>
+      <element name='tlsAuthority'             type='string'  default=""/>
+      <element name='tlsVersions'              type='string'  default="TLSv1.3"/>
       <element name='queueUri'                 type='string'  default=""/>
       <element name='queueFlags'               type='string'  default=""/>
       <element name='latency'                  type='string'  default="none"/>

src/applications/bmqtool/m_bmqtool_application.cpp

@@ -564,6 +564,11 @@ int Application::initialize()
         .setNumProcessingThreads(d_parameters.numProcessingThreads())
         .configureEventQueue(1000, 10 * 1000);
 
+    if (!d_parameters.tlsAuthority().empty()) {
+        options.setTlsDetails(d_parameters.tlsAuthority(),
+                              d_parameters.tlsVersions());
+    }
+
     // Create the session
     if (d_parameters.noSessionEventHandler()) {
         d_session_mp.load(new (*d_allocator_p)