Skip to content

Commit cb68f2a

Browse files
committed
Support TLS listeners
Added ===== - TLS configuration in broker config - Helper script for generating test certs and CAs - TLS options for NtcChannel - Loading certificates and authority data specified from bmqbrkrcfg.json - SessionOptions to bmq package for configuring client sessions - --tls-authority and --tls-version options to bmqtool to configure session options - Client sessions will now require broker TLS sessions when TLS protocol versions are specified - Create CertificateStore component for bmqio - Integration tests for TLS Changed ======= - Update ntf-core and bde dependencies Signed-off-by: Taylor Foxhall <[email protected]> Signed-off-by: Evgeny Malygin <[email protected]>
1 parent 5a6670d commit cb68f2a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+4202
-1335
lines changed

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,5 @@ src/applications/bmqbrkr/etc/etc
3535

3636
# 'sim_cpp11_features.pl' backups
3737
*.bak
38+
/venv
39+

CMakePresets.json

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,42 @@
1717
},
1818
{
1919
"name": "macos-arm64-vcpkg",
20-
"description": "VCPKG based configuration for building on arm-based MacOS",
21-
"toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
20+
"description":
21+
"VCPKG based configuration for building on arm-based MacOS",
22+
"toolchainFile":
23+
"$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
2224
"inherits": "base",
2325
"cacheVariables": {
2426
"VCPKG_INSTALL_OPTIONS": "--allow-unsupported",
2527
"FLEX_INCLUDE_DIR": "/opt/homebrew/opt/flex/include"
2628
}
2729
},
30+
{
31+
"name": "macos-arm64-darwin",
32+
"description":
33+
"build-darwin.sh based configuration for building on arm-based MacOS",
34+
"toolchainFile":
35+
"${sourceDir}/thirdparty/bde-tools/BdeBuildSystem/toolchains/darwin/clang-default.cmake",
36+
"inherits": "base",
37+
"environment": {
38+
"BREW_PKG_CONFIG_PATH":
39+
"/opt/homebrew/lib/pkgconfig:/opt/homebrew/opt/zlib/lib/pkgconfig:/opt/homebrew/opt/googletest/lib/pkgconfig",
40+
"PKG_CONFIG_PATH":
41+
"${sourceDir}/install/lib/pkgconfig:$env{BREW_PKG_CONFIG_PATH}"
42+
},
43+
"cacheVariables": {
44+
"FLEX_ROOT": "/opt/homebrew/opt/flex",
45+
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install",
46+
"CMAKE_PREFIX_PATH":
47+
"${sourceDir}/install;${sourceDir}/thirdparty/bde-tools/BdeBuildSystem"
48+
}
49+
},
2850
{
2951
"name": "macos-x64-vcpkg",
30-
"description": "VCPKG based configuration for building on x86_64-based MacOS",
31-
"toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
52+
"description":
53+
"VCPKG based configuration for building on x86_64-based MacOS",
54+
"toolchainFile":
55+
"$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
3256
"inherits": "base",
3357
"cacheVariables": {
3458
"VCPKG_INSTALL_OPTIONS": "--allow-unsupported",
@@ -37,8 +61,10 @@
3761
},
3862
{
3963
"name": "linux-x64-vcpkg",
40-
"description": "VCPKG based configuration for building on x86_64-based Linux",
41-
"toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
64+
"description":
65+
"VCPKG based configuration for building on x86_64-based Linux",
66+
"toolchainFile":
67+
"$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
4268
"inherits": "base"
4369
}
4470
]

bin/build-darwin.sh

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,19 +33,28 @@ mkdir -p "${DIR_THIRDPARTY}"
3333
DIR_BUILD="${DIR_BUILD:-${DIR_ROOT}/build}"
3434
mkdir -p "${DIR_BUILD}"
3535

36-
DIR_INSTALL="${DIR_INSTALL:-${DIR_ROOT}}"
36+
DIR_INSTALL="${DIR_INSTALL:-${DIR_ROOT}/install}"
3737
mkdir -p "${DIR_INSTALL}"
3838

3939

4040
# :: Clone dependencies :::::::::::::::::::::::::::::::::::::::::::::::::::::::
4141
if [ ! -d "${DIR_THIRDPARTY}/bde-tools" ]; then
42-
git clone --depth 1 --branch 4.8.0.0 https://github.com/bloomberg/bde-tools "${DIR_THIRDPARTY}/bde-tools"
42+
git clone --depth 1 https://github.com/bloomberg/bde-tools "${DIR_THIRDPARTY}/bde-tools"
43+
pushd "${DIR_THIRDPARTY}/bde-tools"
44+
git reset --hard 964f78d36577ef1643b9074412608df4850e5b33 # 4.17.0.0
45+
popd
4346
fi
4447
if [ ! -d "${DIR_THIRDPARTY}/bde" ]; then
45-
git clone --depth 1 --branch 4.8.0.0 https://github.com/bloomberg/bde.git "${DIR_THIRDPARTY}/bde"
48+
git clone --depth 1 https://github.com/bloomberg/bde.git "${DIR_THIRDPARTY}/bde"
49+
pushd "${DIR_THIRDPARTY}/bde"
50+
git reset --hard 4.18.0.0 # 4.18.0.0
51+
popd
4652
fi
4753
if [ ! -d "${DIR_THIRDPARTY}/ntf-core" ]; then
48-
git clone --depth 1 --branch 2.4.2 https://github.com/bloomberg/ntf-core.git "${DIR_THIRDPARTY}/ntf-core"
54+
git clone --depth 1 https://github.com/bloomberg/ntf-core.git "${DIR_THIRDPARTY}/ntf-core"
55+
pushd "${DIR_THIRDPARTY}/ntf-core"
56+
git reset --hard 2.5.4
57+
popd
4958
fi
5059

5160

bin/build-ubuntu.sh

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,19 +75,28 @@ mkdir -p "${DIR_THIRDPARTY}"
7575
DIR_BUILD="${DIR_BUILD:-${DIR_ROOT}/build}"
7676
mkdir -p "${DIR_BUILD}"
7777

78-
DIR_INSTALL="${DIR_INSTALL:-${DIR_ROOT}}"
78+
DIR_INSTALL="${DIR_INSTALL:-${DIR_ROOT}/install}"
7979
mkdir -p "${DIR_INSTALL}"
8080

8181
# :: Clone dependencies :::::::::::::::::::::::::::::::::::::::::::::::::::::::
8282

8383
if [ ! -d "${DIR_THIRDPARTY}/bde-tools" ]; then
84-
git clone --depth 1 --branch 4.8.0.0 https://github.com/bloomberg/bde-tools "${DIR_THIRDPARTY}/bde-tools"
84+
git clone --depth 1 https://github.com/bloomberg/bde-tools "${DIR_THIRDPARTY}/bde-tools"
85+
pushd "${DIR_THIRDPARTY}/bde-tools"
86+
git reset --hard 964f78d36577ef1643b9074412608df4850e5b33 # 4.17.0.0
87+
popd
8588
fi
8689
if [ ! -d "${DIR_THIRDPARTY}/bde" ]; then
87-
git clone --depth 1 --branch 4.8.0.0 https://github.com/bloomberg/bde.git "${DIR_THIRDPARTY}/bde"
90+
git clone --depth 1 https://github.com/bloomberg/bde.git "${DIR_THIRDPARTY}/bde"
91+
pushd "${DIR_THIRDPARTY}/bde"
92+
git reset --hard ec094b4454738c311482fc54c3bde1d21c9f6893 # 4.18.0.0
93+
popd
8894
fi
8995
if [ ! -d "${DIR_THIRDPARTY}/ntf-core" ]; then
90-
git clone --depth 1 --branch 2.4.2 https://github.com/bloomberg/ntf-core.git "${DIR_THIRDPARTY}/ntf-core"
96+
git clone --depth 1 https://github.com/bloomberg/ntf-core.git "${DIR_THIRDPARTY}/ntf-core"
97+
pushd "${DIR_THIRDPARTY}/ntf-core"
98+
git reset --hard 2.5.4
99+
popd
91100
fi
92101
# prometheus-cpp and its dependency for the plugin
93102
if [ "${BUILD_PROMETHEUS}" == true ]; then

bin/gen-tls-certs.sh

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/bin/bash
2+
#
3+
#
4+
# This scripts generates:
5+
# - root CA certificate
6+
# - server certificate and keystore
7+
# - client keys
8+
#
9+
# Based off of
10+
# https://github.com/confluentinc/librdkafka/blob/master/tests/gen-ssl-certs.sh
11+
12+
OP="$1"
13+
CA_CERT="$2"
14+
PFX="$3"
15+
HOST="$4"
16+
17+
C=NN
18+
ST=NN
19+
L=NN
20+
O=NN
21+
OU=NN
22+
CN="$HOST"
23+
24+
25+
# Password
26+
PASS="secret"
27+
28+
# Cert validity, in days
29+
VALIDITY=10000
30+
31+
set -e
32+
33+
export LC_ALL=C
34+
35+
if [[ $OP == "ca" && -n "$CA_CERT" && -n "$3" ]]; then
36+
CN="$3"
37+
openssl req -new -x509 -keyout "${CA_CERT}.key" -out "${CA_CERT}" -days $VALIDITY -passin "pass:$PASS" -passout "pass:$PASS" <<EOF
38+
${C}
39+
${ST}
40+
${L}
41+
${O}
42+
${OU}
43+
${CN}
44+
$USER@${CN}
45+
.
46+
.
47+
EOF
48+
49+
50+
51+
elif [[ $OP == "server" && -n "$CA_CERT" && -n "$PFX" && -n "$CN" ]]; then
52+
HOST_CERT_CONFIG_PATH="${PFX}host_cert.cnf"
53+
HOST_PRIVATE_RSA_KEY_PATH="${PFX}host_private_key_rsa.pem"
54+
HOST_PRIVATE_KEY_PATH="${PFX}private_key.pem"
55+
HOST_CSR_PATH="${PFX}host_csr.pem"
56+
HOST_CERT_PATH="${PFX}host_cert.pem"
57+
HOST_CERT_CHAIN_PATH="${PFX}client_${CN}.pem"
58+
59+
# Create the CA cert config file
60+
echo "Setting up host certs..."
61+
62+
cat <<EOF > "${HOST_CERT_CONFIG_PATH}"
63+
[req]
64+
default_bits = 2048
65+
distinguished_name = req_distinguished_name
66+
req_extensions = v3_req
67+
prompt = no
68+
[req_distinguished_name]
69+
C = ${C}
70+
ST = ${ST}
71+
L = ${L}
72+
O = ${O}
73+
CN = ${CN}
74+
[v3_req]
75+
subjectAltName = @alt_names
76+
[alt_names]
77+
DNS.1 = ${CN}
78+
DNS.2 = localhost.
79+
EOF
80+
81+
#Step 1
82+
echo "############ Generating key"
83+
openssl genrsa -out "${HOST_PRIVATE_RSA_KEY_PATH}" 2048
84+
openssl pkcs8 -nocrypt -topk8 -v1 PBE-SHA1-RC4-128 -inform pem -outform pem -in "${HOST_PRIVATE_RSA_KEY_PATH}" -out "${HOST_PRIVATE_KEY_PATH}"
85+
86+
#Step 2
87+
echo "############ Generate the CSR"
88+
openssl req -nodes -new -extensions v3_req -sha256 -config "${HOST_CERT_CONFIG_PATH}" -key "${HOST_PRIVATE_KEY_PATH}" -out "${HOST_CSR_PATH}"
89+
90+
#Step 3
91+
echo "############ Generate the cert"
92+
openssl x509 -req -in "${HOST_CSR_PATH}" -CA "${CA_CERT}" -CAkey "${CA_CERT}.key" -CAcreateserial -out "${HOST_CERT_PATH}" -days ${VALIDITY} -sha256 -extensions v3_req -extfile "${HOST_CERT_CONFIG_PATH}" -passin "pass:${PASS}"
93+
94+
cat "${HOST_CERT_PATH}" > "${HOST_CERT_CHAIN_PATH}"
95+
96+
97+
elif [[ $OP == "client" && -n "$CA_CERT" && -n "$PFX" && -n "$CN" ]]; then
98+
99+
# Standard OpenSSL keys
100+
echo "############ Generating key"
101+
openssl genrsa -nodes -passout "pass:${PASS}" -out "${PFX}client.key" 2048
102+
103+
echo "############ Generating request"
104+
openssl req -passin "pass:${PASS}" -passout "pass:${PASS}" -key "${PFX}client.key" -new -out "${PFX}client.req" \
105+
<<EOF
106+
$C
107+
$ST
108+
$L
109+
$O
110+
$OU
111+
$CN
112+
.
113+
$PASS
114+
.
115+
EOF
116+
117+
echo "########### Signing key"
118+
openssl x509 -req -passin "pass:${PASS}" -in "${PFX}client.req" -CA "${CA_CERT}" -CAkey "${CA_CERT}.key" -CAcreateserial -out "${PFX}client.pem" -days ${VALIDITY}
119+
120+
121+
else
122+
echo "Usage: $0 ca <ca-cert-file> <CN>"
123+
echo " $0 server|client <ca-cert-file> <file_prefix> <hostname>"
124+
echo ""
125+
exit 1
126+
fi
127+

docker/build_deps.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,9 @@ fetch_git() {
2828
}
2929

3030
fetch_deps() {
31-
fetch_git bloomberg bde-tools 4.8.0.0
32-
fetch_git bloomberg bde 4.8.0.0
33-
fetch_git bloomberg ntf-core 2.4.2
31+
fetch_git bloomberg bde-tools 4.13.0.0
32+
fetch_git bloomberg bde 4.18.0.0
33+
fetch_git bloomberg ntf-core 2.5.4
3434
}
3535

3636
configure() {

docs/docs/features/tls.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# TLS in BlazingMQ
2+
3+
BlazingMQ supports authenticating brokers using TLS.
4+
5+
6+
## Generating Test Certs
7+
8+
```sh
9+
mkdir -p certs && cd certs
10+
../bin/gen-tls-certs.sh ca ca-cert blazingmq
11+
../bin/gen-tls-certs.sh server ca-cert broker_bmqc00_ bmqc00
12+
```

src/applications/bmqbrkr/etc/bmqbrkrcfg.json

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88
"rotationBytes": 268435456,
99
"logfileFormat": "%d (%t) %s %F:%l %m\n\n",
1010
"consoleFormat": "%d (%t) %s %F:%l %m\n",
11-
"loggingVerbosity": "INFO",
12-
"consoleSeverityThreshold": "INFO",
11+
"loggingVerbosity": "TRACE",
12+
"consoleSeverityThreshold": "TRACE",
1313
"categories": [
1414
"BMQBRKR:INFO:green",
1515
"BMQ*:INFO:green",
@@ -88,11 +88,29 @@
8888
"highWatermark": 1073741824,
8989
"nodeLowWatermark": 5242880,
9090
"nodeHighWatermark": 10485760,
91-
"heartbeatIntervalMs": 3000
91+
"heartbeatIntervalMs": 3000,
92+
"listeners": [
93+
{
94+
"name": "TCPListener",
95+
"port": 30114,
96+
"tls": false
97+
},
98+
{
99+
"name": "TLSListener",
100+
"port": 30115,
101+
"tls": true
102+
}
103+
]
92104
}
93105
},
94106
"bmqconfConfig": {
95107
"cacheTTLSeconds": 30
108+
},
109+
"tlsConfig": {
110+
"certificateAuthority": "/blazingmq/certs/ca-cert",
111+
"certificate": "/blazingmq/certs/broker_host_cert.pem",
112+
"key": "/blazingmq/certs/broker_private_key.pem",
113+
"version": ""
96114
}
97115
}
98116
}

src/applications/bmqtool/bmqtool.m.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,18 @@ static bool parseArgs(Parameters* parameters, int argc, const char* argv[])
168168
"address and port of the broker",
169169
balcl::TypeInfo(&params.broker()),
170170
balcl::OccurrenceInfo(params.broker())},
171+
{"tls-authority",
172+
"tlsAuthority",
173+
"Path to the certificate authority for TLS mode."
174+
"The empty string value means that TLS is disabled, "
175+
"non-empty string value means that TLS is enabled",
176+
balcl::TypeInfo(&params.tlsAuthority()),
177+
balcl::OccurrenceInfo(params.tlsAuthority())},
178+
{"tls-versions",
179+
"tlsVersions",
180+
"TLS protocol versions, has effect only in TLS mode",
181+
balcl::TypeInfo(&params.tlsVersions()),
182+
balcl::OccurrenceInfo(params.tlsVersions())},
171183
{"q|queueuri",
172184
"uri",
173185
"URI of the queue (for auto/syschk modes)",

src/applications/bmqtool/bmqtoolcmd.xsd

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,8 @@
223223
<sequence>
224224
<element name='mode' type='string' default="cli"/>
225225
<element name='broker' type='string' default="tcp://localhost:30114"/>
226+
<element name='tlsAuthority' type='string' default=""/>
227+
<element name='tlsVersions' type='string' default="TLSv1.3"/>
226228
<element name='queueUri' type='string' default=""/>
227229
<element name='queueFlags' type='string' default=""/>
228230
<element name='latency' type='string' default="none"/>

0 commit comments

Comments
 (0)