From ec22f535c9e22c0cad26f4c2fcb437ec3c444fda Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 25 Jul 2022 14:29:48 +0200 Subject: [PATCH 01/37] started adding tests --- tests/templates/kuttl/tls/10-assert.yaml | 14 ++++ .../kuttl/tls/10-install-zookeeper.yaml.j2 | 48 ++++++++++++ tests/templates/kuttl/tls/20-assert.yaml | 14 ++++ .../kuttl/tls/20-install-kafka.yaml.j2 | 20 +++++ tests/templates/kuttl/tls/30-assert.yaml | 7 ++ .../kuttl/tls/30-prepare-test-kafka.yaml | 5 ++ tests/templates/kuttl/tls/test_tls.sh.j2 | 73 +++++++++++++++++++ 7 files changed, 181 insertions(+) create mode 100644 tests/templates/kuttl/tls/10-assert.yaml create mode 100644 tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 create mode 100644 tests/templates/kuttl/tls/20-assert.yaml create mode 100644 tests/templates/kuttl/tls/20-install-kafka.yaml.j2 create mode 100644 tests/templates/kuttl/tls/30-assert.yaml create mode 100644 tests/templates/kuttl/tls/30-prepare-test-kafka.yaml create mode 100755 tests/templates/kuttl/tls/test_tls.sh.j2 diff --git a/tests/templates/kuttl/tls/10-assert.yaml b/tests/templates/kuttl/tls/10-assert.yaml new file mode 100644 index 00000000..a1583b3e --- /dev/null +++ b/tests/templates/kuttl/tls/10-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-test-zk +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-zk-server-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 b/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 new file mode 100644 index 00000000..d2dce8b8 --- /dev/null +++ b/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 @@ -0,0 +1,48 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: test-zk +spec: + version: {{ test_scenario['values']['zookeeper'] }} + config: +{% if test_scenario['values']['use-client-tls'] == 'true' %} + tls: + secretClass: tls + clientAuthentication: + authenticationClass: zk-client-tls +{% else %} + tls: null +{% endif %} +{% if test_scenario['values']['use-internal-tls'] == 'true' %} + internalTlsSecretClass: tls +{% endif %} + servers: + roleGroups: + default: + replicas: 1 + +{% if test_scenario['values']['use-client-tls'] == 'true' %} +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: zk-client-tls +spec: + provider: + tls: + clientCertSecretClass: zk-client-auth-secret +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: zk-client-auth-secret +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-zk-client-ca + namespace: default + autoGenerate: true +{% endif %} diff --git a/tests/templates/kuttl/tls/20-assert.yaml b/tests/templates/kuttl/tls/20-assert.yaml new file mode 100644 index 00000000..8dd80887 --- /dev/null +++ b/tests/templates/kuttl/tls/20-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-test-kafka +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-broker-default +status: + readyReplicas: 2 + replicas: 2 diff --git a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 new file mode 100644 index 00000000..cd6d0ea7 --- /dev/null +++ b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: test-kafka-znode +spec: + clusterRef: + name: test-zk +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: simple-kafka +spec: + version: {{ test_scenario['values']['kafka'] }} + zookeeperConfigMapName: test-kafka-znode + brokers: + roleGroups: + default: + replicas: 2 diff --git a/tests/templates/kuttl/tls/30-assert.yaml b/tests/templates/kuttl/tls/30-assert.yaml new file mode 100644 index 00000000..6d3ab737 --- /dev/null +++ b/tests/templates/kuttl/tls/30-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-tls +commands: + - script: kubectl exec -n $NAMESPACE test-zk-server-primary-0 -- /tmp/test_tls.sh $NAMESPACE diff --git a/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml b/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml new file mode 100644 index 00000000..2bc6aebc --- /dev/null +++ b/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl cp -n $NAMESPACE ./test_tls.sh test-kafka-broker-default-0:/tmp diff --git a/tests/templates/kuttl/tls/test_tls.sh.j2 b/tests/templates/kuttl/tls/test_tls.sh.j2 new file mode 100755 index 00000000..0c88e51f --- /dev/null +++ b/tests/templates/kuttl/tls/test_tls.sh.j2 @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Usage: test_tls.sh namespace + +NAMESPACE=$1 + +{% if test_scenario['values']['use-client-tls'] == 'true' %} +SERVER="test-zk-server-primary-1.test-zk-server-primary.${NAMESPACE}.svc.cluster.local:2282" +{% else %} +SERVER="test-zk-server-primary-1.test-zk-server-primary.${NAMESPACE}.svc.cluster.local:2181" +{% endif %} + +# just to be safe... +unset QUORUM_STORE_SECRET +unset CLIENT_STORE_SECRET +unset CLIENT_JVMFLAGS + +echo "Start TLS testing..." +############################################################################ +# Test the plaintext unsecured connection +############################################################################ +if ! /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; +then + echo "[ERROR] Could not establish unsecure connection!" + exit 1 +fi +echo "[SUCCESS] Unsecure client connection established!" + +{% if test_scenario['values']['use-client-tls'] == 'true' %} +############################################################################ +# We set the correct client tls credentials and expect to be able to connect +############################################################################ +CLIENT_STORE_SECRET="$(< /stackable/rwconfig/zoo.cfg grep "ssl.keyStore.password" | cut -d "=" -f2)" +export CLIENT_STORE_SECRET +export CLIENT_JVMFLAGS=" +-Dzookeeper.authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider +-Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty +-Dzookeeper.client.secure=true +-Dzookeeper.ssl.keyStore.location=/stackable/tls/client/keystore.p12 +-Dzookeeper.ssl.keyStore.password=${CLIENT_STORE_SECRET} +-Dzookeeper.ssl.trustStore.location=/stackable/tls/client/truststore.p12 +-Dzookeeper.ssl.trustStore.password=${CLIENT_STORE_SECRET}" + +if ! /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; +then + echo "[ERROR] Could not establish secure connection using client certificates!" + exit 1 +fi +echo "[SUCCESS] Secure and authenticated client connection established!" + +{% endif %} +############################################################################ +# We set the (wrong) quorum tls credentials and expect to fail (wrong certificate) +############################################################################ +QUORUM_STORE_SECRET="$(< /stackable/rwconfig/zoo.cfg grep "ssl.quorum.keyStore.password" | cut -d "=" -f2)" +export QUORUM_STORE_SECRET +export CLIENT_JVMFLAGS=" +-Dzookeeper.authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider +-Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty +-Dzookeeper.client.secure=true +-Dzookeeper.ssl.keyStore.location=/stackable/tls/quorum/keystore.p12 +-Dzookeeper.ssl.keyStore.password=${QUORUM_STORE_SECRET} +-Dzookeeper.ssl.trustStore.location=/stackable/tls/quorum/truststore.p12 +-Dzookeeper.ssl.trustStore.password=${QUORUM_STORE_SECRET}" + +if /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; +then + echo "[ERROR] Could establish secure connection with quorum certificates (should not be happening)!" + exit 1 +fi +echo "[SUCCESS] Could not establish secure connection with (wrong) quorum certificates!" + +echo "All TLS tests successful!" +exit 0 From 8a5ecfde0655c14d933d3592142eba5b585415d2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 25 Jul 2022 14:35:38 +0200 Subject: [PATCH 02/37] added tls structs and regenerated charts --- deploy/crd/kafkacluster.crd.yaml | 40 ++++++++++++++ deploy/helm/kafka-operator/crds/crds.yaml | 40 ++++++++++++++ deploy/manifests/crds.yaml | 40 ++++++++++++++ rust/crd/src/lib.rs | 65 +++++++++++++++++++++++ 4 files changed, 185 insertions(+) diff --git a/deploy/crd/kafkacluster.crd.yaml b/deploy/crd/kafkacluster.crd.yaml index bb41594f..4c8ac5fd 100644 --- a/deploy/crd/kafkacluster.crd.yaml +++ b/deploy/crd/kafkacluster.crd.yaml @@ -275,6 +275,46 @@ spec: required: - roleGroups type: object + config: + default: + tls: + secretClass: tls + internalTls: + secretClass: tls + nullable: true + properties: + clientAuthentication: + description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" + nullable: true + properties: + authenticationClass: + type: string + required: + - authenticationClass + type: object + internalTls: + default: + secretClass: tls + description: "Only affects internal communication. Use mutual verification between Kafka Broker Nodes (mandatory). This setting controls: - Which cert the brokers should use to authenticate themselves against other brokers - Which ca.crt to use when validating the other server" + nullable: true + properties: + secretClass: + type: string + required: + - secretClass + type: object + tls: + default: + secretClass: tls + description: "Only affects client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `TlsSecretClass` { secret_class: \"tls\".to_string() }." + nullable: true + properties: + secretClass: + type: string + required: + - secretClass + type: object + type: object log4j: nullable: true type: string diff --git a/deploy/helm/kafka-operator/crds/crds.yaml b/deploy/helm/kafka-operator/crds/crds.yaml index b1a307fb..cc688927 100644 --- a/deploy/helm/kafka-operator/crds/crds.yaml +++ b/deploy/helm/kafka-operator/crds/crds.yaml @@ -277,6 +277,46 @@ spec: required: - roleGroups type: object + config: + default: + tls: + secretClass: tls + internalTls: + secretClass: tls + nullable: true + properties: + clientAuthentication: + description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" + nullable: true + properties: + authenticationClass: + type: string + required: + - authenticationClass + type: object + internalTls: + default: + secretClass: tls + description: "Only affects internal communication. Use mutual verification between Kafka Broker Nodes (mandatory). This setting controls: - Which cert the brokers should use to authenticate themselves against other brokers - Which ca.crt to use when validating the other server" + nullable: true + properties: + secretClass: + type: string + required: + - secretClass + type: object + tls: + default: + secretClass: tls + description: "Only affects client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `TlsSecretClass` { secret_class: \"tls\".to_string() }." + nullable: true + properties: + secretClass: + type: string + required: + - secretClass + type: object + type: object log4j: nullable: true type: string diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index 31defb16..da4b0432 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -278,6 +278,46 @@ spec: required: - roleGroups type: object + config: + default: + tls: + secretClass: tls + internalTls: + secretClass: tls + nullable: true + properties: + clientAuthentication: + description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" + nullable: true + properties: + authenticationClass: + type: string + required: + - authenticationClass + type: object + internalTls: + default: + secretClass: tls + description: "Only affects internal communication. Use mutual verification between Kafka Broker Nodes (mandatory). This setting controls: - Which cert the brokers should use to authenticate themselves against other brokers - Which ca.crt to use when validating the other server" + nullable: true + properties: + secretClass: + type: string + required: + - secretClass + type: object + tls: + default: + secretClass: tls + description: "Only affects client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `TlsSecretClass` { secret_class: \"tls\".to_string() }." + nullable: true + properties: + secretClass: + type: string + required: + - secretClass + type: object + type: object log4j: nullable: true type: string diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index f3fcdfb7..c535a8e4 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -23,6 +23,7 @@ use strum::{Display, EnumIter, EnumString}; pub const APP_NAME: &str = "kafka"; pub const APP_PORT: u16 = 9092; pub const METRICS_PORT: u16 = 9606; +pub const TLS_DEFAULT_SECRET_CLASS: &str = "tls"; pub const SERVER_PROPERTIES_FILE: &str = "server.properties"; @@ -62,9 +63,73 @@ pub struct KafkaClusterSpec { pub zookeeper_config_map_name: String, pub opa: Option, pub log4j: Option, + #[serde( + default = "global_config_default", + skip_serializing_if = "Option::is_none" + )] + pub config: Option, pub stopped: Option, } +#[derive(Clone, Default, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GlobalKafkaConfig { + /// Only affects client connections. This setting controls: + /// - If TLS encryption is used at all + /// - Which cert the servers should use to authenticate themselves against the client + /// Defaults to `TlsSecretClass` { secret_class: "tls".to_string() }. + #[serde( + default = "tls_secret_class_default", + skip_serializing_if = "Option::is_none" + )] + pub tls: Option, + /// Only affects client connections. This setting controls: + /// - If clients need to authenticate themselves against the server via TLS + /// - Which ca.crt to use when validating the provided client certs + /// Defaults to `None` + #[serde(skip_serializing_if = "Option::is_none")] + pub client_authentication: Option, + /// Only affects internal communication. Use mutual verification between Kafka Broker Nodes + /// (mandatory). This setting controls: + /// - Which cert the brokers should use to authenticate themselves against other brokers + /// - Which ca.crt to use when validating the other server + #[serde( + default = "tls_secret_class_default", + skip_serializing_if = "Option::is_none" + )] + pub internal_tls: Option, +} + +fn global_config_default() -> Option { + Some(GlobalKafkaConfig { + tls: Some(TlsSecretClass { + secret_class: TLS_DEFAULT_SECRET_CLASS.to_string(), + }), + client_authentication: None, + internal_tls: Some(TlsSecretClass { + secret_class: TLS_DEFAULT_SECRET_CLASS.to_string(), + }), + }) +} + +#[derive(Clone, Default, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ClientAuthenticationClass { + pub authentication_class: String, +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TlsSecretClass { + pub secret_class: String, +} + +fn tls_secret_class_default() -> Option { + Some(TlsSecretClass { + secret_class: TLS_DEFAULT_SECRET_CLASS.to_string(), + }) +} + impl KafkaCluster { /// The name of the role-level load-balanced Kubernetes `Service` pub fn broker_role_service_name(&self) -> Option { From 7ca18c98a5a08029f4de2d975f5a72c31b9fb151 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 26 Jul 2022 19:44:26 +0200 Subject: [PATCH 03/37] adapted listeners, retrieving node ports to http and https ports --- examples/tls/simple-kafka-cluster.yaml | 38 +++++ rust/crd/src/lib.rs | 110 +++++++++++-- rust/operator/src/kafka_controller.rs | 196 ++++++++++++++++++++---- rust/operator/src/pod_svc_controller.rs | 7 +- 4 files changed, 305 insertions(+), 46 deletions(-) create mode 100644 examples/tls/simple-kafka-cluster.yaml diff --git a/examples/tls/simple-kafka-cluster.yaml b/examples/tls/simple-kafka-cluster.yaml new file mode 100644 index 00000000..1e617b79 --- /dev/null +++ b/examples/tls/simple-kafka-cluster.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: simple-zk +spec: + version: 3.8.0-stackable0.7.1 + servers: + roleGroups: + default: + selector: + matchLabels: + kubernetes.io/os: linux + replicas: 1 + config: {} +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: simple-kafka-znode +spec: + clusterRef: + name: simple-zk +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: simple-kafka +spec: + version: 3.2.0-stackable0.1.0 + zookeeperConfigMapName: simple-kafka-znode + config: + tls: + secretClass: tls + brokers: + roleGroups: + default: + replicas: 2 diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index c535a8e4..298116e5 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -21,14 +21,32 @@ use std::collections::BTreeMap; use strum::{Display, EnumIter, EnumString}; pub const APP_NAME: &str = "kafka"; -pub const APP_PORT: u16 = 9092; +// ports +pub const CLIENT_PORT_NAME: &str = "http"; +pub const CLIENT_PORT: u16 = 9092; +pub const SECURE_CLIENT_PORT_NAME: &str = "https"; +pub const SECURE_CLIENT_PORT: u16 = 9093; +pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: u16 = 9606; -pub const TLS_DEFAULT_SECRET_CLASS: &str = "tls"; - +// config files pub const SERVER_PROPERTIES_FILE: &str = "server.properties"; - +// env vars pub const KAFKA_HEAP_OPTS: &str = "KAFKA_HEAP_OPTS"; +// server_properties pub const LOG_DIRS_VOLUME_NAME: &str = "log-dirs"; +// TLS +pub const TLS_DEFAULT_SECRET_CLASS: &str = "tls"; +pub const SSL_KEYSTORE_LOCATION: &str = "ssl.keystore.location"; +pub const SSL_KEYSTORE_PASSWORD: &str = "ssl.keystore.password"; +pub const SSL_KEYSTORE_TYPE: &str = "ssl.keystore.type"; +pub const SSL_TRUSTSTORE_LOCATION: &str = "ssl.truststore.location"; +pub const SSL_TRUSTSTORE_PASSWORD: &str = "ssl.truststore.password"; +pub const SSL_TRUSTSTORE_TYPE: &str = "ssl.truststore.type"; +pub const SSL_STORE_PASSWORD: &str = "changeit"; +pub const SSL_CLIENT_AUTH: &str = "ssl.client.auth"; +// directories +pub const CLIENT_TLS_DIR: &str = "/stackable/tls/client"; +pub const INTERNAL_TLS_DIR: &str = "/stackable/tls/internal"; const JVM_HEAP_FACTOR: f32 = 0.8; @@ -266,6 +284,40 @@ impl KafkaCluster { image_version: image_version.to_string(), }) } + + pub fn client_port(&self) -> u16 { + if self.is_client_secure() { + SECURE_CLIENT_PORT + } else { + CLIENT_PORT + } + } + + /// Returns the secret class for client connection encryption. Defaults to `tls`. + pub fn client_tls_secret_class(&self) -> Option<&TlsSecretClass> { + let spec: &KafkaClusterSpec = &self.spec; + spec.config.as_ref().and_then(|c| c.tls.as_ref()) + } + + /// Checks if we should use TLS to encrypt client connections. + pub fn is_client_secure(&self) -> bool { + self.client_tls_secret_class().is_some() + } + + /// Returns the authentication class used for client authentication + pub fn client_tls_authentication_class(&self) -> Option { + let spec: &KafkaClusterSpec = &self.spec; + spec.config + .as_ref() + .and_then(|c| c.client_authentication.as_ref()) + .map(|tls| tls.authentication_class.clone()) + } + + /// Returns the secret class for internal server encryption + pub fn internal_tls_secret_class(&self) -> Option<&TlsSecretClass> { + let spec: &KafkaClusterSpec = &self.spec; + spec.config.as_ref().and_then(|c| c.internal_tls.as_ref()) + } } /// Reference to a single `Pod` that is a component of a [`KafkaCluster`] @@ -345,15 +397,47 @@ impl Configuration for KafkaConfig { ) -> Result>, ConfigError> { let mut config = BTreeMap::new(); - if resource.spec.opa.is_some() && file == SERVER_PROPERTIES_FILE { - config.insert( - "authorizer.class.name".to_string(), - Some("org.openpolicyagent.kafka.OpaAuthorizer".to_string()), - ); - config.insert( - "opa.authorizer.metrics.enabled".to_string(), - Some("true".to_string()), - ); + if file == SERVER_PROPERTIES_FILE { + // OPA + if resource.spec.opa.is_some() { + config.insert( + "authorizer.class.name".to_string(), + Some("org.openpolicyagent.kafka.OpaAuthorizer".to_string()), + ); + config.insert( + "opa.authorizer.metrics.enabled".to_string(), + Some("true".to_string()), + ); + } + // Client TLS + if resource.client_tls_secret_class().is_some() { + config.insert( + SSL_KEYSTORE_LOCATION.to_string(), + Some(format!("{}/keystore.p12", CLIENT_TLS_DIR)), + ); + config.insert( + SSL_KEYSTORE_PASSWORD.to_string(), + Some(SSL_KEYSTORE_PASSWORD.to_string()), + ); + config.insert(SSL_KEYSTORE_TYPE.to_string(), Some("PKCS12".to_string())); + config.insert( + SSL_TRUSTSTORE_LOCATION.to_string(), + Some(format!("{}/truststore.p12", CLIENT_TLS_DIR)), + ); + config.insert( + SSL_TRUSTSTORE_PASSWORD.to_string(), + Some(SSL_KEYSTORE_PASSWORD.to_string()), + ); + config.insert(SSL_TRUSTSTORE_TYPE.to_string(), Some("PKCS12".to_string())); + + // Authentication + if resource.client_tls_authentication_class().is_some() { + config.insert(SSL_CLIENT_AUTH.to_string(), Some("required".to_string())); + } + } + + // Internal TLS + if resource.internal_tls_secret_class().is_some() {} } Ok(config) diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index d41b7207..4a3de6f1 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -1,16 +1,10 @@ //! Ensures that `Pod`s are configured and running for each [`KafkaCluster`] -use std::{ - borrow::Cow, - collections::{BTreeMap, HashMap}, - sync::Arc, - time::Duration, -}; - use snafu::{OptionExt, ResultExt, Snafu}; use stackable_kafka_crd::{ - KafkaCluster, KafkaRole, APP_NAME, APP_PORT, KAFKA_HEAP_OPTS, LOG_DIRS_VOLUME_NAME, - METRICS_PORT, SERVER_PROPERTIES_FILE, + KafkaCluster, KafkaRole, APP_NAME, CLIENT_PORT, CLIENT_PORT_NAME, KAFKA_HEAP_OPTS, + LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, SECURE_CLIENT_PORT, + SECURE_CLIENT_PORT_NAME, SERVER_PROPERTIES_FILE, }; use stackable_operator::{ builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder}, @@ -19,9 +13,10 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, ConfigMapKeySelector, ConfigMapVolumeSource, EmptyDirVolumeSource, - EnvVar, EnvVarSource, ExecAction, ObjectFieldSelector, PodSpec, Probe, - SecurityContext, Service, ServiceAccount, ServicePort, ServiceSpec, Volume, + ConfigMap, ConfigMapKeySelector, ConfigMapVolumeSource, ContainerPort, + EmptyDirVolumeSource, EnvVar, EnvVarSource, ExecAction, ObjectFieldSelector, + PodSpec, Probe, SecurityContext, Service, ServiceAccount, ServicePort, ServiceSpec, + Volume, }, rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject}, }, @@ -37,6 +32,12 @@ use stackable_operator::{ product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, role_utils::RoleGroupRef, }; +use std::{ + borrow::Cow, + collections::{BTreeMap, HashMap}, + sync::Arc, + time::Duration, +}; use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ @@ -297,12 +298,7 @@ pub fn build_broker_role_service(kafka: &KafkaCluster) -> Result { ) .build(), spec: Some(ServiceSpec { - ports: Some(vec![ServicePort { - name: Some("kafka".to_string()), - port: APP_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }]), + ports: Some(service_ports(kafka)), selector: Some(role_selector_labels(kafka, APP_NAME, &role_name)), type_: Some("NodePort".to_string()), ..ServiceSpec::default() @@ -446,7 +442,7 @@ fn build_broker_rolegroup_service( ports: Some(vec![ ServicePort { name: Some("kafka".to_string()), - port: APP_PORT.into(), + port: CLIENT_PORT.into(), protocol: Some("TCP".to_string()), ..ServicePort::default() }, @@ -492,6 +488,23 @@ fn build_broker_rolegroup_statefulset( .context(KafkaVersionParseFailureSnafu)?; let image = format!("docker.stackable.tech/stackable/kafka:{}", image_version); + let get_svc_args = if kafka.client_tls_secret_class().is_some() + && kafka.internal_tls_secret_class().is_some() + { + format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee/stackable/tmp/{name}_nodeport", name = SECURE_CLIENT_PORT_NAME) + } else if kafka.client_tls_secret_class().is_some() + || kafka.internal_tls_secret_class().is_some() + { + [ + format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee/stackable/tmp/{name}_nodeport", name = CLIENT_PORT_NAME), + format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee/stackable/tmp/{name}_nodeport", name = SECURE_CLIENT_PORT_NAME), + ].join(" && ") + } + // If no is TLS specified the HTTP port is sufficient + else { + format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee/stackable/tmp/{name}_nodeport", name = CLIENT_PORT_NAME) + }; + let container_get_svc = ContainerBuilder::new("get-svc") .image("docker.stackable.tech/stackable/tools:0.2.0-stackable0.3.0") .command(vec!["bash".to_string()]) @@ -499,11 +512,7 @@ fn build_broker_rolegroup_statefulset( "-euo".to_string(), "pipefail".to_string(), "-c".to_string(), - [ - "kubectl get service \"$POD_NAME\" -o jsonpath='{.spec.ports[0].nodePort}'", - "tee /stackable/tmp/nodeport", - ] - .join(" | "), + get_svc_args, ]) .add_env_vars(vec![EnvVar { name: "POD_NAME".to_string(), @@ -608,8 +617,10 @@ fn build_broker_rolegroup_statefulset( let jvm_args = format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar={}:/stackable/jmx/broker.yaml", METRICS_PORT); let zookeeper_override = "--override \"zookeeper.connect=$ZOOKEEPER\""; - let advertised_listeners_override = - "--override \"advertised.listeners=PLAINTEXT://$NODE:$(cat /stackable/tmp/nodeport)\""; + let advertised_listeners_override = format!( + "--override \"advertised.listeners={}\"", + get_listeners(kafka) + ); let opa_url_override = opa_connect_string.map_or("".to_string(), |opa| { format!("--override \"opa.authorizer.url={}\"", opa) }); @@ -623,15 +634,14 @@ fn build_broker_rolegroup_statefulset( "bin/kafka-server-start.sh", &format!("/stackable/config/{}", SERVER_PROPERTIES_FILE), zookeeper_override, - advertised_listeners_override, + &advertised_listeners_override, &opa_url_override, ] .join(" "), ]) .add_env_vars(env) .add_env_var("EXTRA_ARGS", jvm_args) - .add_container_port("kafka", APP_PORT.into()) - .add_container_port("metrics", METRICS_PORT.into()) + .add_container_ports(container_ports(kafka)) .add_volume_mount(LOG_DIRS_VOLUME_NAME, "/stackable/data") .add_volume_mount("config", "/stackable/config") .add_volume_mount("tmp", "/stackable/tmp") @@ -651,7 +661,7 @@ fn build_broker_rolegroup_statefulset( command: Some(vec![ "kcat".to_string(), "-b".to_string(), - format!("localhost:{}", APP_PORT), + format!("localhost:{}", CLIENT_PORT), "-L".to_string(), ]), }), @@ -741,3 +751,129 @@ fn build_broker_rolegroup_statefulset( pub fn error_policy(_error: &Error, _ctx: Arc) -> Action { Action::requeue(Duration::from_secs(5)) } + +fn get_listeners(kafka: &KafkaCluster) -> String { + // If both client and internal TLS are set we do not need the HTTP port. + if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { + format!( + "SSL://$NODE:$(cat /stackable/tmp/{secure}_nodeport)", + secure = SECURE_CLIENT_PORT_NAME + ) + // If only client TLS is required we need to set the HTTPS port and keep the HTTP port + // for internal communications. + // If only internal TLS is required we need to set the HTTPS port and keep the HTTP port + // for client communications. + } else if kafka.client_tls_secret_class().is_some() + || kafka.internal_tls_secret_class().is_some() + { + format!("PLANTEXT://$NODE:$(cat /stackable/tmp/{insecure}_nodeport),SSL://$NODE:$(cat /stackable/tmp/{secure}_nodeport)", + insecure = CLIENT_PORT_NAME, secure = SECURE_CLIENT_PORT_NAME ) + } + // If no is TLS specified the HTTP port is sufficient + else { + format!( + "SSL://$NODE:$(cat /stackable/tmp/{insecure}_nodeport)", + insecure = CLIENT_PORT_NAME + ) + } +} + +fn service_ports(kafka: &KafkaCluster) -> Vec { + let mut ports = vec![ServicePort { + name: Some(METRICS_PORT_NAME.to_string()), + port: METRICS_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }]; + + // If both client and internal TLS are set we do not need the HTTP port. + if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { + ports.push(ServicePort { + name: Some(SECURE_CLIENT_PORT_NAME.to_string()), + port: SECURE_CLIENT_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }); + } + // If only client TLS is required we need to set the HTTPS port and keep the HTTP port + // for internal communications. + // If only internal TLS is required we need to set the HTTPS port and keep the HTTP port + // for client communications. + else if kafka.client_tls_secret_class().is_some() + || kafka.internal_tls_secret_class().is_some() + { + ports.push(ServicePort { + name: Some(CLIENT_PORT_NAME.to_string()), + port: CLIENT_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }); + ports.push(ServicePort { + name: Some(SECURE_CLIENT_PORT_NAME.to_string()), + port: SECURE_CLIENT_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }); + } + // If no is TLS specified the HTTP port is sufficient + else { + ports.push(ServicePort { + name: Some(CLIENT_PORT_NAME.to_string()), + port: CLIENT_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }); + } + + ports +} + +fn container_ports(kafka: &KafkaCluster) -> Vec { + let mut ports = vec![ContainerPort { + name: Some(METRICS_PORT_NAME.to_string()), + container_port: METRICS_PORT.into(), + protocol: Some("TCP".to_string()), + ..ContainerPort::default() + }]; + + // If both client and internal TLS are set we do not need the HTTP port. + if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { + ports.push(ContainerPort { + name: Some(SECURE_CLIENT_PORT_NAME.to_string()), + container_port: SECURE_CLIENT_PORT.into(), + protocol: Some("TCP".to_string()), + ..ContainerPort::default() + }); + } + // If only client TLS is required we need to set the HTTPS port and keep the HTTP port + // for internal communications. + // If only internal TLS is required we need to set the HTTPS port and keep the HTTP port + // for client communications. + else if kafka.client_tls_secret_class().is_some() + || kafka.internal_tls_secret_class().is_some() + { + ports.push(ContainerPort { + name: Some(CLIENT_PORT_NAME.to_string()), + container_port: CLIENT_PORT.into(), + protocol: Some("TCP".to_string()), + ..ContainerPort::default() + }); + ports.push(ContainerPort { + name: Some(SECURE_CLIENT_PORT_NAME.to_string()), + container_port: SECURE_CLIENT_PORT.into(), + protocol: Some("TCP".to_string()), + ..ContainerPort::default() + }); + } + // If no is TLS specified the HTTP port is sufficient + else { + ports.push(ContainerPort { + name: Some(CLIENT_PORT_NAME.to_string()), + container_port: CLIENT_PORT.into(), + protocol: Some("TCP".to_string()), + ..ContainerPort::default() + }); + } + + ports +} diff --git a/rust/operator/src/pod_svc_controller.rs b/rust/operator/src/pod_svc_controller.rs index 8431405b..2cd866f1 100644 --- a/rust/operator/src/pod_svc_controller.rs +++ b/rust/operator/src/pod_svc_controller.rs @@ -1,5 +1,5 @@ use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_kafka_crd::APP_PORT; +use stackable_kafka_crd::{CLIENT_PORT, CLIENT_PORT_NAME}; use stackable_operator::{ k8s_openapi::{ api::core::v1::{Pod, Service, ServicePort, ServiceSpec}, @@ -60,9 +60,10 @@ pub async fn reconcile_pod(pod: Arc, ctx: Arc) -> Result { spec: Some(ServiceSpec { type_: Some("NodePort".to_string()), external_traffic_policy: Some("Local".to_string()), + // TODO: get ports from pod? ports: Some(vec![ServicePort { - name: Some("kafka".to_string()), - port: APP_PORT.into(), + name: Some(CLIENT_PORT_NAME.to_string()), + port: CLIENT_PORT.into(), ..ServicePort::default() }]), selector: Some([(LABEL_STS_POD_NAME.to_string(), name)].into()), From 0cd41343d335be7b626cbbe3a2fdfeff1f01f778 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 27 Jul 2022 18:14:35 +0200 Subject: [PATCH 04/37] client tls working, kcat probe fails --- ...ter.yaml => simple-kafka-cluster-tls.yaml} | 2 +- rust/crd/src/lib.rs | 35 ++++-- rust/operator/src/command.rs | 83 ++++++++++++ rust/operator/src/discovery.rs | 4 +- rust/operator/src/kafka_controller.rs | 119 ++++++++++-------- rust/operator/src/lib.rs | 1 + rust/operator/src/pod_svc_controller.rs | 31 +++-- 7 files changed, 206 insertions(+), 69 deletions(-) rename examples/tls/{simple-kafka-cluster.yaml => simple-kafka-cluster-tls.yaml} (97%) create mode 100644 rust/operator/src/command.rs diff --git a/examples/tls/simple-kafka-cluster.yaml b/examples/tls/simple-kafka-cluster-tls.yaml similarity index 97% rename from examples/tls/simple-kafka-cluster.yaml rename to examples/tls/simple-kafka-cluster-tls.yaml index 1e617b79..47ba8960 100644 --- a/examples/tls/simple-kafka-cluster.yaml +++ b/examples/tls/simple-kafka-cluster-tls.yaml @@ -11,7 +11,7 @@ spec: selector: matchLabels: kubernetes.io/os: linux - replicas: 1 + replicas: 3 config: {} --- apiVersion: zookeeper.stackable.tech/v1alpha1 diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 298116e5..5e0edbb7 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -44,9 +44,14 @@ pub const SSL_TRUSTSTORE_PASSWORD: &str = "ssl.truststore.password"; pub const SSL_TRUSTSTORE_TYPE: &str = "ssl.truststore.type"; pub const SSL_STORE_PASSWORD: &str = "changeit"; pub const SSL_CLIENT_AUTH: &str = "ssl.client.auth"; +// TLS internal +pub const SECURITY_INTER_BROKER_PROTOCOL: &str = "security.inter.broker.protocol"; // directories -pub const CLIENT_TLS_DIR: &str = "/stackable/tls/client"; -pub const INTERNAL_TLS_DIR: &str = "/stackable/tls/internal"; +pub const STACKABLE_TMP_DIR: &str = "/stackable/tmp"; +pub const STACKABLE_DATA_DIR: &str = "/stackable/data"; +pub const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; +pub const STACKABLE_TLS_CERTS_DIR: &str = "/stackable/certificates"; +pub const SYSTEM_TRUST_STORE_DIR: &str = "/etc/pki/java/cacerts"; const JVM_HEAP_FACTOR: f32 = 0.8; @@ -301,11 +306,11 @@ impl KafkaCluster { /// Checks if we should use TLS to encrypt client connections. pub fn is_client_secure(&self) -> bool { - self.client_tls_secret_class().is_some() + self.client_tls_secret_class().is_some() || self.client_authentication_class().is_some() } /// Returns the authentication class used for client authentication - pub fn client_tls_authentication_class(&self) -> Option { + pub fn client_authentication_class(&self) -> Option { let spec: &KafkaClusterSpec = &self.spec; spec.config .as_ref() @@ -413,31 +418,41 @@ impl Configuration for KafkaConfig { if resource.client_tls_secret_class().is_some() { config.insert( SSL_KEYSTORE_LOCATION.to_string(), - Some(format!("{}/keystore.p12", CLIENT_TLS_DIR)), + Some(format!("{}/keystore.p12", STACKABLE_TLS_CERTS_DIR)), ); config.insert( SSL_KEYSTORE_PASSWORD.to_string(), - Some(SSL_KEYSTORE_PASSWORD.to_string()), + Some(SSL_STORE_PASSWORD.to_string()), ); config.insert(SSL_KEYSTORE_TYPE.to_string(), Some("PKCS12".to_string())); config.insert( SSL_TRUSTSTORE_LOCATION.to_string(), - Some(format!("{}/truststore.p12", CLIENT_TLS_DIR)), + Some(format!("{}/truststore.p12", STACKABLE_TLS_CERTS_DIR)), ); config.insert( SSL_TRUSTSTORE_PASSWORD.to_string(), - Some(SSL_KEYSTORE_PASSWORD.to_string()), + Some(SSL_STORE_PASSWORD.to_string()), ); config.insert(SSL_TRUSTSTORE_TYPE.to_string(), Some("PKCS12".to_string())); // Authentication - if resource.client_tls_authentication_class().is_some() { + if resource.client_authentication_class().is_some() { config.insert(SSL_CLIENT_AUTH.to_string(), Some("required".to_string())); } } + // We require authentication + if resource.client_authentication_class().is_some() { + config.insert(SSL_CLIENT_AUTH.to_string(), Some("required".to_string())); + } + // Internal TLS - if resource.internal_tls_secret_class().is_some() {} + if resource.internal_tls_secret_class().is_some() { + config.insert( + SECURITY_INTER_BROKER_PROTOCOL.to_string(), + Some("SSL".to_string()), + ); + } } Ok(config) diff --git a/rust/operator/src/command.rs b/rust/operator/src/command.rs new file mode 100644 index 00000000..31232fe9 --- /dev/null +++ b/rust/operator/src/command.rs @@ -0,0 +1,83 @@ +use stackable_kafka_crd::{ + KafkaCluster, CLIENT_PORT, SECURE_CLIENT_PORT, SSL_KEYSTORE_LOCATION, SSL_KEYSTORE_PASSWORD, + SSL_STORE_PASSWORD, SSL_TRUSTSTORE_LOCATION, SSL_TRUSTSTORE_PASSWORD, STACKABLE_DATA_DIR, + STACKABLE_TLS_CERTS_DIR, STACKABLE_TMP_DIR, SYSTEM_TRUST_STORE_DIR, +}; + +pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { + let mut args = vec![ + // Copy system truststore to stackable truststore + format!("keytool -importkeystore -srckeystore {SYSTEM_TRUST_STORE_DIR} -srcstoretype jks -srcstorepass {SSL_STORE_PASSWORD} -destkeystore {STACKABLE_TLS_CERTS_DIR}/truststore.p12 -deststoretype pkcs12 -deststorepass {SSL_STORE_PASSWORD} -noprompt") + ]; + + if kafka.client_tls_secret_class().is_some() { + args.extend(create_key_and_trust_store( + STACKABLE_TLS_CERTS_DIR, + "stackable-ca-cert", + )); + args.extend(chown_and_chmod(STACKABLE_TLS_CERTS_DIR)); + } + + args.extend(chown_and_chmod(STACKABLE_DATA_DIR)); + args.extend(chown_and_chmod(STACKABLE_TMP_DIR)); + + args.join(" && ") +} + +pub fn kcat_container_cmd_args(kafka: &KafkaCluster) -> Vec { + let mut args = vec!["kcat".to_string()]; + + if kafka.client_tls_secret_class().is_some() { + args.push("-b".to_string()); + args.push(format!("localhost:{}", SECURE_CLIENT_PORT)); + args.extend([ + "-X".to_string(), + "security.protocol=SSL".to_string(), + "-X".to_string(), + format!( + "{}={}/keystore.p12", + SSL_KEYSTORE_LOCATION, STACKABLE_TLS_CERTS_DIR + ), + "-X".to_string(), + format!("{}={}", SSL_KEYSTORE_PASSWORD, SSL_STORE_PASSWORD), + "-X".to_string(), + format!( + "{}={}/truststore.p12", + SSL_TRUSTSTORE_LOCATION, STACKABLE_TLS_CERTS_DIR + ), + "-X".to_string(), + format!("{}={}", SSL_TRUSTSTORE_PASSWORD, SSL_STORE_PASSWORD), + ]); + } else { + args.push("-b".to_string()); + args.push(format!("localhost:{}", CLIENT_PORT)); + } + + args.push("-L".to_string()); + + args +} + +/// Generates the shell script to create key and truststores from the certificates provided +/// by the secret operator. +fn create_key_and_trust_store(directory: &str, alias_name: &str) -> Vec { + vec![ + format!("echo [{dir}] Creating truststore", dir = directory), + format!("keytool -importcert -file {dir}/ca.crt -keystore {dir}/truststore.p12 -storetype pkcs12 -noprompt -alias {alias} -storepass {password}", + dir = directory, alias = alias_name, password = SSL_STORE_PASSWORD), + format!("echo [{dir}] Creating certificate chain", dir = directory), + format!("cat {dir}/ca.crt {dir}/tls.crt > {dir}/chain.crt", dir = directory), + format!("echo [{dir}] Creating keystore", dir = directory), + format!("openssl pkcs12 -export -in {dir}/chain.crt -inkey {dir}/tls.key -out {dir}/keystore.p12 --passout pass:{password}", + dir = directory, password = SSL_STORE_PASSWORD), + ] +} + +/// Generates a shell script to chown and chmod the provided directory. +fn chown_and_chmod(directory: &str) -> Vec { + vec![ + format!("echo chown and chmod {dir}", dir = directory), + format!("chown -R stackable:stackable {dir}", dir = directory), + format!("chmod -R a=,u=rwX {dir}", dir = directory), + ] +} diff --git a/rust/operator/src/discovery.rs b/rust/operator/src/discovery.rs index 467fcbbc..e03b08bb 100644 --- a/rust/operator/src/discovery.rs +++ b/rust/operator/src/discovery.rs @@ -47,12 +47,12 @@ pub async fn build_discovery_configmaps( ) -> Result, Error> { let name = owner.name(); Ok(vec![ - build_discovery_configmap(&name, owner, kafka, service_hosts(svc, "kafka")?)?, + build_discovery_configmap(&name, owner, kafka, service_hosts(svc, "https")?)?, build_discovery_configmap( &format!("{}-nodeport", name), owner, kafka, - nodeport_hosts(client, svc, "kafka").await?, + nodeport_hosts(client, svc, "https").await?, )?, ]) } diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index 4a3de6f1..6d6a3db2 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -2,9 +2,13 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_kafka_crd::{ - KafkaCluster, KafkaRole, APP_NAME, CLIENT_PORT, CLIENT_PORT_NAME, KAFKA_HEAP_OPTS, - LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, SECURE_CLIENT_PORT, - SECURE_CLIENT_PORT_NAME, SERVER_PROPERTIES_FILE, + KafkaCluster, KafkaRole, TlsSecretClass, APP_NAME, CLIENT_PORT, CLIENT_PORT_NAME, + KAFKA_HEAP_OPTS, LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, SECURE_CLIENT_PORT, + SECURE_CLIENT_PORT_NAME, SERVER_PROPERTIES_FILE, STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, + STACKABLE_TLS_CERTS_DIR, STACKABLE_TMP_DIR, +}; +use stackable_operator::builder::{ + SecretOperatorVolumeSourceBuilder, SecurityContextBuilder, VolumeBuilder, }; use stackable_operator::{ builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder}, @@ -15,8 +19,7 @@ use stackable_operator::{ core::v1::{ ConfigMap, ConfigMapKeySelector, ConfigMapVolumeSource, ContainerPort, EmptyDirVolumeSource, EnvVar, EnvVarSource, ExecAction, ObjectFieldSelector, - PodSpec, Probe, SecurityContext, Service, ServiceAccount, ServicePort, ServiceSpec, - Volume, + PodSpec, Probe, Service, ServiceAccount, ServicePort, ServiceSpec, Volume, }, rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject}, }, @@ -40,7 +43,9 @@ use std::{ }; use strum::{EnumDiscriminants, IntoStaticStr}; +use crate::command::kcat_container_cmd_args; use crate::{ + command, discovery::{self, build_discovery_configmaps}, pod_svc_controller, utils::{self, ObjectRefExt}, @@ -476,6 +481,10 @@ fn build_broker_rolegroup_statefulset( serviceaccount: &ObjectRef, opa_connect_string: Option<&str>, ) -> Result { + let mut cb_kafka = ContainerBuilder::new(APP_NAME); + let mut cb_prepare = ContainerBuilder::new("prepare"); + let mut pod_builder = PodBuilder::new(); + let role = kafka.spec.brokers.as_ref().context(NoBrokerRoleSnafu)?; let rolegroup = role .role_groups @@ -491,20 +500,26 @@ fn build_broker_rolegroup_statefulset( let get_svc_args = if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { - format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee/stackable/tmp/{name}_nodeport", name = SECURE_CLIENT_PORT_NAME) + format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee {dir}/{name}_nodeport", dir = STACKABLE_TMP_DIR, name = SECURE_CLIENT_PORT_NAME) } else if kafka.client_tls_secret_class().is_some() || kafka.internal_tls_secret_class().is_some() { [ - format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee/stackable/tmp/{name}_nodeport", name = CLIENT_PORT_NAME), - format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee/stackable/tmp/{name}_nodeport", name = SECURE_CLIENT_PORT_NAME), + format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee {dir}/{name}_nodeport", dir = STACKABLE_TMP_DIR, name = CLIENT_PORT_NAME), + format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee {dir}/{name}_nodeport", dir = STACKABLE_TMP_DIR, name = SECURE_CLIENT_PORT_NAME), ].join(" && ") } // If no is TLS specified the HTTP port is sufficient else { - format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee/stackable/tmp/{name}_nodeport", name = CLIENT_PORT_NAME) + format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee {dir}/{name}_nodeport", dir = STACKABLE_TMP_DIR, name = CLIENT_PORT_NAME) }; + if let Some(tls) = kafka.client_tls_secret_class() { + cb_prepare.add_volume_mount("tls-certificate", STACKABLE_TLS_CERTS_DIR); + cb_kafka.add_volume_mount("tls-certificate", STACKABLE_TLS_CERTS_DIR); + pod_builder.add_volume(create_tls_volume("tls-certificate", Some(tls))); + } + let container_get_svc = ContainerBuilder::new("get-svc") .image("docker.stackable.tech/stackable/tools:0.2.0-stackable0.3.0") .command(vec!["bash".to_string()]) @@ -525,35 +540,21 @@ fn build_broker_rolegroup_statefulset( }), ..EnvVar::default() }]) - .add_volume_mount("tmp", "/stackable/tmp") + .add_volume_mount("tmp", STACKABLE_TMP_DIR) .build(); - // For most storage classes the mounts will belong to the root user and not be writeable to - // other users. - // Since kafka runs as the user stackable inside of the container the data directory needs to be - // chowned to that user for it to be able to store data there. - let mut container_chown = ContainerBuilder::new("chown-data") - .image(&image) + cb_prepare + .image("docker.stackable.tech/stackable/tools:0.2.0-stackable0.3.0") .command(vec![ "/bin/bash".to_string(), "-euo".to_string(), "pipefail".to_string(), "-c".to_string(), ]) - .args(vec![[ - "echo chowning data directory", - "chown -R stackable:stackable /stackable/data", - "echo chmodding data directory", - "chmod -R a=,u=rwX /stackable/data", - ] - .join(" && ")]) - .add_volume_mount(LOG_DIRS_VOLUME_NAME, "/stackable/data") - .build(); - - container_chown - .security_context - .get_or_insert_with(SecurityContext::default) - .run_as_user = Some(0); + .args(vec![command::prepare_container_cmd_args(kafka)]) + .add_volume_mount(LOG_DIRS_VOLUME_NAME, STACKABLE_DATA_DIR) + .add_volume_mount("tmp", STACKABLE_TMP_DIR) + .security_context(SecurityContextBuilder::run_as_root()); let (pvc, resources) = kafka.resources(rolegroup_ref); @@ -617,6 +618,10 @@ fn build_broker_rolegroup_statefulset( let jvm_args = format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar={}:/stackable/jmx/broker.yaml", METRICS_PORT); let zookeeper_override = "--override \"zookeeper.connect=$ZOOKEEPER\""; + let listeners_override = format!( + "--override \"listeners=PLAINTEXT://0.0.0.0:{},SSL://0.0.0.0:{}\"", + CLIENT_PORT, SECURE_CLIENT_PORT + ); let advertised_listeners_override = format!( "--override \"advertised.listeners={}\"", get_listeners(kafka) @@ -625,7 +630,7 @@ fn build_broker_rolegroup_statefulset( format!("--override \"opa.authorizer.url={}\"", opa) }); - let container_kafka = ContainerBuilder::new("kafka") + cb_kafka .image(image) .args(vec![ "sh".to_string(), @@ -634,6 +639,7 @@ fn build_broker_rolegroup_statefulset( "bin/kafka-server-start.sh", &format!("/stackable/config/{}", SERVER_PROPERTIES_FILE), zookeeper_override, + &listeners_override, &advertised_listeners_override, &opa_url_override, ] @@ -642,11 +648,10 @@ fn build_broker_rolegroup_statefulset( .add_env_vars(env) .add_env_var("EXTRA_ARGS", jvm_args) .add_container_ports(container_ports(kafka)) - .add_volume_mount(LOG_DIRS_VOLUME_NAME, "/stackable/data") - .add_volume_mount("config", "/stackable/config") - .add_volume_mount("tmp", "/stackable/tmp") - .resources(resources) - .build(); + .add_volume_mount(LOG_DIRS_VOLUME_NAME, STACKABLE_DATA_DIR) + .add_volume_mount("config", STACKABLE_CONFIG_DIR) + .add_volume_mount("tmp", STACKABLE_TMP_DIR) + .resources(resources); // Use kcat sidecar for probing container status rather than the official Kafka tools, since they incur a lot of // unacceptable perf overhead @@ -658,12 +663,7 @@ fn build_broker_rolegroup_statefulset( .readiness_probe(Probe { exec: Some(ExecAction { // If the broker is able to get its fellow cluster members then it has at least completed basic registration at some point - command: Some(vec![ - "kcat".to_string(), - "-b".to_string(), - format!("localhost:{}", CLIENT_PORT), - "-L".to_string(), - ]), + command: Some(kcat_container_cmd_args(kafka)), }), timeout_seconds: Some(3), period_seconds: Some(1), @@ -671,7 +671,7 @@ fn build_broker_rolegroup_statefulset( }) .build(); container_kcat_prober.stdin = Some(true); - let mut pod_template = PodBuilder::new() + let mut pod_template = pod_builder .metadata_builder(|m| { m.with_recommended_labels( kafka, @@ -682,9 +682,9 @@ fn build_broker_rolegroup_statefulset( ) .with_label(pod_svc_controller::LABEL_ENABLE, "true") }) - .add_init_container(container_chown) + .add_init_container(cb_prepare.build()) .add_init_container(container_get_svc) - .add_container(container_kafka) + .add_container(cb_kafka.build()) .add_container(container_kcat_prober) .add_volume(Volume { name: "config".to_string(), @@ -756,7 +756,8 @@ fn get_listeners(kafka: &KafkaCluster) -> String { // If both client and internal TLS are set we do not need the HTTP port. if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { format!( - "SSL://$NODE:$(cat /stackable/tmp/{secure}_nodeport)", + "SSL://$NODE:$(cat {dir}/{secure}_nodeport)", + dir = STACKABLE_TMP_DIR, secure = SECURE_CLIENT_PORT_NAME ) // If only client TLS is required we need to set the HTTPS port and keep the HTTP port @@ -766,13 +767,16 @@ fn get_listeners(kafka: &KafkaCluster) -> String { } else if kafka.client_tls_secret_class().is_some() || kafka.internal_tls_secret_class().is_some() { - format!("PLANTEXT://$NODE:$(cat /stackable/tmp/{insecure}_nodeport),SSL://$NODE:$(cat /stackable/tmp/{secure}_nodeport)", - insecure = CLIENT_PORT_NAME, secure = SECURE_CLIENT_PORT_NAME ) + format!("PLAINTEXT://$NODE:$(cat {dir}/{insecure}_nodeport),SSL://$NODE:$(cat /{dir}/{secure}_nodeport)", + dir = STACKABLE_TMP_DIR, + insecure = CLIENT_PORT_NAME, + secure = SECURE_CLIENT_PORT_NAME ) } // If no is TLS specified the HTTP port is sufficient else { format!( - "SSL://$NODE:$(cat /stackable/tmp/{insecure}_nodeport)", + "SSL://$NODE:$(cat {dir}/{insecure}_nodeport)", + dir = STACKABLE_TMP_DIR, insecure = CLIENT_PORT_NAME ) } @@ -877,3 +881,20 @@ fn container_ports(kafka: &KafkaCluster) -> Vec { ports } + +fn create_tls_volume(volume_name: &str, tls_secret_class: Option<&TlsSecretClass>) -> Volume { + let mut secret_class_name = "tls"; + + if let Some(tls) = tls_secret_class { + secret_class_name = &tls.secret_class; + } + + VolumeBuilder::new(volume_name) + .ephemeral( + SecretOperatorVolumeSourceBuilder::new(secret_class_name) + .with_pod_scope() + .with_node_scope() + .build(), + ) + .build() +} diff --git a/rust/operator/src/lib.rs b/rust/operator/src/lib.rs index 98660b2f..6f3d36d6 100644 --- a/rust/operator/src/lib.rs +++ b/rust/operator/src/lib.rs @@ -1,3 +1,4 @@ +mod command; mod discovery; mod kafka_controller; mod pod_svc_controller; diff --git a/rust/operator/src/pod_svc_controller.rs b/rust/operator/src/pod_svc_controller.rs index 2cd866f1..05f54e7f 100644 --- a/rust/operator/src/pod_svc_controller.rs +++ b/rust/operator/src/pod_svc_controller.rs @@ -1,8 +1,8 @@ use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_kafka_crd::{CLIENT_PORT, CLIENT_PORT_NAME}; +use stackable_kafka_crd::APP_NAME; use stackable_operator::{ k8s_openapi::{ - api::core::v1::{Pod, Service, ServicePort, ServiceSpec}, + api::core::v1::{Container, Pod, Service, ServicePort, ServiceSpec}, apimachinery::pkg::apis::meta::v1::OwnerReference, }, kube::{core::ObjectMeta, runtime::controller::Action}, @@ -44,6 +44,27 @@ impl ReconcilerError for Error { pub async fn reconcile_pod(pod: Arc, ctx: Arc) -> Result { tracing::info!("Starting reconcile"); let name = pod.metadata.name.clone().context(ObjectHasNoNameSnafu)?; + let mut ports: Vec = vec![]; + + if let Some(spec) = &pod.spec { + for container in &spec + .containers + .iter() + .filter(|container| container.name == APP_NAME) + .collect::>() + { + if let Some(container_ports) = &container.ports { + for port in container_ports { + ports.push(ServicePort { + name: port.name.clone(), + port: port.container_port, + ..ServicePort::default() + }); + } + } + } + } + let svc = Service { metadata: ObjectMeta { namespace: pod.metadata.namespace.clone(), @@ -61,11 +82,7 @@ pub async fn reconcile_pod(pod: Arc, ctx: Arc) -> Result { type_: Some("NodePort".to_string()), external_traffic_policy: Some("Local".to_string()), // TODO: get ports from pod? - ports: Some(vec![ServicePort { - name: Some(CLIENT_PORT_NAME.to_string()), - port: CLIENT_PORT.into(), - ..ServicePort::default() - }]), + ports: Some(ports), selector: Some([(LABEL_STS_POD_NAME.to_string(), name)].into()), publish_not_ready_addresses: Some(true), ..ServiceSpec::default() From 46d74d27086dbbd9d7909f6a837f8e9efe60bed2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 28 Jul 2022 18:42:40 +0200 Subject: [PATCH 05/37] kcat probe working --- rust/operator/src/command.rs | 16 +++++----------- rust/operator/src/kafka_controller.rs | 4 +++- rust/operator/src/pod_svc_controller.rs | 1 - 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/rust/operator/src/command.rs b/rust/operator/src/command.rs index 31232fe9..cb31a942 100644 --- a/rust/operator/src/command.rs +++ b/rust/operator/src/command.rs @@ -1,6 +1,5 @@ use stackable_kafka_crd::{ - KafkaCluster, CLIENT_PORT, SECURE_CLIENT_PORT, SSL_KEYSTORE_LOCATION, SSL_KEYSTORE_PASSWORD, - SSL_STORE_PASSWORD, SSL_TRUSTSTORE_LOCATION, SSL_TRUSTSTORE_PASSWORD, STACKABLE_DATA_DIR, + KafkaCluster, CLIENT_PORT, SECURE_CLIENT_PORT, SSL_STORE_PASSWORD, STACKABLE_DATA_DIR, STACKABLE_TLS_CERTS_DIR, STACKABLE_TMP_DIR, SYSTEM_TRUST_STORE_DIR, }; @@ -34,19 +33,14 @@ pub fn kcat_container_cmd_args(kafka: &KafkaCluster) -> Vec { "-X".to_string(), "security.protocol=SSL".to_string(), "-X".to_string(), - format!( - "{}={}/keystore.p12", - SSL_KEYSTORE_LOCATION, STACKABLE_TLS_CERTS_DIR - ), - "-X".to_string(), - format!("{}={}", SSL_KEYSTORE_PASSWORD, SSL_STORE_PASSWORD), + format!("ssl.key.location={}/tls.key", STACKABLE_TLS_CERTS_DIR), "-X".to_string(), format!( - "{}={}/truststore.p12", - SSL_TRUSTSTORE_LOCATION, STACKABLE_TLS_CERTS_DIR + "ssl.certificate.location={}/tls.crt", + STACKABLE_TLS_CERTS_DIR ), "-X".to_string(), - format!("{}={}", SSL_TRUSTSTORE_PASSWORD, SSL_STORE_PASSWORD), + format!("ssl.ca.location={}/ca.crt", STACKABLE_TLS_CERTS_DIR), ]); } else { args.push("-b".to_string()); diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index 6d6a3db2..a3ceca42 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -483,6 +483,7 @@ fn build_broker_rolegroup_statefulset( ) -> Result { let mut cb_kafka = ContainerBuilder::new(APP_NAME); let mut cb_prepare = ContainerBuilder::new("prepare"); + let mut cb_kcat_prober = ContainerBuilder::new("kcat-prober"); let mut pod_builder = PodBuilder::new(); let role = kafka.spec.brokers.as_ref().context(NoBrokerRoleSnafu)?; @@ -517,6 +518,7 @@ fn build_broker_rolegroup_statefulset( if let Some(tls) = kafka.client_tls_secret_class() { cb_prepare.add_volume_mount("tls-certificate", STACKABLE_TLS_CERTS_DIR); cb_kafka.add_volume_mount("tls-certificate", STACKABLE_TLS_CERTS_DIR); + cb_kcat_prober.add_volume_mount("tls-certificate", STACKABLE_TLS_CERTS_DIR); pod_builder.add_volume(create_tls_volume("tls-certificate", Some(tls))); } @@ -655,7 +657,7 @@ fn build_broker_rolegroup_statefulset( // Use kcat sidecar for probing container status rather than the official Kafka tools, since they incur a lot of // unacceptable perf overhead - let mut container_kcat_prober = ContainerBuilder::new("kcat-prober") + let mut container_kcat_prober = cb_kcat_prober .image("edenhill/kcat:1.7.0") .command(vec!["sh".to_string()]) // Only allow the global load balancing service to send traffic to pods that are members of the quorum diff --git a/rust/operator/src/pod_svc_controller.rs b/rust/operator/src/pod_svc_controller.rs index 05f54e7f..a4d00ccc 100644 --- a/rust/operator/src/pod_svc_controller.rs +++ b/rust/operator/src/pod_svc_controller.rs @@ -81,7 +81,6 @@ pub async fn reconcile_pod(pod: Arc, ctx: Arc) -> Result { spec: Some(ServiceSpec { type_: Some("NodePort".to_string()), external_traffic_policy: Some("Local".to_string()), - // TODO: get ports from pod? ports: Some(ports), selector: Some([(LABEL_STS_POD_NAME.to_string(), name)].into()), publish_not_ready_addresses: Some(true), From b61cf8b1590513914c231320c502a4f8c74d1638 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 28 Jul 2022 18:59:12 +0200 Subject: [PATCH 06/37] added get_node_port method --- rust/operator/src/kafka_controller.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index a3ceca42..604edfad 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -501,18 +501,19 @@ fn build_broker_rolegroup_statefulset( let get_svc_args = if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { - format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee {dir}/{name}_nodeport", dir = STACKABLE_TMP_DIR, name = SECURE_CLIENT_PORT_NAME) + get_node_port(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME) } else if kafka.client_tls_secret_class().is_some() || kafka.internal_tls_secret_class().is_some() { [ - format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee {dir}/{name}_nodeport", dir = STACKABLE_TMP_DIR, name = CLIENT_PORT_NAME), - format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee {dir}/{name}_nodeport", dir = STACKABLE_TMP_DIR, name = SECURE_CLIENT_PORT_NAME), - ].join(" && ") + get_node_port(STACKABLE_TMP_DIR, CLIENT_PORT_NAME), + get_node_port(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME), + ] + .join(" && ") } // If no is TLS specified the HTTP port is sufficient else { - format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee {dir}/{name}_nodeport", dir = STACKABLE_TMP_DIR, name = CLIENT_PORT_NAME) + get_node_port(STACKABLE_TMP_DIR, CLIENT_PORT_NAME) }; if let Some(tls) = kafka.client_tls_secret_class() { @@ -900,3 +901,7 @@ fn create_tls_volume(volume_name: &str, tls_secret_class: Option<&TlsSecretClass ) .build() } + +fn get_node_port(directory: &str, port_name: &str) -> String { + format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee {dir}/{name}_nodeport", dir = directory, name = port_name) +} From 5159703f0c6bc380dca94cabb99bea63aa7bccb2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 29 Jul 2022 11:34:30 +0200 Subject: [PATCH 07/37] made GlobalKafkaConfig non optional --- deploy/crd/kafkacluster.crd.yaml | 7 +- deploy/helm/kafka-operator/crds/crds.yaml | 7 +- deploy/manifests/crds.yaml | 7 +- rust/crd/src/lib.rs | 206 +++++++++++++++++----- rust/operator/src/kafka_controller.rs | 32 +++- 5 files changed, 194 insertions(+), 65 deletions(-) diff --git a/deploy/crd/kafkacluster.crd.yaml b/deploy/crd/kafkacluster.crd.yaml index 4c8ac5fd..326d334f 100644 --- a/deploy/crd/kafkacluster.crd.yaml +++ b/deploy/crd/kafkacluster.crd.yaml @@ -279,9 +279,6 @@ spec: default: tls: secretClass: tls - internalTls: - secretClass: tls - nullable: true properties: clientAuthentication: description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" @@ -293,9 +290,7 @@ spec: - authenticationClass type: object internalTls: - default: - secretClass: tls - description: "Only affects internal communication. Use mutual verification between Kafka Broker Nodes (mandatory). This setting controls: - Which cert the brokers should use to authenticate themselves against other brokers - Which ca.crt to use when validating the other server" + description: "Only affects internal communication. Use mutual verification between Trino nodes This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" nullable: true properties: secretClass: diff --git a/deploy/helm/kafka-operator/crds/crds.yaml b/deploy/helm/kafka-operator/crds/crds.yaml index cc688927..099a199e 100644 --- a/deploy/helm/kafka-operator/crds/crds.yaml +++ b/deploy/helm/kafka-operator/crds/crds.yaml @@ -281,9 +281,6 @@ spec: default: tls: secretClass: tls - internalTls: - secretClass: tls - nullable: true properties: clientAuthentication: description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" @@ -295,9 +292,7 @@ spec: - authenticationClass type: object internalTls: - default: - secretClass: tls - description: "Only affects internal communication. Use mutual verification between Kafka Broker Nodes (mandatory). This setting controls: - Which cert the brokers should use to authenticate themselves against other brokers - Which ca.crt to use when validating the other server" + description: "Only affects internal communication. Use mutual verification between Trino nodes This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" nullable: true properties: secretClass: diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index da4b0432..8c8ec280 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -282,9 +282,6 @@ spec: default: tls: secretClass: tls - internalTls: - secretClass: tls - nullable: true properties: clientAuthentication: description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" @@ -296,9 +293,7 @@ spec: - authenticationClass type: object internalTls: - default: - secretClass: tls - description: "Only affects internal communication. Use mutual verification between Kafka Broker Nodes (mandatory). This setting controls: - Which cert the brokers should use to authenticate themselves against other brokers - Which ca.crt to use when validating the other server" + description: "Only affects internal communication. Use mutual verification between Trino nodes This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" nullable: true properties: secretClass: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 5e0edbb7..d2c69041 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -86,15 +86,12 @@ pub struct KafkaClusterSpec { pub zookeeper_config_map_name: String, pub opa: Option, pub log4j: Option, - #[serde( - default = "global_config_default", - skip_serializing_if = "Option::is_none" - )] - pub config: Option, + #[serde(default)] + pub config: GlobalKafkaConfig, pub stopped: Option, } -#[derive(Clone, Default, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct GlobalKafkaConfig { /// Only affects client connections. This setting controls: @@ -112,27 +109,22 @@ pub struct GlobalKafkaConfig { /// Defaults to `None` #[serde(skip_serializing_if = "Option::is_none")] pub client_authentication: Option, - /// Only affects internal communication. Use mutual verification between Kafka Broker Nodes - /// (mandatory). This setting controls: - /// - Which cert the brokers should use to authenticate themselves against other brokers + /// Only affects internal communication. Use mutual verification between Trino nodes + /// This setting controls: + /// - Which cert the servers should use to authenticate themselves against other servers /// - Which ca.crt to use when validating the other server - #[serde( - default = "tls_secret_class_default", - skip_serializing_if = "Option::is_none" - )] + #[serde(skip_serializing_if = "Option::is_none")] pub internal_tls: Option, } -fn global_config_default() -> Option { - Some(GlobalKafkaConfig { - tls: Some(TlsSecretClass { - secret_class: TLS_DEFAULT_SECRET_CLASS.to_string(), - }), - client_authentication: None, - internal_tls: Some(TlsSecretClass { - secret_class: TLS_DEFAULT_SECRET_CLASS.to_string(), - }), - }) +impl Default for GlobalKafkaConfig { + fn default() -> Self { + GlobalKafkaConfig { + tls: tls_secret_class_default(), + client_authentication: None, + internal_tls: None, + } + } } #[derive(Clone, Default, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -290,38 +282,30 @@ impl KafkaCluster { }) } - pub fn client_port(&self) -> u16 { - if self.is_client_secure() { - SECURE_CLIENT_PORT - } else { - CLIENT_PORT - } - } - /// Returns the secret class for client connection encryption. Defaults to `tls`. pub fn client_tls_secret_class(&self) -> Option<&TlsSecretClass> { let spec: &KafkaClusterSpec = &self.spec; - spec.config.as_ref().and_then(|c| c.tls.as_ref()) - } - - /// Checks if we should use TLS to encrypt client connections. - pub fn is_client_secure(&self) -> bool { - self.client_tls_secret_class().is_some() || self.client_authentication_class().is_some() + spec.config.tls.as_ref() } /// Returns the authentication class used for client authentication - pub fn client_authentication_class(&self) -> Option { + pub fn client_authentication_class(&self) -> Option<&str> { let spec: &KafkaClusterSpec = &self.spec; spec.config + .client_authentication .as_ref() - .and_then(|c| c.client_authentication.as_ref()) - .map(|tls| tls.authentication_class.clone()) + .map(|tls| tls.authentication_class.as_ref()) } /// Returns the secret class for internal server encryption pub fn internal_tls_secret_class(&self) -> Option<&TlsSecretClass> { let spec: &KafkaClusterSpec = &self.spec; - spec.config.as_ref().and_then(|c| c.internal_tls.as_ref()) + spec.config.internal_tls.as_ref() + } + + /// Checks if we should use TLS to encrypt client connections. + pub fn is_client_secure(&self) -> bool { + self.client_tls_secret_class().is_some() || self.client_authentication_class().is_some() } } @@ -458,3 +442,143 @@ impl Configuration for KafkaConfig { Ok(config) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_client_tls() { + let input = r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + spec: + version: abc + zookeeperConfigMapName: xyz + "#; + let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); + assert_eq!( + kafka.client_tls_secret_class().unwrap().secret_class, + TLS_DEFAULT_SECRET_CLASS.to_string() + ); + assert_eq!(kafka.internal_tls_secret_class(), None); + + let input = r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + spec: + version: abc + zookeeperConfigMapName: xyz + config: + tls: + secretClass: simple-kafka-client-tls + "#; + let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); + assert_eq!( + kafka.client_tls_secret_class().unwrap().secret_class, + "simple-kafka-client-tls".to_string() + ); + assert_eq!(kafka.internal_tls_secret_class(), None); + + let input = r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + spec: + version: abc + zookeeperConfigMapName: xyz + config: + tls: null + "#; + let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); + assert_eq!(kafka.client_tls_secret_class(), None); + assert_eq!(kafka.internal_tls_secret_class(), None); + + let input = r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + spec: + version: abc + zookeeperConfigMapName: xyz + config: + internalTls: + secretClass: simple-kafka-internal-tls + "#; + let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); + assert_eq!( + kafka.client_tls_secret_class().unwrap().secret_class, + TLS_DEFAULT_SECRET_CLASS.to_string() + ); + assert_eq!( + kafka.internal_tls_secret_class().unwrap().secret_class, + "simple-kafka-internal-tls" + ); + } + + #[test] + fn test_internal_tls() { + let input = r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + spec: + version: abc + zookeeperConfigMapName: xyz + "#; + let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); + assert_eq!(kafka.internal_tls_secret_class(), None); + assert_eq!( + kafka.client_tls_secret_class().unwrap().secret_class, + TLS_DEFAULT_SECRET_CLASS + ); + + let input = r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + spec: + version: abc + zookeeperConfigMapName: xyz + config: + internalTls: + secretClass: simple-kafka-internal-tls + "#; + let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); + assert_eq!( + kafka.internal_tls_secret_class().unwrap().secret_class, + "simple-kafka-internal-tls".to_string() + ); + assert_eq!( + kafka.client_tls_secret_class().unwrap().secret_class, + TLS_DEFAULT_SECRET_CLASS + ); + + let input = r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + spec: + version: abc + zookeeperConfigMapName: xyz + config: + tls: + secretClass: simple-kafka-client-tls + "#; + let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); + assert_eq!(kafka.internal_tls_secret_class(), None); + assert_eq!( + kafka.client_tls_secret_class().unwrap().secret_class, + "simple-kafka-client-tls" + ); + } +} diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index 604edfad..5f51e719 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -621,13 +621,10 @@ fn build_broker_rolegroup_statefulset( let jvm_args = format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar={}:/stackable/jmx/broker.yaml", METRICS_PORT); let zookeeper_override = "--override \"zookeeper.connect=$ZOOKEEPER\""; - let listeners_override = format!( - "--override \"listeners=PLAINTEXT://0.0.0.0:{},SSL://0.0.0.0:{}\"", - CLIENT_PORT, SECURE_CLIENT_PORT - ); + let listeners_override = format!("--override \"listeners={}\"", get_listeners(kafka)); let advertised_listeners_override = format!( "--override \"advertised.listeners={}\"", - get_listeners(kafka) + get_advertised_listeners(kafka) ); let opa_url_override = opa_connect_string.map_or("".to_string(), |opa| { format!("--override \"opa.authorizer.url={}\"", opa) @@ -756,6 +753,29 @@ pub fn error_policy(_error: &Error, _ctx: Arc) -> Action { } fn get_listeners(kafka: &KafkaCluster) -> String { + // If both client and internal TLS are set we do not need the HTTP port. + if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { + format!("SSL://0.0.0.0:{}", secure = SECURE_CLIENT_PORT) + // If only client TLS is required we need to set the HTTPS port and keep the HTTP port + // for internal communications. + // If only internal TLS is required we need to set the HTTPS port and keep the HTTP port + // for client communications. + } else if kafka.client_tls_secret_class().is_some() + || kafka.internal_tls_secret_class().is_some() + { + format!( + "PLAINTEXT://0.0.0.0:{insecure},SSL://0.0.0.0:{secure}", + insecure = CLIENT_PORT, + secure = SECURE_CLIENT_PORT + ) + } + // If no is TLS specified the HTTP port is sufficient + else { + format!("PLAINTEXT://0.0.0.0:{insecure}", insecure = CLIENT_PORT) + } +} + +fn get_advertised_listeners(kafka: &KafkaCluster) -> String { // If both client and internal TLS are set we do not need the HTTP port. if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { format!( @@ -778,7 +798,7 @@ fn get_listeners(kafka: &KafkaCluster) -> String { // If no is TLS specified the HTTP port is sufficient else { format!( - "SSL://$NODE:$(cat {dir}/{insecure}_nodeport)", + "PLAINTEXT://$NODE:$(cat {dir}/{insecure}_nodeport)", dir = STACKABLE_TMP_DIR, insecure = CLIENT_PORT_NAME ) From 29c82763c395b4414ee485a9d8a53aa771465bb5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 29 Jul 2022 16:19:34 +0200 Subject: [PATCH 08/37] added client tls tests --- examples/tls/simple-kafka-cluster-tls.yaml | 2 +- rust/crd/src/lib.rs | 6 +- tests/templates/kuttl/tls/20-assert.yaml | 4 +- .../kuttl/tls/20-install-kafka.yaml.j2 | 42 ++++++++++- tests/templates/kuttl/tls/30-assert.yaml | 7 -- tests/templates/kuttl/tls/30-assert.yaml.j2 | 12 +++ tests/templates/kuttl/tls/test_client_tls.sh | 62 ++++++++++++++++ .../templates/kuttl/tls/test_internal_tls.sh | 0 tests/templates/kuttl/tls/test_tls.sh.j2 | 73 ------------------- tests/test-definition.yaml | 36 ++++++--- 10 files changed, 146 insertions(+), 98 deletions(-) delete mode 100644 tests/templates/kuttl/tls/30-assert.yaml create mode 100644 tests/templates/kuttl/tls/30-assert.yaml.j2 create mode 100755 tests/templates/kuttl/tls/test_client_tls.sh create mode 100755 tests/templates/kuttl/tls/test_internal_tls.sh delete mode 100755 tests/templates/kuttl/tls/test_tls.sh.j2 diff --git a/examples/tls/simple-kafka-cluster-tls.yaml b/examples/tls/simple-kafka-cluster-tls.yaml index 47ba8960..c36615a6 100644 --- a/examples/tls/simple-kafka-cluster-tls.yaml +++ b/examples/tls/simple-kafka-cluster-tls.yaml @@ -35,4 +35,4 @@ spec: brokers: roleGroups: default: - replicas: 2 + replicas: 3 diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index d2c69041..9d8866d2 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -34,7 +34,8 @@ pub const SERVER_PROPERTIES_FILE: &str = "server.properties"; pub const KAFKA_HEAP_OPTS: &str = "KAFKA_HEAP_OPTS"; // server_properties pub const LOG_DIRS_VOLUME_NAME: &str = "log-dirs"; -// TLS +// - listener +// - TLS pub const TLS_DEFAULT_SECRET_CLASS: &str = "tls"; pub const SSL_KEYSTORE_LOCATION: &str = "ssl.keystore.location"; pub const SSL_KEYSTORE_PASSWORD: &str = "ssl.keystore.password"; @@ -44,13 +45,14 @@ pub const SSL_TRUSTSTORE_PASSWORD: &str = "ssl.truststore.password"; pub const SSL_TRUSTSTORE_TYPE: &str = "ssl.truststore.type"; pub const SSL_STORE_PASSWORD: &str = "changeit"; pub const SSL_CLIENT_AUTH: &str = "ssl.client.auth"; -// TLS internal +// - TLS internal pub const SECURITY_INTER_BROKER_PROTOCOL: &str = "security.inter.broker.protocol"; // directories pub const STACKABLE_TMP_DIR: &str = "/stackable/tmp"; pub const STACKABLE_DATA_DIR: &str = "/stackable/data"; pub const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; pub const STACKABLE_TLS_CERTS_DIR: &str = "/stackable/certificates"; +pub const STACKABLE_TLS_CERTS_INTERNAL_DIR: &str = "/stackable/certificates_internal"; pub const SYSTEM_TRUST_STORE_DIR: &str = "/etc/pki/java/cacerts"; const JVM_HEAP_FACTOR: f32 = 0.8; diff --git a/tests/templates/kuttl/tls/20-assert.yaml b/tests/templates/kuttl/tls/20-assert.yaml index 8dd80887..821a471f 100644 --- a/tests/templates/kuttl/tls/20-assert.yaml +++ b/tests/templates/kuttl/tls/20-assert.yaml @@ -10,5 +10,5 @@ kind: StatefulSet metadata: name: test-kafka-broker-default status: - readyReplicas: 2 - replicas: 2 + readyReplicas: 3 + replicas: 3 diff --git a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 index cd6d0ea7..a1e6d252 100644 --- a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 +++ b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 @@ -10,11 +10,49 @@ spec: apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaCluster metadata: - name: simple-kafka + name: test-kafka spec: version: {{ test_scenario['values']['kafka'] }} zookeeperConfigMapName: test-kafka-znode + config: +{% if test_scenario['values']['use-client-tls'] == 'true' %} + tls: + secretClass: tls + clientAuthentication: + authenticationClass: kafka-client-tls +{% else %} + tls: null +{% endif %} +{% if test_scenario['values']['use-internal-tls'] == 'true' %} + internalTls: + secretClass: tls +{% endif %} brokers: roleGroups: default: - replicas: 2 + replicas: 3 + +{% if test_scenario['values']['use-client-tls'] == 'true' %} +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: kafka-client-tls +spec: + provider: + tls: + clientCertSecretClass: kafka-client-tls +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: kafka-client-tls +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-kafka-client-ca + namespace: default + autoGenerate: true +{% endif %} diff --git a/tests/templates/kuttl/tls/30-assert.yaml b/tests/templates/kuttl/tls/30-assert.yaml deleted file mode 100644 index 6d3ab737..00000000 --- a/tests/templates/kuttl/tls/30-assert.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -metadata: - name: test-tls -commands: - - script: kubectl exec -n $NAMESPACE test-zk-server-primary-0 -- /tmp/test_tls.sh $NAMESPACE diff --git a/tests/templates/kuttl/tls/30-assert.yaml.j2 b/tests/templates/kuttl/tls/30-assert.yaml.j2 new file mode 100644 index 00000000..799d975e --- /dev/null +++ b/tests/templates/kuttl/tls/30-assert.yaml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-tls +commands: +{% if test_scenario['values']['use-client-tls'] == 'true' %} + - script: kubectl exec -n $NAMESPACE test-kafka-broker-default-0 -- /tmp/test_client_tls.sh $NAMESPACE +{% endif %} +{% if test_scenario['values']['use-internal-tls'] == 'true' %} + - script: kubectl exec -n $NAMESPACE test-kafka-broker-default-0 -- /tmp/test_internal_tls.sh $NAMESPACE +{% endif %} diff --git a/tests/templates/kuttl/tls/test_client_tls.sh b/tests/templates/kuttl/tls/test_client_tls.sh new file mode 100755 index 00000000..f5178eb6 --- /dev/null +++ b/tests/templates/kuttl/tls/test_client_tls.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Usage: test_tls.sh namespace + +NAMESPACE=$1 + +# to be safe +unset TOPIC +unset BAD_TOPIC + +SERVER="test-kafka-broker-default-0.test-kafka-broker-default.${NAMESPACE}.svc.cluster.local:9093" + +echo "Start TLS testing..." +############################################################################ +# Test the secured connection +############################################################################ +# create random topics +TOPIC=$(tr -dc A-Za-z0-9 /tmp/client.config + +if /stackable/kafka/bin/kafka-topics.sh --create --topic $TOPIC --bootstrap-server $SERVER --command-config /tmp/client.config +then + echo "[SUCCESS] Secure client topic created!" +else + echo "[ERROR] Secure client topic creation failed!" + exit 1 +fi + +if /stackable/kafka/bin/kafka-topics.sh --list --topic $TOPIC --bootstrap-server $SERVER --command-config /tmp/client.config | grep $TOPIC +then + echo "[SUCCESS] Secure client topic read!" +else + echo "[ERROR] Secure client topic read failed!" + exit 1 +fi + +############################################################################ +# Test the connection without certificates +############################################################################ +if /stackable/kafka/bin/kafka-topics.sh --create --topic $BAD_TOPIC --bootstrap-server $SERVER &> /dev/null +then + echo "[ERROR] Secure client topic created without certificates!" + exit 1 +else + echo "[SUCCESS] Secure client topic creation failed without certificates!" +fi + +############################################################################ +# Test the connection with bad host name +############################################################################ +if /stackable/kafka/bin/kafka-topics.sh --create --topic $BAD_TOPIC --bootstrap-server localhost:9093 --command-config /tmp/client.config &> /dev/null +then + echo "[ERROR] Secure client topic created with bad host name!" + exit 1 +else + echo "[SUCCESS] Secure client topic creation failed with bad host name!" +fi + +echo "All TLS tests successful!" +exit 0 diff --git a/tests/templates/kuttl/tls/test_internal_tls.sh b/tests/templates/kuttl/tls/test_internal_tls.sh new file mode 100755 index 00000000..e69de29b diff --git a/tests/templates/kuttl/tls/test_tls.sh.j2 b/tests/templates/kuttl/tls/test_tls.sh.j2 deleted file mode 100755 index 0c88e51f..00000000 --- a/tests/templates/kuttl/tls/test_tls.sh.j2 +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash -# Usage: test_tls.sh namespace - -NAMESPACE=$1 - -{% if test_scenario['values']['use-client-tls'] == 'true' %} -SERVER="test-zk-server-primary-1.test-zk-server-primary.${NAMESPACE}.svc.cluster.local:2282" -{% else %} -SERVER="test-zk-server-primary-1.test-zk-server-primary.${NAMESPACE}.svc.cluster.local:2181" -{% endif %} - -# just to be safe... -unset QUORUM_STORE_SECRET -unset CLIENT_STORE_SECRET -unset CLIENT_JVMFLAGS - -echo "Start TLS testing..." -############################################################################ -# Test the plaintext unsecured connection -############################################################################ -if ! /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; -then - echo "[ERROR] Could not establish unsecure connection!" - exit 1 -fi -echo "[SUCCESS] Unsecure client connection established!" - -{% if test_scenario['values']['use-client-tls'] == 'true' %} -############################################################################ -# We set the correct client tls credentials and expect to be able to connect -############################################################################ -CLIENT_STORE_SECRET="$(< /stackable/rwconfig/zoo.cfg grep "ssl.keyStore.password" | cut -d "=" -f2)" -export CLIENT_STORE_SECRET -export CLIENT_JVMFLAGS=" --Dzookeeper.authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider --Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty --Dzookeeper.client.secure=true --Dzookeeper.ssl.keyStore.location=/stackable/tls/client/keystore.p12 --Dzookeeper.ssl.keyStore.password=${CLIENT_STORE_SECRET} --Dzookeeper.ssl.trustStore.location=/stackable/tls/client/truststore.p12 --Dzookeeper.ssl.trustStore.password=${CLIENT_STORE_SECRET}" - -if ! /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; -then - echo "[ERROR] Could not establish secure connection using client certificates!" - exit 1 -fi -echo "[SUCCESS] Secure and authenticated client connection established!" - -{% endif %} -############################################################################ -# We set the (wrong) quorum tls credentials and expect to fail (wrong certificate) -############################################################################ -QUORUM_STORE_SECRET="$(< /stackable/rwconfig/zoo.cfg grep "ssl.quorum.keyStore.password" | cut -d "=" -f2)" -export QUORUM_STORE_SECRET -export CLIENT_JVMFLAGS=" --Dzookeeper.authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider --Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty --Dzookeeper.client.secure=true --Dzookeeper.ssl.keyStore.location=/stackable/tls/quorum/keystore.p12 --Dzookeeper.ssl.keyStore.password=${QUORUM_STORE_SECRET} --Dzookeeper.ssl.trustStore.location=/stackable/tls/quorum/truststore.p12 --Dzookeeper.ssl.trustStore.password=${QUORUM_STORE_SECRET}" - -if /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; -then - echo "[ERROR] Could establish secure connection with quorum certificates (should not be happening)!" - exit 1 -fi -echo "[SUCCESS] Could not establish secure connection with (wrong) quorum certificates!" - -echo "All TLS tests successful!" -exit 0 diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 79e91463..956b5aa7 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -2,14 +2,14 @@ dimensions: - name: kafka values: - - 2.7.1-stackable0 - - 2.8.1-stackable0 - - 3.1.0-stackable0 + #- 2.7.1-stackable0 + #- 2.8.1-stackable0 + #- 3.1.0-stackable0 - 3.2.0-stackable0.1.0 - name: zookeeper values: - - 3.6.3-stackable0.7.1 - - 3.7.0-stackable0.7.1 + #- 3.6.3-stackable0.7.1 + #- 3.7.0-stackable0.7.1 - 3.8.0-stackable0.7.1 - name: upgrade_old values: @@ -19,13 +19,27 @@ dimensions: - name: upgrade_new values: - 3.2.0-stackable0.1.0 + - name: use-client-tls + values: + - "true" + #- "false" + - name: use-internal-tls + values: + #- "true" + - "false" tests: - - name: smoke +# - name: smoke +# dimensions: +# - kafka +# - zookeeper +# - name: upgrade +# dimensions: +# - zookeeper +# - upgrade_new +# - upgrade_old + - name: tls dimensions: - kafka - zookeeper - - name: upgrade - dimensions: - - zookeeper - - upgrade_new - - upgrade_old + - use-client-tls + - use-internal-tls From 940e05ef66320ee36eb1ca93667701393e459e16 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 29 Jul 2022 16:19:49 +0200 Subject: [PATCH 09/37] added client tls tests --- tests/templates/kuttl/tls/30-prepare-test-kafka.yaml | 5 ----- .../templates/kuttl/tls/30-prepare-test-kafka.yaml.j2 | 10 ++++++++++ 2 files changed, 10 insertions(+), 5 deletions(-) delete mode 100644 tests/templates/kuttl/tls/30-prepare-test-kafka.yaml create mode 100644 tests/templates/kuttl/tls/30-prepare-test-kafka.yaml.j2 diff --git a/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml b/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml deleted file mode 100644 index 2bc6aebc..00000000 --- a/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: kubectl cp -n $NAMESPACE ./test_tls.sh test-kafka-broker-default-0:/tmp diff --git a/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml.j2 b/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml.j2 new file mode 100644 index 00000000..75569f35 --- /dev/null +++ b/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +{% if test_scenario['values']['use-client-tls'] == 'true' %} + - script: kubectl cp -n $NAMESPACE ./test_client_tls.sh test-kafka-broker-default-0:/tmp +{% endif %} +{% if test_scenario['values']['use-internal-tls'] == 'true' %} + - script: kubectl cp -n $NAMESPACE ./test_internal_tls.sh test-kafka-broker-default-0:/tmp +{% endif %} From d5ff5fe4b247a31a5ac7702d031d8ef14a6278cd Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 29 Jul 2022 16:55:42 +0200 Subject: [PATCH 10/37] wip --- examples/tls/simple-kafka-cluster-tls.yaml | 2 ++ rust/operator/src/kafka_controller.rs | 15 +-------------- tests/templates/kuttl/tls/test_internal_tls.sh | 4 ++++ 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/examples/tls/simple-kafka-cluster-tls.yaml b/examples/tls/simple-kafka-cluster-tls.yaml index c36615a6..18c90535 100644 --- a/examples/tls/simple-kafka-cluster-tls.yaml +++ b/examples/tls/simple-kafka-cluster-tls.yaml @@ -32,6 +32,8 @@ spec: config: tls: secretClass: tls + internalTls: + secretClass: tls brokers: roleGroups: default: diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index 5f51e719..4405bc14 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -444,20 +444,7 @@ fn build_broker_rolegroup_service( .build(), spec: Some(ServiceSpec { cluster_ip: Some("None".to_string()), - ports: Some(vec![ - ServicePort { - name: Some("kafka".to_string()), - port: CLIENT_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }, - ServicePort { - name: Some("metrics".to_string()), - port: METRICS_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }, - ]), + ports: Some(service_ports(kafka)), selector: Some(role_group_selector_labels( kafka, APP_NAME, diff --git a/tests/templates/kuttl/tls/test_internal_tls.sh b/tests/templates/kuttl/tls/test_internal_tls.sh index e69de29b..03640117 100755 --- a/tests/templates/kuttl/tls/test_internal_tls.sh +++ b/tests/templates/kuttl/tls/test_internal_tls.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Usage: test_tls.sh namespace + +NAMESPACE=$1 From fbba669dd5e8a7da70aa81739f5ba7f673b75833 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 1 Aug 2022 12:41:05 +0200 Subject: [PATCH 11/37] internal and external tls working --- examples/tls/simple-kafka-cluster-tls.yaml | 15 +- rust/crd/src/lib.rs | 68 +++++++- rust/operator/src/command.rs | 13 +- rust/operator/src/kafka_controller.rs | 159 +++++++++++------- .../kuttl/tls/20-install-kafka.yaml.j2 | 18 +- 5 files changed, 201 insertions(+), 72 deletions(-) diff --git a/examples/tls/simple-kafka-cluster-tls.yaml b/examples/tls/simple-kafka-cluster-tls.yaml index 18c90535..f07aac00 100644 --- a/examples/tls/simple-kafka-cluster-tls.yaml +++ b/examples/tls/simple-kafka-cluster-tls.yaml @@ -22,6 +22,19 @@ spec: clusterRef: name: simple-zk --- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: kafka-internal-tls +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-kafka-internal-tls-ca + namespace: default + autoGenerate: true +--- apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaCluster metadata: @@ -33,7 +46,7 @@ spec: tls: secretClass: tls internalTls: - secretClass: tls + secretClass: kafka-internal-tls brokers: roleGroups: default: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 9d8866d2..b44a8982 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -26,6 +26,8 @@ pub const CLIENT_PORT_NAME: &str = "http"; pub const CLIENT_PORT: u16 = 9092; pub const SECURE_CLIENT_PORT_NAME: &str = "https"; pub const SECURE_CLIENT_PORT: u16 = 9093; +pub const INTERNAL_PORT: u16 = 19092; +pub const SECURE_INTERNAL_PORT: u16 = 19093; pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: u16 = 9606; // config files @@ -35,6 +37,9 @@ pub const KAFKA_HEAP_OPTS: &str = "KAFKA_HEAP_OPTS"; // server_properties pub const LOG_DIRS_VOLUME_NAME: &str = "log-dirs"; // - listener +pub const LISTENER_SECURITY_PROTOCOL_MAP: &str = "listener.security.protocol.map"; +pub const LISTENER: &str = "listeners"; +pub const ADVERTISED_LISTENER: &str = "advertised.listeners"; // - TLS pub const TLS_DEFAULT_SECRET_CLASS: &str = "tls"; pub const SSL_KEYSTORE_LOCATION: &str = "ssl.keystore.location"; @@ -45,8 +50,20 @@ pub const SSL_TRUSTSTORE_PASSWORD: &str = "ssl.truststore.password"; pub const SSL_TRUSTSTORE_TYPE: &str = "ssl.truststore.type"; pub const SSL_STORE_PASSWORD: &str = "changeit"; pub const SSL_CLIENT_AUTH: &str = "ssl.client.auth"; +pub const SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: &str = "ssl.endpoint.identification.algorithm"; // - TLS internal pub const SECURITY_INTER_BROKER_PROTOCOL: &str = "security.inter.broker.protocol"; +pub const INTER_BROKER_LISTENER_NAME: &str = "inter.broker.listener.name"; +pub const INTER_SSL_KEYSTORE_LOCATION: &str = "listener.name.internal.ssl.keystore.location"; +pub const INTER_SSL_KEYSTORE_PASSWORD: &str = "listener.name.internal.ssl.keystore.password"; +pub const INTER_SSL_KEYSTORE_TYPE: &str = "listener.name.internal.ssl.keystore.type"; +pub const INTER_SSL_TRUSTSTORE_LOCATION: &str = "listener.name.internal.ssl.truststore.location"; +pub const INTER_SSL_TRUSTSTORE_PASSWORD: &str = "listener.name.internal.ssl.truststore.password"; +pub const INTER_SSL_TRUSTSTORE_TYPE: &str = "listener.name.internal.ssl.truststore.type"; +pub const INTER_SSL_STORE_PASSWORD: &str = "changeit"; +pub const INTER_SSL_CLIENT_AUTH: &str = "listener.name.internal.ssl.client.auth"; +pub const INTER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: &str = + "listener.name.internal.ssl.endpoint.identification.algorithm"; // directories pub const STACKABLE_TMP_DIR: &str = "/stackable/tmp"; pub const STACKABLE_DATA_DIR: &str = "/stackable/data"; @@ -400,6 +417,7 @@ impl Configuration for KafkaConfig { Some("true".to_string()), ); } + // Client TLS if resource.client_tls_secret_class().is_some() { config.insert( @@ -424,21 +442,57 @@ impl Configuration for KafkaConfig { // Authentication if resource.client_authentication_class().is_some() { config.insert(SSL_CLIENT_AUTH.to_string(), Some("required".to_string())); + config.insert( + SSL_ENDPOINT_IDENTIFICATION_ALGORITHM.to_string(), + Some("HTTPS".to_string()), + ); } } - // We require authentication - if resource.client_authentication_class().is_some() { - config.insert(SSL_CLIENT_AUTH.to_string(), Some("required".to_string())); - } - // Internal TLS if resource.internal_tls_secret_class().is_some() { config.insert( - SECURITY_INTER_BROKER_PROTOCOL.to_string(), - Some("SSL".to_string()), + INTER_SSL_KEYSTORE_LOCATION.to_string(), + Some(format!("{}/keystore.p12", STACKABLE_TLS_CERTS_INTERNAL_DIR)), + ); + config.insert( + INTER_SSL_KEYSTORE_PASSWORD.to_string(), + Some(INTER_SSL_STORE_PASSWORD.to_string()), + ); + config.insert( + INTER_SSL_KEYSTORE_TYPE.to_string(), + Some("PKCS12".to_string()), + ); + config.insert( + INTER_SSL_TRUSTSTORE_LOCATION.to_string(), + Some(format!( + "{}/truststore.p12", + STACKABLE_TLS_CERTS_INTERNAL_DIR + )), + ); + config.insert( + INTER_SSL_TRUSTSTORE_PASSWORD.to_string(), + Some(INTER_SSL_STORE_PASSWORD.to_string()), + ); + config.insert( + INTER_SSL_TRUSTSTORE_TYPE.to_string(), + Some("PKCS12".to_string()), + ); + config.insert( + INTER_SSL_CLIENT_AUTH.to_string(), + Some("required".to_string()), + ); + config.insert( + INTER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM.to_string(), + Some("HTTPS".to_string()), ); } + + // common + config.insert( + INTER_BROKER_LISTENER_NAME.to_string(), + Some("internal".to_string()), + ); } Ok(config) diff --git a/rust/operator/src/command.rs b/rust/operator/src/command.rs index cb31a942..17f695b2 100644 --- a/rust/operator/src/command.rs +++ b/rust/operator/src/command.rs @@ -1,6 +1,7 @@ use stackable_kafka_crd::{ KafkaCluster, CLIENT_PORT, SECURE_CLIENT_PORT, SSL_STORE_PASSWORD, STACKABLE_DATA_DIR, - STACKABLE_TLS_CERTS_DIR, STACKABLE_TMP_DIR, SYSTEM_TRUST_STORE_DIR, + STACKABLE_TLS_CERTS_DIR, STACKABLE_TLS_CERTS_INTERNAL_DIR, STACKABLE_TMP_DIR, + SYSTEM_TRUST_STORE_DIR, }; pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { @@ -12,11 +13,19 @@ pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { if kafka.client_tls_secret_class().is_some() { args.extend(create_key_and_trust_store( STACKABLE_TLS_CERTS_DIR, - "stackable-ca-cert", + "stackable-tls-ca-cert", )); args.extend(chown_and_chmod(STACKABLE_TLS_CERTS_DIR)); } + if kafka.internal_tls_secret_class().is_some() { + args.extend(create_key_and_trust_store( + STACKABLE_TLS_CERTS_INTERNAL_DIR, + "stackable-internal-tls-ca-cert", + )); + args.extend(chown_and_chmod(STACKABLE_TLS_CERTS_INTERNAL_DIR)); + } + args.extend(chown_and_chmod(STACKABLE_DATA_DIR)); args.extend(chown_and_chmod(STACKABLE_TMP_DIR)); diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index 4405bc14..4f47e046 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -3,13 +3,15 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_kafka_crd::{ KafkaCluster, KafkaRole, TlsSecretClass, APP_NAME, CLIENT_PORT, CLIENT_PORT_NAME, - KAFKA_HEAP_OPTS, LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, SECURE_CLIENT_PORT, - SECURE_CLIENT_PORT_NAME, SERVER_PROPERTIES_FILE, STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, - STACKABLE_TLS_CERTS_DIR, STACKABLE_TMP_DIR, + INTERNAL_PORT, KAFKA_HEAP_OPTS, LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, + SECURE_CLIENT_PORT, SECURE_CLIENT_PORT_NAME, SECURE_INTERNAL_PORT, SERVER_PROPERTIES_FILE, + STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_TLS_CERTS_DIR, + STACKABLE_TLS_CERTS_INTERNAL_DIR, STACKABLE_TMP_DIR, }; use stackable_operator::builder::{ SecretOperatorVolumeSourceBuilder, SecurityContextBuilder, VolumeBuilder, }; +use stackable_operator::kube::ResourceExt; use stackable_operator::{ builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder}, commons::opa::OpaApiVersion, @@ -66,6 +68,8 @@ pub struct Ctx { pub enum Error { #[snafu(display("object has no name"))] ObjectHasNoName, + #[snafu(display("object has no namespace"))] + ObjectHasNoNamespace, #[snafu(display("object defines no version"))] ObjectHasNoVersion, #[snafu(display("object defines no broker role"))] @@ -510,6 +514,15 @@ fn build_broker_rolegroup_statefulset( pod_builder.add_volume(create_tls_volume("tls-certificate", Some(tls))); } + if let Some(tls_internal) = kafka.internal_tls_secret_class() { + cb_prepare.add_volume_mount("internal-tls-certificate", STACKABLE_TLS_CERTS_INTERNAL_DIR); + cb_kafka.add_volume_mount("internal-tls-certificate", STACKABLE_TLS_CERTS_INTERNAL_DIR); + pod_builder.add_volume(create_tls_volume( + "internal-tls-certificate", + Some(tls_internal), + )); + } + let container_get_svc = ContainerBuilder::new("get-svc") .image("docker.stackable.tech/stackable/tools:0.2.0-stackable0.3.0") .command(vec!["bash".to_string()]) @@ -594,6 +607,17 @@ fn build_broker_rolegroup_statefulset( }), ..EnvVar::default() }); + env.push(EnvVar { + name: "POD_NAME".to_string(), + value_from: Some(EnvVarSource { + field_ref: Some(ObjectFieldSelector { + api_version: Some("v1".to_string()), + field_path: "metadata.name".to_string(), + }), + ..EnvVarSource::default() + }), + ..EnvVar::default() + }); // add env var for log4j if set if kafka.spec.log4j.is_some() { @@ -608,10 +632,16 @@ fn build_broker_rolegroup_statefulset( let jvm_args = format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar={}:/stackable/jmx/broker.yaml", METRICS_PORT); let zookeeper_override = "--override \"zookeeper.connect=$ZOOKEEPER\""; - let listeners_override = format!("--override \"listeners={}\"", get_listeners(kafka)); + + let kafka_listeners = get_kafka_listener(kafka, rolegroup_ref)?; + let listeners_override = format!("--override \"listeners={}\"", kafka_listeners.listener); let advertised_listeners_override = format!( "--override \"advertised.listeners={}\"", - get_advertised_listeners(kafka) + kafka_listeners.advertised_listener + ); + let listener_security_protocol_map_override = format!( + "--override \"listener.security.protocol.map={}\"", + kafka_listeners.listener_security_protocol_map ); let opa_url_override = opa_connect_string.map_or("".to_string(), |opa| { format!("--override \"opa.authorizer.url={}\"", opa) @@ -628,6 +658,7 @@ fn build_broker_rolegroup_statefulset( zookeeper_override, &listeners_override, &advertised_listeners_override, + &listener_security_protocol_map_override, &opa_url_override, ] .join(" "), @@ -652,8 +683,8 @@ fn build_broker_rolegroup_statefulset( // If the broker is able to get its fellow cluster members then it has at least completed basic registration at some point command: Some(kcat_container_cmd_args(kafka)), }), - timeout_seconds: Some(3), - period_seconds: Some(1), + timeout_seconds: Some(5), + period_seconds: Some(2), ..Probe::default() }) .build(); @@ -739,59 +770,6 @@ pub fn error_policy(_error: &Error, _ctx: Arc) -> Action { Action::requeue(Duration::from_secs(5)) } -fn get_listeners(kafka: &KafkaCluster) -> String { - // If both client and internal TLS are set we do not need the HTTP port. - if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { - format!("SSL://0.0.0.0:{}", secure = SECURE_CLIENT_PORT) - // If only client TLS is required we need to set the HTTPS port and keep the HTTP port - // for internal communications. - // If only internal TLS is required we need to set the HTTPS port and keep the HTTP port - // for client communications. - } else if kafka.client_tls_secret_class().is_some() - || kafka.internal_tls_secret_class().is_some() - { - format!( - "PLAINTEXT://0.0.0.0:{insecure},SSL://0.0.0.0:{secure}", - insecure = CLIENT_PORT, - secure = SECURE_CLIENT_PORT - ) - } - // If no is TLS specified the HTTP port is sufficient - else { - format!("PLAINTEXT://0.0.0.0:{insecure}", insecure = CLIENT_PORT) - } -} - -fn get_advertised_listeners(kafka: &KafkaCluster) -> String { - // If both client and internal TLS are set we do not need the HTTP port. - if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { - format!( - "SSL://$NODE:$(cat {dir}/{secure}_nodeport)", - dir = STACKABLE_TMP_DIR, - secure = SECURE_CLIENT_PORT_NAME - ) - // If only client TLS is required we need to set the HTTPS port and keep the HTTP port - // for internal communications. - // If only internal TLS is required we need to set the HTTPS port and keep the HTTP port - // for client communications. - } else if kafka.client_tls_secret_class().is_some() - || kafka.internal_tls_secret_class().is_some() - { - format!("PLAINTEXT://$NODE:$(cat {dir}/{insecure}_nodeport),SSL://$NODE:$(cat /{dir}/{secure}_nodeport)", - dir = STACKABLE_TMP_DIR, - insecure = CLIENT_PORT_NAME, - secure = SECURE_CLIENT_PORT_NAME ) - } - // If no is TLS specified the HTTP port is sufficient - else { - format!( - "PLAINTEXT://$NODE:$(cat {dir}/{insecure}_nodeport)", - dir = STACKABLE_TMP_DIR, - insecure = CLIENT_PORT_NAME - ) - } -} - fn service_ports(kafka: &KafkaCluster) -> Vec { let mut ports = vec![ServicePort { name: Some(METRICS_PORT_NAME.to_string()), @@ -909,6 +887,67 @@ fn create_tls_volume(volume_name: &str, tls_secret_class: Option<&TlsSecretClass .build() } +/// Extract the nodeport from the nodeport service fn get_node_port(directory: &str, port_name: &str) -> String { format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee {dir}/{name}_nodeport", dir = directory, name = port_name) } + +struct KafkaListener { + pub listener: String, + pub advertised_listener: String, + pub listener_security_protocol_map: String, +} + +/// Returns the `listener`, `advertised.listener` and `listener.security.protocol.map` properties +/// depending on internal and external TLS settings. +fn get_kafka_listener( + kafka: &KafkaCluster, + rolegroup_ref: &RoleGroupRef, +) -> Result { + let pod_fqdn = format!( + "$POD_NAME.{}.{}.svc.cluster.local", + rolegroup_ref.object_name(), + kafka.namespace().context(ObjectHasNoNamespaceSnafu)? + ); + + if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { + // If both client and internal TLS are set we do not need the HTTP port. + Ok(KafkaListener { + listener: format!("internal://0.0.0.0:{SECURE_INTERNAL_PORT},SSL://0.0.0.0:{SECURE_CLIENT_PORT}"), + advertised_listener: format!( + "internal://{pod}:{SECURE_INTERNAL_PORT},SSL://$NODE:$(cat {STACKABLE_TMP_DIR}/{SECURE_CLIENT_PORT_NAME}_nodeport)", pod = pod_fqdn + ), + listener_security_protocol_map: "internal:SSL,SSL:SSL".to_string() + }) + } else if kafka.client_tls_secret_class().is_some() { + // If only client TLS is required we need to set the HTTPS port and keep the HTTP port + // for internal communications. + Ok(KafkaListener { + listener: format!("internal://0.0.0.0:{INTERNAL_PORT},SSL://0.0.0.0:{SECURE_CLIENT_PORT}"), + advertised_listener: format!( + "internal://{pod}:{INTERNAL_PORT},SSL://$NODE:$(cat {STACKABLE_TMP_DIR}/{SECURE_CLIENT_PORT_NAME}_nodeport)", pod = pod_fqdn + ), + listener_security_protocol_map: "internal:PLAINTEXT,SSL:SSL".to_string() + }) + } else if kafka.internal_tls_secret_class().is_some() { + // If only internal TLS is required we need to set the HTTPS port and keep the HTTP port + // for client communications. + Ok(KafkaListener { + listener: format!("internal://0.0.0.0:{SECURE_INTERNAL_PORT},SSL://0.0.0.0:{CLIENT_PORT}"), + advertised_listener: format!( + "internal://{pod}:{SECURE_INTERNAL_PORT},PLAINTEXT://$NODE:$(cat {STACKABLE_TMP_DIR}/{CLIENT_PORT_NAME}_nodeport)", pod = pod_fqdn + ), + listener_security_protocol_map: "internal:SSL,PLAINTEXT:PLAINTEXT".to_string() + }) + } else { + // If no is TLS specified the HTTP port is sufficient + Ok(KafkaListener { + listener: format!("internal://0.0.0.0:{INTERNAL_PORT},PLAINTEXT://0.0.0.0:{CLIENT_PORT}"), + advertised_listener: format!( + "internal://{pod}:{INTERNAL_PORT},PLAINTEXT://$NODE:$(cat {STACKABLE_TMP_DIR}/{CLIENT_PORT_NAME}_nodeport)", + pod = pod_fqdn + ), + listener_security_protocol_map: "internal:PLAINTEXT,PLAINTEXT:PLAINTEXT".to_string() + }) + } +} diff --git a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 index a1e6d252..3171748c 100644 --- a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 +++ b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 @@ -6,6 +6,21 @@ metadata: spec: clusterRef: name: test-zk +{% if test_scenario['values']['use-internal-tls'] == 'true' %} +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: test-kafka-internal-tls +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-test-kafka-internal-tls-ca + namespace: default + autoGenerate: true +{% endif %} --- apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaCluster @@ -25,13 +40,12 @@ spec: {% endif %} {% if test_scenario['values']['use-internal-tls'] == 'true' %} internalTls: - secretClass: tls + secretClass: test-kafka-internal-tls {% endif %} brokers: roleGroups: default: replicas: 3 - {% if test_scenario['values']['use-client-tls'] == 'true' %} --- apiVersion: authentication.stackable.tech/v1alpha1 From e798b76fbd0012b50ed91ee29e7da0ad3b9002a0 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 1 Aug 2022 12:50:24 +0200 Subject: [PATCH 12/37] cleanup --- rust/operator/src/command.rs | 29 ++++++++++++++++++++++++--- rust/operator/src/kafka_controller.rs | 25 ++--------------------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/rust/operator/src/command.rs b/rust/operator/src/command.rs index 17f695b2..013d75b1 100644 --- a/rust/operator/src/command.rs +++ b/rust/operator/src/command.rs @@ -1,7 +1,7 @@ use stackable_kafka_crd::{ - KafkaCluster, CLIENT_PORT, SECURE_CLIENT_PORT, SSL_STORE_PASSWORD, STACKABLE_DATA_DIR, - STACKABLE_TLS_CERTS_DIR, STACKABLE_TLS_CERTS_INTERNAL_DIR, STACKABLE_TMP_DIR, - SYSTEM_TRUST_STORE_DIR, + KafkaCluster, CLIENT_PORT, CLIENT_PORT_NAME, SECURE_CLIENT_PORT, SECURE_CLIENT_PORT_NAME, + SSL_STORE_PASSWORD, STACKABLE_DATA_DIR, STACKABLE_TLS_CERTS_DIR, + STACKABLE_TLS_CERTS_INTERNAL_DIR, STACKABLE_TMP_DIR, SYSTEM_TRUST_STORE_DIR, }; pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { @@ -32,6 +32,24 @@ pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { args.join(" && ") } +pub fn get_svc_container_args(kafka: &KafkaCluster) -> String { + if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { + get_node_port(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME) + } else if kafka.client_tls_secret_class().is_some() + || kafka.internal_tls_secret_class().is_some() + { + [ + get_node_port(STACKABLE_TMP_DIR, CLIENT_PORT_NAME), + get_node_port(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME), + ] + .join(" && ") + } + // If no is TLS specified the HTTP port is sufficient + else { + get_node_port(STACKABLE_TMP_DIR, CLIENT_PORT_NAME) + } +} + pub fn kcat_container_cmd_args(kafka: &KafkaCluster) -> Vec { let mut args = vec!["kcat".to_string()]; @@ -84,3 +102,8 @@ fn chown_and_chmod(directory: &str) -> Vec { format!("chmod -R a=,u=rwX {dir}", dir = directory), ] } + +/// Extract the nodeport from the nodeport service +fn get_node_port(directory: &str, port_name: &str) -> String { + format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee {dir}/{name}_nodeport", dir = directory, name = port_name) +} diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index 4f47e046..b3a8fe97 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -45,7 +45,7 @@ use std::{ }; use strum::{EnumDiscriminants, IntoStaticStr}; -use crate::command::kcat_container_cmd_args; +use crate::command::{get_svc_container_args, kcat_container_cmd_args}; use crate::{ command, discovery::{self, build_discovery_configmaps}, @@ -489,23 +489,7 @@ fn build_broker_rolegroup_statefulset( .context(KafkaVersionParseFailureSnafu)?; let image = format!("docker.stackable.tech/stackable/kafka:{}", image_version); - let get_svc_args = if kafka.client_tls_secret_class().is_some() - && kafka.internal_tls_secret_class().is_some() - { - get_node_port(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME) - } else if kafka.client_tls_secret_class().is_some() - || kafka.internal_tls_secret_class().is_some() - { - [ - get_node_port(STACKABLE_TMP_DIR, CLIENT_PORT_NAME), - get_node_port(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME), - ] - .join(" && ") - } - // If no is TLS specified the HTTP port is sufficient - else { - get_node_port(STACKABLE_TMP_DIR, CLIENT_PORT_NAME) - }; + let get_svc_args = get_svc_container_args(kafka); if let Some(tls) = kafka.client_tls_secret_class() { cb_prepare.add_volume_mount("tls-certificate", STACKABLE_TLS_CERTS_DIR); @@ -887,11 +871,6 @@ fn create_tls_volume(volume_name: &str, tls_secret_class: Option<&TlsSecretClass .build() } -/// Extract the nodeport from the nodeport service -fn get_node_port(directory: &str, port_name: &str) -> String { - format!("kubectl get service \"$POD_NAME\" -o jsonpath='{{.spec.ports[?(@.name==\"{name}\")].nodePort}}' | tee {dir}/{name}_nodeport", dir = directory, name = port_name) -} - struct KafkaListener { pub listener: String, pub advertised_listener: String, From 2e13c5816c0c9a28fa3ab05f269303228ec8f01c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 1 Aug 2022 18:32:22 +0200 Subject: [PATCH 13/37] wip - non tls tests failing due to patch -> null bug --- rust/crd/src/lib.rs | 5 -- rust/operator/src/command.rs | 9 ++- rust/operator/src/discovery.rs | 13 +++- rust/operator/src/kafka_controller.rs | 60 ++-------------- .../kuttl/smoke/01-install-kafka.yaml.j2 | 7 ++ .../kuttl/tls/20-install-kafka.yaml.j2 | 50 ++++++------- .../kuttl/upgrade/01-install-kafka.yaml.j2 | 7 ++ .../kuttl/upgrade/02-write-data.yaml | 27 ------- .../kuttl/upgrade/02-write-data.yaml.j2 | 53 ++++++++++++++ .../kuttl/upgrade/03-upgrade-kafka.yaml.j2 | 7 ++ .../templates/kuttl/upgrade/04-read-data.yaml | 35 ---------- .../kuttl/upgrade/04-read-data.yaml.j2 | 70 +++++++++++++++++++ 12 files changed, 190 insertions(+), 153 deletions(-) delete mode 100644 tests/templates/kuttl/upgrade/02-write-data.yaml create mode 100644 tests/templates/kuttl/upgrade/02-write-data.yaml.j2 delete mode 100644 tests/templates/kuttl/upgrade/04-read-data.yaml create mode 100644 tests/templates/kuttl/upgrade/04-read-data.yaml.j2 diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index b44a8982..95ef10d5 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -321,11 +321,6 @@ impl KafkaCluster { let spec: &KafkaClusterSpec = &self.spec; spec.config.internal_tls.as_ref() } - - /// Checks if we should use TLS to encrypt client connections. - pub fn is_client_secure(&self) -> bool { - self.client_tls_secret_class().is_some() || self.client_authentication_class().is_some() - } } /// Reference to a single `Pod` that is a component of a [`KafkaCluster`] diff --git a/rust/operator/src/command.rs b/rust/operator/src/command.rs index 013d75b1..f432a41a 100644 --- a/rust/operator/src/command.rs +++ b/rust/operator/src/command.rs @@ -5,12 +5,11 @@ use stackable_kafka_crd::{ }; pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { - let mut args = vec![ - // Copy system truststore to stackable truststore - format!("keytool -importkeystore -srckeystore {SYSTEM_TRUST_STORE_DIR} -srcstoretype jks -srcstorepass {SSL_STORE_PASSWORD} -destkeystore {STACKABLE_TLS_CERTS_DIR}/truststore.p12 -deststoretype pkcs12 -deststorepass {SSL_STORE_PASSWORD} -noprompt") - ]; + let mut args = vec![]; if kafka.client_tls_secret_class().is_some() { + // Copy system truststore to stackable truststore + args.push(format!("keytool -importkeystore -srckeystore {SYSTEM_TRUST_STORE_DIR} -srcstoretype jks -srcstorepass {SSL_STORE_PASSWORD} -destkeystore {STACKABLE_TLS_CERTS_DIR}/truststore.p12 -deststoretype pkcs12 -deststorepass {SSL_STORE_PASSWORD} -noprompt")); args.extend(create_key_and_trust_store( STACKABLE_TLS_CERTS_DIR, "stackable-tls-ca-cert", @@ -32,7 +31,7 @@ pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { args.join(" && ") } -pub fn get_svc_container_args(kafka: &KafkaCluster) -> String { +pub fn get_svc_container_cmd_args(kafka: &KafkaCluster) -> String { if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { get_node_port(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME) } else if kafka.client_tls_secret_class().is_some() diff --git a/rust/operator/src/discovery.rs b/rust/operator/src/discovery.rs index e03b08bb..0f87c83d 100644 --- a/rust/operator/src/discovery.rs +++ b/rust/operator/src/discovery.rs @@ -1,7 +1,9 @@ use std::{collections::BTreeSet, convert::TryInto, num::TryFromIntError}; use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_kafka_crd::{KafkaCluster, KafkaRole, APP_NAME}; +use stackable_kafka_crd::{ + KafkaCluster, KafkaRole, APP_NAME, CLIENT_PORT_NAME, SECURE_CLIENT_PORT_NAME, +}; use stackable_operator::{ builder::{ConfigMapBuilder, ObjectMetaBuilder}, k8s_openapi::api::core::v1::{ConfigMap, Endpoints, Service, ServicePort}, @@ -46,13 +48,18 @@ pub async fn build_discovery_configmaps( svc: &Service, ) -> Result, Error> { let name = owner.name(); + let port_name = if kafka.client_tls_secret_class().is_some() { + SECURE_CLIENT_PORT_NAME + } else { + CLIENT_PORT_NAME + }; Ok(vec![ - build_discovery_configmap(&name, owner, kafka, service_hosts(svc, "https")?)?, + build_discovery_configmap(&name, owner, kafka, service_hosts(svc, port_name)?)?, build_discovery_configmap( &format!("{}-nodeport", name), owner, kafka, - nodeport_hosts(client, svc, "https").await?, + nodeport_hosts(client, svc, port_name).await?, )?, ]) } diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index b3a8fe97..0689e8be 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -45,7 +45,7 @@ use std::{ }; use strum::{EnumDiscriminants, IntoStaticStr}; -use crate::command::{get_svc_container_args, kcat_container_cmd_args}; +use crate::command::{get_svc_container_cmd_args, kcat_container_cmd_args}; use crate::{ command, discovery::{self, build_discovery_configmaps}, @@ -489,7 +489,7 @@ fn build_broker_rolegroup_statefulset( .context(KafkaVersionParseFailureSnafu)?; let image = format!("docker.stackable.tech/stackable/kafka:{}", image_version); - let get_svc_args = get_svc_container_args(kafka); + let get_svc_args = get_svc_container_cmd_args(kafka); if let Some(tls) = kafka.client_tls_secret_class() { cb_prepare.add_volume_mount("tls-certificate", STACKABLE_TLS_CERTS_DIR); @@ -762,37 +762,14 @@ fn service_ports(kafka: &KafkaCluster) -> Vec { ..ServicePort::default() }]; - // If both client and internal TLS are set we do not need the HTTP port. - if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { + if kafka.client_tls_secret_class().is_some() { ports.push(ServicePort { name: Some(SECURE_CLIENT_PORT_NAME.to_string()), port: SECURE_CLIENT_PORT.into(), protocol: Some("TCP".to_string()), ..ServicePort::default() }); - } - // If only client TLS is required we need to set the HTTPS port and keep the HTTP port - // for internal communications. - // If only internal TLS is required we need to set the HTTPS port and keep the HTTP port - // for client communications. - else if kafka.client_tls_secret_class().is_some() - || kafka.internal_tls_secret_class().is_some() - { - ports.push(ServicePort { - name: Some(CLIENT_PORT_NAME.to_string()), - port: CLIENT_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }); - ports.push(ServicePort { - name: Some(SECURE_CLIENT_PORT_NAME.to_string()), - port: SECURE_CLIENT_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }); - } - // If no is TLS specified the HTTP port is sufficient - else { + } else { ports.push(ServicePort { name: Some(CLIENT_PORT_NAME.to_string()), port: CLIENT_PORT.into(), @@ -812,37 +789,14 @@ fn container_ports(kafka: &KafkaCluster) -> Vec { ..ContainerPort::default() }]; - // If both client and internal TLS are set we do not need the HTTP port. - if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { + if kafka.client_tls_secret_class().is_some() { ports.push(ContainerPort { name: Some(SECURE_CLIENT_PORT_NAME.to_string()), container_port: SECURE_CLIENT_PORT.into(), protocol: Some("TCP".to_string()), ..ContainerPort::default() }); - } - // If only client TLS is required we need to set the HTTPS port and keep the HTTP port - // for internal communications. - // If only internal TLS is required we need to set the HTTPS port and keep the HTTP port - // for client communications. - else if kafka.client_tls_secret_class().is_some() - || kafka.internal_tls_secret_class().is_some() - { - ports.push(ContainerPort { - name: Some(CLIENT_PORT_NAME.to_string()), - container_port: CLIENT_PORT.into(), - protocol: Some("TCP".to_string()), - ..ContainerPort::default() - }); - ports.push(ContainerPort { - name: Some(SECURE_CLIENT_PORT_NAME.to_string()), - container_port: SECURE_CLIENT_PORT.into(), - protocol: Some("TCP".to_string()), - ..ContainerPort::default() - }); - } - // If no is TLS specified the HTTP port is sufficient - else { + } else { ports.push(ContainerPort { name: Some(CLIENT_PORT_NAME.to_string()), container_port: CLIENT_PORT.into(), @@ -912,7 +866,7 @@ fn get_kafka_listener( // If only internal TLS is required we need to set the HTTPS port and keep the HTTP port // for client communications. Ok(KafkaListener { - listener: format!("internal://0.0.0.0:{SECURE_INTERNAL_PORT},SSL://0.0.0.0:{CLIENT_PORT}"), + listener: format!("internal://0.0.0.0:{SECURE_INTERNAL_PORT},PLAINTEXT://0.0.0.0:{CLIENT_PORT}"), advertised_listener: format!( "internal://{pod}:{SECURE_INTERNAL_PORT},PLAINTEXT://$NODE:$(cat {STACKABLE_TMP_DIR}/{CLIENT_PORT_NAME}_nodeport)", pod = pod_fqdn ), diff --git a/tests/templates/kuttl/smoke/01-install-kafka.yaml.j2 b/tests/templates/kuttl/smoke/01-install-kafka.yaml.j2 index bd107566..9e6ef15e 100644 --- a/tests/templates/kuttl/smoke/01-install-kafka.yaml.j2 +++ b/tests/templates/kuttl/smoke/01-install-kafka.yaml.j2 @@ -12,6 +12,13 @@ metadata: spec: version: {{ test_scenario['values']['kafka'] }} zookeeperConfigMapName: kafka-zk + config: +{% if test_scenario['values']['use-client-tls'] == 'true' %} + tls: + secretClass: tls +{% else %} + tls: null +{% endif %} brokers: config: resources: diff --git a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 index 3171748c..81d32d36 100644 --- a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 +++ b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 @@ -6,6 +6,30 @@ metadata: spec: clusterRef: name: test-zk +{% if test_scenario['values']['use-client-tls'] == 'true' %} +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: kafka-client-tls +spec: + provider: + tls: + clientCertSecretClass: kafka-client-tls +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: test-kafka-client-tls +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-kafka-client-ca + namespace: default + autoGenerate: true +{% endif %} {% if test_scenario['values']['use-internal-tls'] == 'true' %} --- apiVersion: secrets.stackable.tech/v1alpha1 @@ -34,7 +58,7 @@ spec: tls: secretClass: tls clientAuthentication: - authenticationClass: kafka-client-tls + authenticationClass: test-kafka-client-tls {% else %} tls: null {% endif %} @@ -46,27 +70,3 @@ spec: roleGroups: default: replicas: 3 -{% if test_scenario['values']['use-client-tls'] == 'true' %} ---- -apiVersion: authentication.stackable.tech/v1alpha1 -kind: AuthenticationClass -metadata: - name: kafka-client-tls -spec: - provider: - tls: - clientCertSecretClass: kafka-client-tls ---- -apiVersion: secrets.stackable.tech/v1alpha1 -kind: SecretClass -metadata: - name: kafka-client-tls -spec: - backend: - autoTls: - ca: - secret: - name: secret-provisioner-tls-kafka-client-ca - namespace: default - autoGenerate: true -{% endif %} diff --git a/tests/templates/kuttl/upgrade/01-install-kafka.yaml.j2 b/tests/templates/kuttl/upgrade/01-install-kafka.yaml.j2 index 8ce54655..362ae0a7 100644 --- a/tests/templates/kuttl/upgrade/01-install-kafka.yaml.j2 +++ b/tests/templates/kuttl/upgrade/01-install-kafka.yaml.j2 @@ -10,6 +10,13 @@ metadata: spec: version: {{ test_scenario['values']['upgrade_old'] }} zookeeperConfigMapName: kafka-zk + config: +{% if test_scenario['values']['use-client-tls'] == 'true' %} + tls: + secretClass: tls +{% else %} + tls: null +{% endif %} brokers: roleGroups: default: diff --git a/tests/templates/kuttl/upgrade/02-write-data.yaml b/tests/templates/kuttl/upgrade/02-write-data.yaml deleted file mode 100644 index 15bf418c..00000000 --- a/tests/templates/kuttl/upgrade/02-write-data.yaml +++ /dev/null @@ -1,27 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -timeout: 300 ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: write-data -spec: - template: - spec: - containers: - - name: write-data - image: edenhill/kcat:1.7.1 - command: [sh, -euo, pipefail, -c] - args: - - | - echo "message written before upgrade" > message - kcat -b $KAFKA -t upgrade-test-data -P message - env: - - name: KAFKA - valueFrom: - configMapKeyRef: - name: simple-kafka - key: KAFKA - restartPolicy: Never diff --git a/tests/templates/kuttl/upgrade/02-write-data.yaml.j2 b/tests/templates/kuttl/upgrade/02-write-data.yaml.j2 new file mode 100644 index 00000000..730f0117 --- /dev/null +++ b/tests/templates/kuttl/upgrade/02-write-data.yaml.j2 @@ -0,0 +1,53 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: write-data +spec: + template: + spec: + containers: + - name: write-data + image: edenhill/kcat:1.7.1 + command: [sh, -euo, pipefail, -c] + args: + - | + echo "message written before upgrade" > message +{% if test_scenario['values']['use-client-tls'] == 'true' %} + kcat -b $KAFKA -X security.protocol=SSL -X ssl.key.location=/stackable/certificates/tls.key -X ssl.certificate.location=/stackable/certificates/tls.crt -X ssl.ca.location=/stackable/certificates/ca.crt -t upgrade-test-data -P message +{% else %} + kcat -b $KAFKA -t upgrade-test-data -P message +{% endif %} + env: + - name: KAFKA + valueFrom: + configMapKeyRef: + name: simple-kafka + key: KAFKA +{% if test_scenario['values']['use-client-tls'] == 'true' %} + volumeMounts: + - mountPath: /stackable/certificates + name: tls + volumes: + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: pod,node + creationTimestamp: null + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + volumeMode: Filesystem + name: tls +{% endif %} + restartPolicy: Never diff --git a/tests/templates/kuttl/upgrade/03-upgrade-kafka.yaml.j2 b/tests/templates/kuttl/upgrade/03-upgrade-kafka.yaml.j2 index 37fb742e..cb662438 100644 --- a/tests/templates/kuttl/upgrade/03-upgrade-kafka.yaml.j2 +++ b/tests/templates/kuttl/upgrade/03-upgrade-kafka.yaml.j2 @@ -10,6 +10,13 @@ metadata: spec: version: {{ test_scenario['values']['upgrade_new'] }} zookeeperConfigMapName: kafka-zk + config: +{% if test_scenario['values']['use-client-tls'] == 'true' %} + tls: + secretClass: tls +{% else %} + tls: null +{% endif %} brokers: roleGroups: default: diff --git a/tests/templates/kuttl/upgrade/04-read-data.yaml b/tests/templates/kuttl/upgrade/04-read-data.yaml deleted file mode 100644 index 599d83c6..00000000 --- a/tests/templates/kuttl/upgrade/04-read-data.yaml +++ /dev/null @@ -1,35 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -timeout: 300 ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: read-data -spec: - template: - spec: - containers: - - name: read-data - image: edenhill/kcat:1.7.1 - command: [sh, -euo, pipefail, -c] - args: - - | - echo "message written after upgrade" > message - kcat -b $KAFKA -t upgrade-test-data -P message - - echo "message written before upgrade" > expected-messages - echo >> expected-messages - cat message >> expected-messages - echo >> expected-messages - kcat -b $KAFKA -t upgrade-test-data -C -e > read-messages - diff read-messages expected-messages - cmp read-messages expected-messages - env: - - name: KAFKA - valueFrom: - configMapKeyRef: - name: simple-kafka - key: KAFKA - restartPolicy: Never diff --git a/tests/templates/kuttl/upgrade/04-read-data.yaml.j2 b/tests/templates/kuttl/upgrade/04-read-data.yaml.j2 new file mode 100644 index 00000000..31d967ac --- /dev/null +++ b/tests/templates/kuttl/upgrade/04-read-data.yaml.j2 @@ -0,0 +1,70 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: read-data +spec: + template: + spec: + containers: + - name: read-data + image: edenhill/kcat:1.7.1 + command: [sh, -euo, pipefail, -c] + args: + - | +{% if test_scenario['values']['use-client-tls'] == 'true' %} + echo "message written after upgrade" > message + kcat -b $KAFKA -X security.protocol=SSL -X ssl.key.location=/stackable/certificates/tls.key -X ssl.certificate.location=/stackable/certificates/tls.crt -X ssl.ca.location=/stackable/certificates/ca.crt -t upgrade-test-data -P message + + echo "message written before upgrade" > expected-messages + echo >> expected-messages + cat message >> expected-messages + echo >> expected-messages + kcat -b $KAFKA -X security.protocol=SSL -X ssl.key.location=/stackable/certificates/tls.key -X ssl.certificate.location=/stackable/certificates/tls.crt -X ssl.ca.location=/stackable/certificates/ca.crt -t upgrade-test-data -C -e > read-messages + diff read-messages expected-messages + cmp read-messages expected-messages +{% else %} + echo "message written after upgrade" > message + kcat -b $KAFKA -t upgrade-test-data -P message + + echo "message written before upgrade" > expected-messages + echo >> expected-messages + cat message >> expected-messages + echo >> expected-messages + kcat -b $KAFKA -t upgrade-test-data -C -e > read-messages + diff read-messages expected-messages + cmp read-messages expected-messages +{% endif %} + env: + - name: KAFKA + valueFrom: + configMapKeyRef: + name: simple-kafka + key: KAFKA +{% if test_scenario['values']['use-client-tls'] == 'true' %} + volumeMounts: + - mountPath: /stackable/certificates + name: tls + volumes: + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: pod,node + creationTimestamp: null + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + volumeMode: Filesystem + name: tls +{% endif %} + restartPolicy: Never From f1c96dc72c9618e69de4a5bcf34787ade9aaec66 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 2 Aug 2022 11:33:18 +0200 Subject: [PATCH 14/37] adding client authentication, tls tests working --- examples/tls/simple-kafka-cluster-tls.yaml | 24 +++ rust/crd/src/lib.rs | 1 + rust/operator/src/command.rs | 10 +- rust/operator/src/kafka_controller.rs | 155 +++++++++++++++--- .../kuttl/upgrade/03-upgrade-kafka.yaml.j2 | 24 +-- 5 files changed, 166 insertions(+), 48 deletions(-) diff --git a/examples/tls/simple-kafka-cluster-tls.yaml b/examples/tls/simple-kafka-cluster-tls.yaml index f07aac00..f076fccf 100644 --- a/examples/tls/simple-kafka-cluster-tls.yaml +++ b/examples/tls/simple-kafka-cluster-tls.yaml @@ -35,6 +35,28 @@ spec: namespace: default autoGenerate: true --- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: kafka-client-tls +spec: + provider: + tls: + clientCertSecretClass: kafka-client-auth-secret +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: kafka-client-auth-secret +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-kafka-client-ca + namespace: default + autoGenerate: true +--- apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaCluster metadata: @@ -45,6 +67,8 @@ spec: config: tls: secretClass: tls + clientAuthentication: + authenticationClass: kafka-client-tls internalTls: secretClass: kafka-internal-tls brokers: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 95ef10d5..d810d12f 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -70,6 +70,7 @@ pub const STACKABLE_DATA_DIR: &str = "/stackable/data"; pub const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; pub const STACKABLE_TLS_CERTS_DIR: &str = "/stackable/certificates"; pub const STACKABLE_TLS_CERTS_INTERNAL_DIR: &str = "/stackable/certificates_internal"; +pub const STACKABLE_TLS_CERTS_AUTHENTICATION_DIR: &str = "/stackable/certificates_authentication"; pub const SYSTEM_TRUST_STORE_DIR: &str = "/etc/pki/java/cacerts"; const JVM_HEAP_FACTOR: f32 = 0.8; diff --git a/rust/operator/src/command.rs b/rust/operator/src/command.rs index f432a41a..069a1dcf 100644 --- a/rust/operator/src/command.rs +++ b/rust/operator/src/command.rs @@ -1,7 +1,8 @@ use stackable_kafka_crd::{ KafkaCluster, CLIENT_PORT, CLIENT_PORT_NAME, SECURE_CLIENT_PORT, SECURE_CLIENT_PORT_NAME, - SSL_STORE_PASSWORD, STACKABLE_DATA_DIR, STACKABLE_TLS_CERTS_DIR, - STACKABLE_TLS_CERTS_INTERNAL_DIR, STACKABLE_TMP_DIR, SYSTEM_TRUST_STORE_DIR, + SSL_STORE_PASSWORD, STACKABLE_DATA_DIR, STACKABLE_TLS_CERTS_AUTHENTICATION_DIR, + STACKABLE_TLS_CERTS_DIR, STACKABLE_TLS_CERTS_INTERNAL_DIR, STACKABLE_TMP_DIR, + SYSTEM_TRUST_STORE_DIR, }; pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { @@ -15,6 +16,11 @@ pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { "stackable-tls-ca-cert", )); args.extend(chown_and_chmod(STACKABLE_TLS_CERTS_DIR)); + + if kafka.client_authentication_class().is_some() { + args.push(format!("echo [{STACKABLE_TLS_CERTS_DIR}] Importing client authentication cert to truststore")); + args.push(format!("keytool -importcert -file {STACKABLE_TLS_CERTS_AUTHENTICATION_DIR}/ca.crt -keystore {STACKABLE_TLS_CERTS_DIR}/truststore.p12 -storetype pkcs12 -noprompt -alias stackable-tls-client-ca-cert -storepass {SSL_STORE_PASSWORD}")); + } } if kafka.internal_tls_secret_class().is_some() { diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index 0689e8be..bcd254d3 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -5,16 +5,19 @@ use stackable_kafka_crd::{ KafkaCluster, KafkaRole, TlsSecretClass, APP_NAME, CLIENT_PORT, CLIENT_PORT_NAME, INTERNAL_PORT, KAFKA_HEAP_OPTS, LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, SECURE_CLIENT_PORT, SECURE_CLIENT_PORT_NAME, SECURE_INTERNAL_PORT, SERVER_PROPERTIES_FILE, - STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_TLS_CERTS_DIR, - STACKABLE_TLS_CERTS_INTERNAL_DIR, STACKABLE_TMP_DIR, + STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_TLS_CERTS_AUTHENTICATION_DIR, + STACKABLE_TLS_CERTS_DIR, STACKABLE_TLS_CERTS_INTERNAL_DIR, STACKABLE_TMP_DIR, }; -use stackable_operator::builder::{ - SecretOperatorVolumeSourceBuilder, SecurityContextBuilder, VolumeBuilder, -}; -use stackable_operator::kube::ResourceExt; +use stackable_operator::commons::tls::TlsAuthenticationProvider; use stackable_operator::{ - builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder}, - commons::opa::OpaApiVersion, + builder::{ + ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder, + SecretOperatorVolumeSourceBuilder, SecurityContextBuilder, VolumeBuilder, + }, + commons::{ + authentication::{AuthenticationClass, AuthenticationClassProvider}, + opa::OpaApiVersion, + }, k8s_openapi::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, @@ -28,7 +31,11 @@ use stackable_operator::{ apimachinery::pkg::apis::meta::v1::LabelSelector, Resource, }, - kube::runtime::{controller::Action, reflector::ObjectRef}, + kube::{ + api::DynamicObject, + runtime::{controller::Action, reflector::ObjectRef}, + ResourceExt, + }, labels::{role_group_selector_labels, role_selector_labels}, logging::controller::ReconcilerError, product_config::{ @@ -149,6 +156,21 @@ pub enum Error { }, #[snafu(display("failed to parse Kafka version/image"))] KafkaVersionParseFailure { source: stackable_kafka_crd::Error }, + #[snafu(display("failed to retrieve {}", authentication_class))] + AuthenticationClassRetrieval { + source: stackable_operator::error::Error, + authentication_class: ObjectRef, + }, + #[snafu(display( + "failed to use authentication mechanism {} - supported methods: {:?}", + method, + supported + ))] + AuthenticationMethodNotSupported { + authentication_class: ObjectRef, + supported: Vec, + method: String, + }, } type Result = std::result::Result; @@ -156,6 +178,42 @@ impl ReconcilerError for Error { fn category(&self) -> &'static str { ErrorDiscriminants::from(self).into() } + + fn secondary_object(&self) -> Option> { + match self { + Error::ObjectHasNoName => None, + Error::ObjectHasNoNamespace => None, + Error::ObjectHasNoVersion => None, + Error::NoBrokerRole => None, + Error::GlobalServiceNameNotFound => None, + Error::ApplyRoleService { .. } => None, + Error::ApplyRoleServiceAccount { .. } => None, + Error::ApplyRoleRoleBinding { .. } => None, + Error::ApplyRoleGroupService { .. } => None, + Error::BuildRoleGroupConfig { .. } => None, + Error::ApplyRoleGroupConfig { .. } => None, + Error::ApplyRoleGroupStatefulSet { .. } => None, + Error::GenerateProductConfig { .. } => None, + Error::InvalidProductConfig { .. } => None, + Error::SerializeZooCfg { .. } => None, + Error::ObjectMissingMetadataForOwnerRef { .. } => None, + Error::BuildDiscoveryConfig { .. } => None, + Error::ApplyDiscoveryConfig { .. } => None, + Error::RoleGroupNotFound { .. } => None, + Error::InvalidServiceAccount { .. } => None, + Error::InvalidOpaConfig { .. } => None, + Error::InvalidJavaHeapConfig { .. } => None, + Error::KafkaVersionParseFailure { .. } => None, + Error::AuthenticationClassRetrieval { + authentication_class, + .. + } => Some(authentication_class.clone().erase()), + Error::AuthenticationMethodNotSupported { + authentication_class, + .. + } => Some(authentication_class.clone().erase()), + } + } } pub async fn reconcile_kafka(kafka: Arc, ctx: Arc) -> Result { @@ -191,6 +249,38 @@ pub async fn reconcile_kafka(kafka: Arc, ctx: Arc) -> Result< .map(Cow::Borrowed) .unwrap_or_default(); + let client_authentication_class = if let Some(auth_class) = kafka.client_authentication_class() + { + Some( + client + .get::(auth_class, None) // AuthenticationClass has ClusterScope + .await + .context(AuthenticationClassRetrievalSnafu { + authentication_class: ObjectRef::::new(auth_class), + })?, + ) + } else { + None + }; + + // Assemble the OPA connection string from the discovery and the given path if provided + // Will be passed as --override parameter in the cli in the state ful set + let opa_connect = if let Some(opa_spec) = &kafka.spec.opa { + Some( + opa_spec + .full_document_url_from_config_map( + client, + &*kafka, + Some("allow"), + OpaApiVersion::V1, + ) + .await + .context(InvalidOpaConfigSnafu)?, + ) + } else { + None + }; + let broker_role_service = build_broker_role_service(&kafka)?; let (broker_role_serviceaccount, broker_role_rolebinding) = build_broker_role_serviceaccount(&kafka, &ctx.controller_config)?; @@ -220,24 +310,6 @@ pub async fn reconcile_kafka(kafka: Arc, ctx: Arc) -> Result< .await .context(ApplyRoleRoleBindingSnafu)?; - // Assemble the OPA connection string from the discovery and the given path if provided - // Will be passed as --override parameter in the cli in the state ful set - let opa_connect = if let Some(opa_spec) = &kafka.spec.opa { - Some( - opa_spec - .full_document_url_from_config_map( - client, - &*kafka, - Some("allow"), - OpaApiVersion::V1, - ) - .await - .context(InvalidOpaConfigSnafu)?, - ) - } else { - None - }; - for (rolegroup_name, rolegroup_config) in role_broker_config.iter() { let rolegroup = kafka.broker_rolegroup_ref(rolegroup_name); @@ -249,6 +321,7 @@ pub async fn reconcile_kafka(kafka: Arc, ctx: Arc) -> Result< rolegroup_config, &broker_role_serviceaccount_ref, opa_connect.as_deref(), + client_authentication_class.as_ref(), )?; client .apply_patch(FIELD_MANAGER_SCOPE, &rg_service, &rg_service) @@ -471,6 +544,7 @@ fn build_broker_rolegroup_statefulset( broker_config: &HashMap>, serviceaccount: &ObjectRef, opa_connect_string: Option<&str>, + client_authentication_class: Option<&AuthenticationClass>, ) -> Result { let mut cb_kafka = ContainerBuilder::new(APP_NAME); let mut cb_prepare = ContainerBuilder::new("prepare"); @@ -496,6 +570,33 @@ fn build_broker_rolegroup_statefulset( cb_kafka.add_volume_mount("tls-certificate", STACKABLE_TLS_CERTS_DIR); cb_kcat_prober.add_volume_mount("tls-certificate", STACKABLE_TLS_CERTS_DIR); pod_builder.add_volume(create_tls_volume("tls-certificate", Some(tls))); + + // add client authentication if required + if let Some(auth_class) = client_authentication_class { + match &auth_class.spec.provider { + AuthenticationClassProvider::Tls(TlsAuthenticationProvider { + client_cert_secret_class: Some(secret_class), + }) => { + cb_prepare.add_volume_mount( + "client-tls-authentication-certificate", + STACKABLE_TLS_CERTS_AUTHENTICATION_DIR, + ); + pod_builder.add_volume(create_tls_volume( + "client-tls-authentication-certificate", + Some(&TlsSecretClass { + secret_class: secret_class.clone(), + }), + )); + } + _ => { + return Err(Error::AuthenticationMethodNotSupported { + authentication_class: ObjectRef::from_obj(auth_class), + supported: vec!["tls".to_string()], + method: auth_class.spec.provider.to_string(), + }) + } + } + } } if let Some(tls_internal) = kafka.internal_tls_secret_class() { diff --git a/tests/templates/kuttl/upgrade/03-upgrade-kafka.yaml.j2 b/tests/templates/kuttl/upgrade/03-upgrade-kafka.yaml.j2 index cb662438..8e81a2fd 100644 --- a/tests/templates/kuttl/upgrade/03-upgrade-kafka.yaml.j2 +++ b/tests/templates/kuttl/upgrade/03-upgrade-kafka.yaml.j2 @@ -2,22 +2,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 300 ---- -apiVersion: kafka.stackable.tech/v1alpha1 -kind: KafkaCluster -metadata: - name: simple-kafka -spec: - version: {{ test_scenario['values']['upgrade_new'] }} - zookeeperConfigMapName: kafka-zk - config: -{% if test_scenario['values']['use-client-tls'] == 'true' %} - tls: - secretClass: tls -{% else %} - tls: null -{% endif %} - brokers: - roleGroups: - default: - replicas: 1 +commands: + - script: >- + kubectl --namespace $NAMESPACE + patch kafkaclusters.kafka.stackable.tech simple-kafka + --type=merge --patch '{ "spec": { "version": "{{ test_scenario['values']['upgrade_new'] }}" }}' From e8767a9ece32008a9da57f66c1842b07b850eb21 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 2 Aug 2022 12:42:32 +0200 Subject: [PATCH 15/37] adapted docs --- docs/modules/ROOT/pages/usage.adoc | 230 +++++++++++++++++++++-------- 1 file changed, 167 insertions(+), 63 deletions(-) diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index 2c1f12f3..7d84008b 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -1,76 +1,81 @@ = Usage -After installation, the CRD for this operator must be created: +If you are not installing the operator using Helm then after installation the CRD for this operator must be created: kubectl apply -f /etc/stackable/kafka-operator/crd/kafkacluster.crd.yaml To create an Apache Kafka (v3.2.0) cluster named `simple-kafka` assuming that you already have a Zookeeper cluster named `simple-zk`: - cat < + internalTls: + secretClass: kafka-internal-tls # <2> + brokers: + roleGroups: + default: + replicas: 3 +---- +<1> The `tls.secretClass` refers to the client-to-server encryption. Defaults to the `tls` secret. It can be deactivated by setting `config.tls` to `null`. +<2> The `internalTls.secretClass` refers to the broker-to-broker internal encryption. This must be explicitly set. + +The `tls` secret is deployed from the xref:secret-operator::index.adoc[Secret Operator] and looks like this: + +[source,yaml] +---- +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: tls +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-ca + namespace: default + autoGenerate: true +---- + +You can create your own secrets and reference them e.g. in the `tls.secretClass` or `internalTls.secretClass` to use different certificates. + +== Authentication + +The internal or broker-to-broker communication is authenticated via TLS. In order to enforce TLS authentication for client-to-server communication, you can set an `AuthenticationClass` reference in the custom resource provided by the xref:commons-operator::index.adoc[Commons Operator]. + +[source,yaml] +---- +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: kafka-client-tls # <2> +spec: + provider: + tls: + clientCertSecretClass: kafka-client-auth-secret # <3> +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: kafka-client-auth-secret # <4> +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-kafka-client-ca + namespace: default + autoGenerate: true +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: simple-kafka +spec: + version: 3.2.0-stackable0.1.0 + zookeeperConfigMapName: simple-kafka-znode + config: + tls: + secretClass: tls + clientAuthentication: + authenticationClass: kafka-client-tls # <1> + brokers: + roleGroups: + default: + replicas: 3 +---- +<1> The `config.clientAuthentication.authenticationClass` can be set to use TLS for authentication. This is optional. +<2> The referenced `AuthenticationClass` that references a `SecretClass` to provide certificates. +<3> The reference to a `SecretClass`. +<4> The `SecretClass` that is referenced by the `AuthenticationClass` in order to provide certificates. + == Configuration & Environment Overrides The cluster definition also supports overriding configuration properties and environment variables, either per role or per role group, where the more specific override (role group) has precedence over the less specific one (role). From 46be7d133380c44f2f1f5160d55fe8f08ec8e314 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 2 Aug 2022 16:20:57 +0200 Subject: [PATCH 16/37] fixed auth class --- tests/templates/kuttl/tls/20-install-kafka.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 index 81d32d36..b3241c05 100644 --- a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 +++ b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 @@ -11,11 +11,11 @@ spec: apiVersion: authentication.stackable.tech/v1alpha1 kind: AuthenticationClass metadata: - name: kafka-client-tls + name: test-kafka-client-tls spec: provider: tls: - clientCertSecretClass: kafka-client-tls + clientCertSecretClass: test-kafka-client-tls --- apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass From 5362dbcc685cd272c09cabda607cf031ae64cac4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Aug 2022 11:43:04 +0200 Subject: [PATCH 17/37] extracted listener creation to extra module. introduced one listener for each client, client authentication and internal communication --- rust/crd/src/lib.rs | 114 +++++--- rust/crd/src/listener.rs | 365 ++++++++++++++++++++++++++ rust/operator/src/command.rs | 66 ++--- rust/operator/src/kafka_controller.rs | 163 ++++-------- 4 files changed, 539 insertions(+), 169 deletions(-) create mode 100644 rust/crd/src/listener.rs diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index d810d12f..a25fb9db 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -1,6 +1,7 @@ +pub mod listener; + use serde::{Deserialize, Serialize}; use snafu::{OptionExt, Snafu}; -use stackable_operator::error::OperatorResult; use stackable_operator::memory::to_java_heap; use stackable_operator::{ commons::{ @@ -8,6 +9,7 @@ use stackable_operator::{ resources::{CpuLimits, MemoryLimits, NoRuntimeLimits, PvcConfig, Resources}, }, config::merge::Merge, + error::OperatorResult, k8s_openapi::{ api::core::v1::{PersistentVolumeClaim, ResourceRequirements}, apimachinery::pkg::api::resource::Quantity, @@ -40,7 +42,7 @@ pub const LOG_DIRS_VOLUME_NAME: &str = "log-dirs"; pub const LISTENER_SECURITY_PROTOCOL_MAP: &str = "listener.security.protocol.map"; pub const LISTENER: &str = "listeners"; pub const ADVERTISED_LISTENER: &str = "advertised.listeners"; -// - TLS +// - TLS global pub const TLS_DEFAULT_SECRET_CLASS: &str = "tls"; pub const SSL_KEYSTORE_LOCATION: &str = "ssl.keystore.location"; pub const SSL_KEYSTORE_PASSWORD: &str = "ssl.keystore.password"; @@ -49,8 +51,25 @@ pub const SSL_TRUSTSTORE_LOCATION: &str = "ssl.truststore.location"; pub const SSL_TRUSTSTORE_PASSWORD: &str = "ssl.truststore.password"; pub const SSL_TRUSTSTORE_TYPE: &str = "ssl.truststore.type"; pub const SSL_STORE_PASSWORD: &str = "changeit"; -pub const SSL_CLIENT_AUTH: &str = "ssl.client.auth"; -pub const SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: &str = "ssl.endpoint.identification.algorithm"; +// - TLS client +pub const CLIENT_SSL_KEYSTORE_LOCATION: &str = "listener.name.client.ssl.keystore.location"; +pub const CLIENT_SSL_KEYSTORE_PASSWORD: &str = "listener.name.client.ssl.keystore.password"; +pub const CLIENT_SSL_KEYSTORE_TYPE: &str = "listener.name.client.ssl.keystore.type"; +pub const CLIENT_SSL_TRUSTSTORE_LOCATION: &str = "listener.name.client.ssl.truststore.location"; +pub const CLIENT_SSL_TRUSTSTORE_PASSWORD: &str = "listener.name.client.ssl.truststore.password"; +pub const CLIENT_SSL_TRUSTSTORE_TYPE: &str = "listener.name.client.ssl.truststore.type"; +// - TLS client authentication +pub const CLIENT_AUTH_SSL_KEYSTORE_LOCATION: &str = + "listener.name.client_auth.ssl.keystore.location"; +pub const CLIENT_AUTH_SSL_KEYSTORE_PASSWORD: &str = + "listener.name.client_auth.ssl.keystore.password"; +pub const CLIENT_AUTH_SSL_KEYSTORE_TYPE: &str = "listener.name.client_auth.ssl.keystore.type"; +pub const CLIENT_AUTH_SSL_TRUSTSTORE_LOCATION: &str = + "listener.name.client_auth.ssl.truststore.location"; +pub const CLIENT_AUTH_SSL_TRUSTSTORE_PASSWORD: &str = + "listener.name.client_auth.ssl.truststore.password"; +pub const CLIENT_AUTH_SSL_TRUSTSTORE_TYPE: &str = "listener.name.client_auth.ssl.truststore.type"; +pub const CLIENT_AUTH_SSL_CLIENT_AUTH: &str = "listener.name.client_auth.ssl.client.auth"; // - TLS internal pub const SECURITY_INTER_BROKER_PROTOCOL: &str = "security.inter.broker.protocol"; pub const INTER_BROKER_LISTENER_NAME: &str = "inter.broker.listener.name"; @@ -60,7 +79,6 @@ pub const INTER_SSL_KEYSTORE_TYPE: &str = "listener.name.internal.ssl.keystore.t pub const INTER_SSL_TRUSTSTORE_LOCATION: &str = "listener.name.internal.ssl.truststore.location"; pub const INTER_SSL_TRUSTSTORE_PASSWORD: &str = "listener.name.internal.ssl.truststore.password"; pub const INTER_SSL_TRUSTSTORE_TYPE: &str = "listener.name.internal.ssl.truststore.type"; -pub const INTER_SSL_STORE_PASSWORD: &str = "changeit"; pub const INTER_SSL_CLIENT_AUTH: &str = "listener.name.internal.ssl.client.auth"; pub const INTER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: &str = "listener.name.internal.ssl.endpoint.identification.algorithm"; @@ -68,9 +86,9 @@ pub const INTER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: &str = pub const STACKABLE_TMP_DIR: &str = "/stackable/tmp"; pub const STACKABLE_DATA_DIR: &str = "/stackable/data"; pub const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; -pub const STACKABLE_TLS_CERTS_DIR: &str = "/stackable/certificates"; -pub const STACKABLE_TLS_CERTS_INTERNAL_DIR: &str = "/stackable/certificates_internal"; -pub const STACKABLE_TLS_CERTS_AUTHENTICATION_DIR: &str = "/stackable/certificates_authentication"; +pub const STACKABLE_TLS_CLIENT_DIR: &str = "/stackable/tls_client"; +pub const STACKABLE_TLS_CLIENT_AUTH_DIR: &str = "/stackable/tls_client_auth"; +pub const STACKABLE_TLS_INTERNAL_DIR: &str = "/stackable/tls_internal"; pub const SYSTEM_TRUST_STORE_DIR: &str = "/etc/pki/java/cacerts"; const JVM_HEAP_FACTOR: f32 = 0.8; @@ -414,46 +432,79 @@ impl Configuration for KafkaConfig { ); } - // Client TLS - if resource.client_tls_secret_class().is_some() { + if resource.client_authentication_class().is_some() { config.insert( - SSL_KEYSTORE_LOCATION.to_string(), - Some(format!("{}/keystore.p12", STACKABLE_TLS_CERTS_DIR)), + CLIENT_AUTH_SSL_KEYSTORE_LOCATION.to_string(), + Some(format!("{}/keystore.p12", STACKABLE_TLS_CLIENT_AUTH_DIR)), ); config.insert( - SSL_KEYSTORE_PASSWORD.to_string(), + CLIENT_AUTH_SSL_KEYSTORE_PASSWORD.to_string(), Some(SSL_STORE_PASSWORD.to_string()), ); - config.insert(SSL_KEYSTORE_TYPE.to_string(), Some("PKCS12".to_string())); config.insert( - SSL_TRUSTSTORE_LOCATION.to_string(), - Some(format!("{}/truststore.p12", STACKABLE_TLS_CERTS_DIR)), + CLIENT_AUTH_SSL_KEYSTORE_TYPE.to_string(), + Some("PKCS12".to_string()), + ); + config.insert( + CLIENT_AUTH_SSL_TRUSTSTORE_LOCATION.to_string(), + Some(format!("{}/truststore.p12", STACKABLE_TLS_CLIENT_AUTH_DIR)), ); config.insert( - SSL_TRUSTSTORE_PASSWORD.to_string(), + CLIENT_AUTH_SSL_TRUSTSTORE_PASSWORD.to_string(), Some(SSL_STORE_PASSWORD.to_string()), ); - config.insert(SSL_TRUSTSTORE_TYPE.to_string(), Some("PKCS12".to_string())); + config.insert( + CLIENT_AUTH_SSL_TRUSTSTORE_TYPE.to_string(), + Some("PKCS12".to_string()), + ); // Authentication - if resource.client_authentication_class().is_some() { - config.insert(SSL_CLIENT_AUTH.to_string(), Some("required".to_string())); - config.insert( - SSL_ENDPOINT_IDENTIFICATION_ALGORITHM.to_string(), - Some("HTTPS".to_string()), - ); - } + config.insert( + CLIENT_AUTH_SSL_CLIENT_AUTH.to_string(), + Some("required".to_string()), + ); + // config.insert( + // SSL_ENDPOINT_IDENTIFICATION_ALGORITHM.to_string(), + // Some("HTTPS".to_string()), + // ); + } + // Client TLS + else if resource.client_tls_secret_class().is_some() { + config.insert( + CLIENT_SSL_KEYSTORE_LOCATION.to_string(), + Some(format!("{}/keystore.p12", STACKABLE_TLS_CLIENT_DIR)), + ); + config.insert( + CLIENT_SSL_KEYSTORE_PASSWORD.to_string(), + Some(SSL_STORE_PASSWORD.to_string()), + ); + config.insert( + CLIENT_SSL_KEYSTORE_TYPE.to_string(), + Some("PKCS12".to_string()), + ); + config.insert( + CLIENT_SSL_TRUSTSTORE_LOCATION.to_string(), + Some(format!("{}/truststore.p12", STACKABLE_TLS_CLIENT_DIR)), + ); + config.insert( + CLIENT_SSL_TRUSTSTORE_PASSWORD.to_string(), + Some(SSL_STORE_PASSWORD.to_string()), + ); + config.insert( + CLIENT_SSL_TRUSTSTORE_TYPE.to_string(), + Some("PKCS12".to_string()), + ); } // Internal TLS if resource.internal_tls_secret_class().is_some() { config.insert( INTER_SSL_KEYSTORE_LOCATION.to_string(), - Some(format!("{}/keystore.p12", STACKABLE_TLS_CERTS_INTERNAL_DIR)), + Some(format!("{}/keystore.p12", STACKABLE_TLS_INTERNAL_DIR)), ); config.insert( INTER_SSL_KEYSTORE_PASSWORD.to_string(), - Some(INTER_SSL_STORE_PASSWORD.to_string()), + Some(SSL_STORE_PASSWORD.to_string()), ); config.insert( INTER_SSL_KEYSTORE_TYPE.to_string(), @@ -461,14 +512,11 @@ impl Configuration for KafkaConfig { ); config.insert( INTER_SSL_TRUSTSTORE_LOCATION.to_string(), - Some(format!( - "{}/truststore.p12", - STACKABLE_TLS_CERTS_INTERNAL_DIR - )), + Some(format!("{}/truststore.p12", STACKABLE_TLS_INTERNAL_DIR)), ); config.insert( INTER_SSL_TRUSTSTORE_PASSWORD.to_string(), - Some(INTER_SSL_STORE_PASSWORD.to_string()), + Some(SSL_STORE_PASSWORD.to_string()), ); config.insert( INTER_SSL_TRUSTSTORE_TYPE.to_string(), @@ -487,7 +535,7 @@ impl Configuration for KafkaConfig { // common config.insert( INTER_BROKER_LISTENER_NAME.to_string(), - Some("internal".to_string()), + Some(listener::KafkaListenerName::Internal.to_string()), ); } diff --git a/rust/crd/src/listener.rs b/rust/crd/src/listener.rs new file mode 100644 index 00000000..d7d6c15e --- /dev/null +++ b/rust/crd/src/listener.rs @@ -0,0 +1,365 @@ +use crate::{ + KafkaCluster, CLIENT_PORT, CLIENT_PORT_NAME, INTERNAL_PORT, SECURE_CLIENT_PORT, + SECURE_CLIENT_PORT_NAME, SECURE_INTERNAL_PORT, STACKABLE_TMP_DIR, +}; +use snafu::{OptionExt, Snafu}; +use stackable_operator::kube::ResourceExt; +use std::collections::BTreeMap; +use std::fmt::{Display, Formatter}; +use strum::{EnumDiscriminants, EnumString}; + +const LISTENER_LOCAL_ADDRESS: &str = "0.0.0.0"; +const LISTENER_NODE_ADDRESS: &str = "$NODE"; + +#[derive(Snafu, Debug, EnumDiscriminants)] +pub enum KafkaListenerError { + #[snafu(display("object has no namespace"))] + ObjectHasNoNamespace, +} + +#[derive(strum::Display, Debug, EnumString)] +pub enum KafkaListenerProtocol { + /// Unencrypted and authenticated HTTP connections + #[strum(serialize = "PLAINTEXT")] + Plaintext, + /// Encrypted and server-authenticated HTTPS connections + #[strum(serialize = "SSL")] + Ssl, +} + +#[derive(strum::Display, Debug, EnumString, Ord, Eq, PartialEq, PartialOrd)] +pub enum KafkaListenerName { + #[strum(serialize = "CLIENT")] + Client, + #[strum(serialize = "CLIENT_AUTH")] + ClientAuth, + #[strum(serialize = "INTERNAL")] + Internal, +} + +#[derive(Debug)] +pub struct KafkaListenerConfig { + listeners: Vec, + advertised_listeners: Vec, + listener_security_protocol_map: BTreeMap, +} + +impl KafkaListenerConfig { + /// Returns the `listeners` for the Kafka `server.properties` config. + pub fn listeners(&self) -> String { + self.listeners + .iter() + .map(|listener| listener.to_string()) + .collect::>() + .join(",") + } + + /// Returns the `advertised.listeners` for the Kafka `server.properties` config. + /// May contain ENV variables and therefore should be used as cli argument + /// like --override \"advertised.listeners=xxx\". + pub fn advertised_listeners(&self) -> String { + self.advertised_listeners + .iter() + .map(|listener| listener.to_string()) + .collect::>() + .join(",") + } + + /// Returns the `listener.security.protocol.map` for the Kafka `server.properties` config. + pub fn listener_security_protocol_map(&self) -> String { + self.listener_security_protocol_map + .iter() + .map(|(name, protocol)| format!("{}:{}", name, protocol)) + .collect::>() + .join(",") + } +} + +#[derive(Debug)] +struct KafkaListener { + name: KafkaListenerName, + host: String, + port: String, +} + +impl Display for KafkaListener { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}://{}:{}", self.name, self.host, self.port) + } +} + +pub fn get_kafka_listener_config( + kafka: &KafkaCluster, + object_name: &str, +) -> Result { + let pod_fqdn = pod_fqdn(kafka, object_name)?; + let mut listeners = vec![]; + let mut advertised_listeners = vec![]; + let mut listener_security_protocol_map = BTreeMap::new(); + + if kafka.client_authentication_class().is_some() { + // 1) If client authentication required, we expose only CLIENT_AUTH connection with SSL + listeners.push(KafkaListener { + name: KafkaListenerName::ClientAuth, + host: LISTENER_LOCAL_ADDRESS.to_string(), + port: SECURE_CLIENT_PORT.to_string(), + }); + advertised_listeners.push(KafkaListener { + name: KafkaListenerName::ClientAuth, + host: LISTENER_NODE_ADDRESS.to_string(), + port: node_port_cmd(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME), + }); + listener_security_protocol_map + .insert(KafkaListenerName::ClientAuth, KafkaListenerProtocol::Ssl); + } else if kafka.client_tls_secret_class().is_some() { + // 2) If no client authentication but tls is required we expose CLIENT with SSL + listeners.push(KafkaListener { + name: KafkaListenerName::Client, + host: LISTENER_LOCAL_ADDRESS.to_string(), + port: SECURE_CLIENT_PORT.to_string(), + }); + advertised_listeners.push(KafkaListener { + name: KafkaListenerName::Client, + host: LISTENER_NODE_ADDRESS.to_string(), + port: node_port_cmd(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME), + }); + listener_security_protocol_map + .insert(KafkaListenerName::Client, KafkaListenerProtocol::Ssl); + } else { + // 3) If no client auth or tls is required we expose CLIENT with PLAINTEXT + listeners.push(KafkaListener { + name: KafkaListenerName::Client, + host: LISTENER_LOCAL_ADDRESS.to_string(), + port: CLIENT_PORT.to_string(), + }); + advertised_listeners.push(KafkaListener { + name: KafkaListenerName::Client, + host: LISTENER_NODE_ADDRESS.to_string(), + port: node_port_cmd(STACKABLE_TMP_DIR, CLIENT_PORT_NAME), + }); + listener_security_protocol_map + .insert(KafkaListenerName::Client, KafkaListenerProtocol::Plaintext); + } + + if kafka.internal_tls_secret_class().is_some() { + // 4) If internal tls is required we expose INTERNAL as SSL + listeners.push(KafkaListener { + name: KafkaListenerName::Internal, + host: LISTENER_LOCAL_ADDRESS.to_string(), + port: SECURE_INTERNAL_PORT.to_string(), + }); + advertised_listeners.push(KafkaListener { + name: KafkaListenerName::Internal, + host: pod_fqdn, + port: SECURE_INTERNAL_PORT.to_string(), + }); + listener_security_protocol_map + .insert(KafkaListenerName::Internal, KafkaListenerProtocol::Ssl); + } else { + // 5) If no internal tls is required we expose INTERNAL as PLAINTEXT + listeners.push(KafkaListener { + name: KafkaListenerName::Internal, + host: LISTENER_LOCAL_ADDRESS.to_string(), + port: INTERNAL_PORT.to_string(), + }); + advertised_listeners.push(KafkaListener { + name: KafkaListenerName::Internal, + host: pod_fqdn, + port: INTERNAL_PORT.to_string(), + }); + listener_security_protocol_map.insert( + KafkaListenerName::Internal, + KafkaListenerProtocol::Plaintext, + ); + } + + Ok(KafkaListenerConfig { + listeners, + advertised_listeners, + listener_security_protocol_map, + }) +} + +fn node_port_cmd(directory: &str, port_name: &str) -> String { + format!("$(cat {}/{}_nodeport)", directory, port_name) +} + +fn pod_fqdn(kafka: &KafkaCluster, object_name: &str) -> Result { + Ok(format!( + "$POD_NAME.{}.{}.svc.cluster.local", + object_name, + kafka.namespace().context(ObjectHasNoNamespaceSnafu)? + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_kafka_listeners_config() { + let object_name = "simple-kafka-broker-default"; + + let input = r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + spec: + version: abc + zookeeperConfigMapName: xyz + config: + tls: + secretClass: tls + clientAuthentication: + authenticationClass: kafka-client-tls + internalTls: + secretClass: internalTls + "#; + let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); + let config = get_kafka_listener_config(&kafka, object_name).unwrap(); + + assert_eq!( + config.listeners(), + format!( + "{name}://{host}:{port},{internal_name}://{internal_host}:{internal_port}", + name = KafkaListenerName::ClientAuth, + host = LISTENER_LOCAL_ADDRESS, + port = SECURE_CLIENT_PORT, + internal_name = KafkaListenerName::Internal, + internal_host = LISTENER_LOCAL_ADDRESS, + internal_port = SECURE_INTERNAL_PORT, + ) + ); + + assert_eq!( + config.advertised_listeners(), + format!( + "{name}://{host}:{port},{internal_name}://{internal_host}:{internal_port}", + name = KafkaListenerName::ClientAuth, + host = LISTENER_NODE_ADDRESS, + port = node_port_cmd(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME), + internal_name = KafkaListenerName::Internal, + internal_host = pod_fqdn(&kafka, object_name).unwrap(), + internal_port = SECURE_INTERNAL_PORT, + ) + ); + + assert_eq!( + config.listener_security_protocol_map(), + format!( + "{name}:{protocol},{internal_name}:{internal_protocol}", + name = KafkaListenerName::ClientAuth, + protocol = KafkaListenerProtocol::Ssl, + internal_name = KafkaListenerName::Internal, + internal_protocol = KafkaListenerProtocol::Ssl + ) + ); + + let input = r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + spec: + version: abc + zookeeperConfigMapName: xyz + config: + tls: + secretClass: tls + "#; + let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); + let config = get_kafka_listener_config(&kafka, object_name).unwrap(); + + assert_eq!( + config.listeners(), + format!( + "{name}://{host}:{port},{internal_name}://{internal_host}:{internal_port}", + name = KafkaListenerName::Client, + host = LISTENER_LOCAL_ADDRESS, + port = SECURE_CLIENT_PORT, + internal_name = KafkaListenerName::Internal, + internal_host = LISTENER_LOCAL_ADDRESS, + internal_port = INTERNAL_PORT, + ) + ); + + assert_eq!( + config.advertised_listeners(), + format!( + "{name}://{host}:{port},{internal_name}://{internal_host}:{internal_port}", + name = KafkaListenerName::Client, + host = LISTENER_NODE_ADDRESS, + port = node_port_cmd(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME), + internal_name = KafkaListenerName::Internal, + internal_host = pod_fqdn(&kafka, object_name).unwrap(), + internal_port = INTERNAL_PORT, + ) + ); + + assert_eq!( + config.listener_security_protocol_map(), + format!( + "{name}:{protocol},{internal_name}:{internal_protocol}", + name = KafkaListenerName::Client, + protocol = KafkaListenerProtocol::Ssl, + internal_name = KafkaListenerName::Internal, + internal_protocol = KafkaListenerProtocol::Plaintext + ) + ); + + let input = r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + spec: + version: abc + zookeeperConfigMapName: xyz + config: + tls: null + "#; + let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); + let config = get_kafka_listener_config(&kafka, object_name).unwrap(); + + assert_eq!( + config.listeners(), + format!( + "{name}://{host}:{port},{internal_name}://{internal_host}:{internal_port}", + name = KafkaListenerName::Client, + host = LISTENER_LOCAL_ADDRESS, + port = CLIENT_PORT, + internal_name = KafkaListenerName::Internal, + internal_host = LISTENER_LOCAL_ADDRESS, + internal_port = INTERNAL_PORT, + ) + ); + + assert_eq!( + config.advertised_listeners(), + format!( + "{name}://{host}:{port},{internal_name}://{internal_host}:{internal_port}", + name = KafkaListenerName::Client, + host = LISTENER_NODE_ADDRESS, + port = node_port_cmd(STACKABLE_TMP_DIR, CLIENT_PORT_NAME), + internal_name = KafkaListenerName::Internal, + internal_host = pod_fqdn(&kafka, object_name).unwrap(), + internal_port = INTERNAL_PORT, + ) + ); + + assert_eq!( + config.listener_security_protocol_map(), + format!( + "{name}:{protocol},{internal_name}:{internal_protocol}", + name = KafkaListenerName::Client, + protocol = KafkaListenerProtocol::Plaintext, + internal_name = KafkaListenerName::Internal, + internal_protocol = KafkaListenerProtocol::Plaintext + ) + ); + } +} diff --git a/rust/operator/src/command.rs b/rust/operator/src/command.rs index 069a1dcf..242f51d5 100644 --- a/rust/operator/src/command.rs +++ b/rust/operator/src/command.rs @@ -1,34 +1,34 @@ use stackable_kafka_crd::{ KafkaCluster, CLIENT_PORT, CLIENT_PORT_NAME, SECURE_CLIENT_PORT, SECURE_CLIENT_PORT_NAME, - SSL_STORE_PASSWORD, STACKABLE_DATA_DIR, STACKABLE_TLS_CERTS_AUTHENTICATION_DIR, - STACKABLE_TLS_CERTS_DIR, STACKABLE_TLS_CERTS_INTERNAL_DIR, STACKABLE_TMP_DIR, - SYSTEM_TRUST_STORE_DIR, + SSL_STORE_PASSWORD, STACKABLE_DATA_DIR, STACKABLE_TLS_CLIENT_AUTH_DIR, + STACKABLE_TLS_CLIENT_DIR, STACKABLE_TLS_INTERNAL_DIR, STACKABLE_TMP_DIR, }; pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { let mut args = vec![]; - if kafka.client_tls_secret_class().is_some() { + if kafka.client_authentication_class().is_some() { + args.extend(create_key_and_trust_store( + STACKABLE_TLS_CLIENT_AUTH_DIR, + "stackable-tls-client-auth-ca-cert", + )); + args.extend(chown_and_chmod(STACKABLE_TLS_CLIENT_AUTH_DIR)); + } else if kafka.client_tls_secret_class().is_some() { // Copy system truststore to stackable truststore - args.push(format!("keytool -importkeystore -srckeystore {SYSTEM_TRUST_STORE_DIR} -srcstoretype jks -srcstorepass {SSL_STORE_PASSWORD} -destkeystore {STACKABLE_TLS_CERTS_DIR}/truststore.p12 -deststoretype pkcs12 -deststorepass {SSL_STORE_PASSWORD} -noprompt")); + //args.push(format!("keytool -importkeystore -srckeystore {SYSTEM_TRUST_STORE_DIR} -srcstoretype jks -srcstorepass {SSL_STORE_PASSWORD} -destkeystore {STACKABLE_TLS_CLIENT_DIR}/truststore.p12 -deststoretype pkcs12 -deststorepass {SSL_STORE_PASSWORD} -noprompt")); args.extend(create_key_and_trust_store( - STACKABLE_TLS_CERTS_DIR, - "stackable-tls-ca-cert", + STACKABLE_TLS_CLIENT_DIR, + "stackable-tls-client-ca-cert", )); - args.extend(chown_and_chmod(STACKABLE_TLS_CERTS_DIR)); - - if kafka.client_authentication_class().is_some() { - args.push(format!("echo [{STACKABLE_TLS_CERTS_DIR}] Importing client authentication cert to truststore")); - args.push(format!("keytool -importcert -file {STACKABLE_TLS_CERTS_AUTHENTICATION_DIR}/ca.crt -keystore {STACKABLE_TLS_CERTS_DIR}/truststore.p12 -storetype pkcs12 -noprompt -alias stackable-tls-client-ca-cert -storepass {SSL_STORE_PASSWORD}")); - } + args.extend(chown_and_chmod(STACKABLE_TLS_CLIENT_DIR)); } if kafka.internal_tls_secret_class().is_some() { args.extend(create_key_and_trust_store( - STACKABLE_TLS_CERTS_INTERNAL_DIR, - "stackable-internal-tls-ca-cert", + STACKABLE_TLS_INTERNAL_DIR, + "stackable-tls-internal-ca-cert", )); - args.extend(chown_and_chmod(STACKABLE_TLS_CERTS_INTERNAL_DIR)); + args.extend(chown_and_chmod(STACKABLE_TLS_INTERNAL_DIR)); } args.extend(chown_and_chmod(STACKABLE_DATA_DIR)); @@ -58,32 +58,36 @@ pub fn get_svc_container_cmd_args(kafka: &KafkaCluster) -> String { pub fn kcat_container_cmd_args(kafka: &KafkaCluster) -> Vec { let mut args = vec!["kcat".to_string()]; - if kafka.client_tls_secret_class().is_some() { + if kafka.client_authentication_class().is_some() { + args.push("-b".to_string()); + args.push(format!("localhost:{}", SECURE_CLIENT_PORT)); + args.extend(kcat_client_ssl(STACKABLE_TLS_CLIENT_AUTH_DIR)); + } else if kafka.client_tls_secret_class().is_some() { args.push("-b".to_string()); args.push(format!("localhost:{}", SECURE_CLIENT_PORT)); - args.extend([ - "-X".to_string(), - "security.protocol=SSL".to_string(), - "-X".to_string(), - format!("ssl.key.location={}/tls.key", STACKABLE_TLS_CERTS_DIR), - "-X".to_string(), - format!( - "ssl.certificate.location={}/tls.crt", - STACKABLE_TLS_CERTS_DIR - ), - "-X".to_string(), - format!("ssl.ca.location={}/ca.crt", STACKABLE_TLS_CERTS_DIR), - ]); + args.extend(kcat_client_ssl(STACKABLE_TLS_CLIENT_DIR)); } else { args.push("-b".to_string()); args.push(format!("localhost:{}", CLIENT_PORT)); } args.push("-L".to_string()); - args } +fn kcat_client_ssl(cert_directory: &str) -> Vec { + vec![ + "-X".to_string(), + "security.protocol=SSL".to_string(), + "-X".to_string(), + format!("ssl.key.location={}/tls.key", cert_directory), + "-X".to_string(), + format!("ssl.certificate.location={}/tls.crt", cert_directory), + "-X".to_string(), + format!("ssl.ca.location={}/ca.crt", cert_directory), + ] +} + /// Generates the shell script to create key and truststores from the certificates provided /// by the secret operator. fn create_key_and_trust_store(directory: &str, alias_name: &str) -> Vec { diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index bcd254d3..058ebcae 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -2,13 +2,12 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_kafka_crd::{ - KafkaCluster, KafkaRole, TlsSecretClass, APP_NAME, CLIENT_PORT, CLIENT_PORT_NAME, - INTERNAL_PORT, KAFKA_HEAP_OPTS, LOG_DIRS_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, - SECURE_CLIENT_PORT, SECURE_CLIENT_PORT_NAME, SECURE_INTERNAL_PORT, SERVER_PROPERTIES_FILE, - STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_TLS_CERTS_AUTHENTICATION_DIR, - STACKABLE_TLS_CERTS_DIR, STACKABLE_TLS_CERTS_INTERNAL_DIR, STACKABLE_TMP_DIR, + listener::get_kafka_listener_config, KafkaCluster, KafkaRole, TlsSecretClass, APP_NAME, + CLIENT_PORT, CLIENT_PORT_NAME, KAFKA_HEAP_OPTS, LOG_DIRS_VOLUME_NAME, METRICS_PORT, + METRICS_PORT_NAME, SECURE_CLIENT_PORT, SECURE_CLIENT_PORT_NAME, SERVER_PROPERTIES_FILE, + STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_TLS_CLIENT_AUTH_DIR, + STACKABLE_TLS_CLIENT_DIR, STACKABLE_TLS_INTERNAL_DIR, STACKABLE_TMP_DIR, }; -use stackable_operator::commons::tls::TlsAuthenticationProvider; use stackable_operator::{ builder::{ ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder, @@ -17,6 +16,7 @@ use stackable_operator::{ commons::{ authentication::{AuthenticationClass, AuthenticationClassProvider}, opa::OpaApiVersion, + tls::TlsAuthenticationProvider, }, k8s_openapi::{ api::{ @@ -34,7 +34,6 @@ use stackable_operator::{ kube::{ api::DynamicObject, runtime::{controller::Action, reflector::ObjectRef}, - ResourceExt, }, labels::{role_group_selector_labels, role_selector_labels}, logging::controller::ReconcilerError, @@ -171,6 +170,10 @@ pub enum Error { supported: Vec, method: String, }, + #[snafu(display("invalid kafka listeners"))] + InvalidKafkaListeners { + source: stackable_kafka_crd::listener::KafkaListenerError, + }, } type Result = std::result::Result; @@ -212,6 +215,7 @@ impl ReconcilerError for Error { authentication_class, .. } => Some(authentication_class.clone().erase()), + Error::InvalidKafkaListeners { .. } => None, } } } @@ -565,43 +569,49 @@ fn build_broker_rolegroup_statefulset( let get_svc_args = get_svc_container_cmd_args(kafka); - if let Some(tls) = kafka.client_tls_secret_class() { - cb_prepare.add_volume_mount("tls-certificate", STACKABLE_TLS_CERTS_DIR); - cb_kafka.add_volume_mount("tls-certificate", STACKABLE_TLS_CERTS_DIR); - cb_kcat_prober.add_volume_mount("tls-certificate", STACKABLE_TLS_CERTS_DIR); - pod_builder.add_volume(create_tls_volume("tls-certificate", Some(tls))); - - // add client authentication if required - if let Some(auth_class) = client_authentication_class { - match &auth_class.spec.provider { - AuthenticationClassProvider::Tls(TlsAuthenticationProvider { - client_cert_secret_class: Some(secret_class), - }) => { - cb_prepare.add_volume_mount( - "client-tls-authentication-certificate", - STACKABLE_TLS_CERTS_AUTHENTICATION_DIR, - ); - pod_builder.add_volume(create_tls_volume( - "client-tls-authentication-certificate", - Some(&TlsSecretClass { - secret_class: secret_class.clone(), - }), - )); - } - _ => { - return Err(Error::AuthenticationMethodNotSupported { - authentication_class: ObjectRef::from_obj(auth_class), - supported: vec!["tls".to_string()], - method: auth_class.spec.provider.to_string(), - }) - } + // add client authentication volumes if required + if let Some(auth_class) = client_authentication_class { + match &auth_class.spec.provider { + AuthenticationClassProvider::Tls(TlsAuthenticationProvider { + client_cert_secret_class: Some(secret_class), + }) => { + cb_prepare.add_volume_mount( + "client-tls-authentication-certificate", + STACKABLE_TLS_CLIENT_AUTH_DIR, + ); + cb_kafka.add_volume_mount( + "client-tls-authentication-certificate", + STACKABLE_TLS_CLIENT_AUTH_DIR, + ); + cb_kcat_prober.add_volume_mount( + "client-tls-authentication-certificate", + STACKABLE_TLS_CLIENT_AUTH_DIR, + ); + pod_builder.add_volume(create_tls_volume( + "client-tls-authentication-certificate", + Some(&TlsSecretClass { + secret_class: secret_class.clone(), + }), + )); + } + _ => { + return Err(Error::AuthenticationMethodNotSupported { + authentication_class: ObjectRef::from_obj(auth_class), + supported: vec!["tls".to_string()], + method: auth_class.spec.provider.to_string(), + }) } } + } else if let Some(tls) = kafka.client_tls_secret_class() { + cb_prepare.add_volume_mount("client-tls-certificate", STACKABLE_TLS_CLIENT_DIR); + cb_kafka.add_volume_mount("client-tls-certificate", STACKABLE_TLS_CLIENT_DIR); + cb_kcat_prober.add_volume_mount("client-tls-certificate", STACKABLE_TLS_CLIENT_DIR); + pod_builder.add_volume(create_tls_volume("client-tls-certificate", Some(tls))); } if let Some(tls_internal) = kafka.internal_tls_secret_class() { - cb_prepare.add_volume_mount("internal-tls-certificate", STACKABLE_TLS_CERTS_INTERNAL_DIR); - cb_kafka.add_volume_mount("internal-tls-certificate", STACKABLE_TLS_CERTS_INTERNAL_DIR); + cb_prepare.add_volume_mount("internal-tls-certificate", STACKABLE_TLS_INTERNAL_DIR); + cb_kafka.add_volume_mount("internal-tls-certificate", STACKABLE_TLS_INTERNAL_DIR); pod_builder.add_volume(create_tls_volume( "internal-tls-certificate", Some(tls_internal), @@ -718,15 +728,16 @@ fn build_broker_rolegroup_statefulset( let jvm_args = format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar={}:/stackable/jmx/broker.yaml", METRICS_PORT); let zookeeper_override = "--override \"zookeeper.connect=$ZOOKEEPER\""; - let kafka_listeners = get_kafka_listener(kafka, rolegroup_ref)?; - let listeners_override = format!("--override \"listeners={}\"", kafka_listeners.listener); + let kafka_listeners = get_kafka_listener_config(kafka, &rolegroup_ref.object_name()) + .context(InvalidKafkaListenersSnafu)?; + let listeners_override = format!("--override \"listeners={}\"", kafka_listeners.listeners()); let advertised_listeners_override = format!( "--override \"advertised.listeners={}\"", - kafka_listeners.advertised_listener + kafka_listeners.advertised_listeners() ); let listener_security_protocol_map_override = format!( "--override \"listener.security.protocol.map={}\"", - kafka_listeners.listener_security_protocol_map + kafka_listeners.listener_security_protocol_map() ); let opa_url_override = opa_connect_string.map_or("".to_string(), |opa| { format!("--override \"opa.authorizer.url={}\"", opa) @@ -855,6 +866,7 @@ pub fn error_policy(_error: &Error, _ctx: Arc) -> Action { Action::requeue(Duration::from_secs(5)) } +/// We only expose client HTTP / HTTPS and Metrics ports. fn service_ports(kafka: &KafkaCluster) -> Vec { let mut ports = vec![ServicePort { name: Some(METRICS_PORT_NAME.to_string()), @@ -863,7 +875,7 @@ fn service_ports(kafka: &KafkaCluster) -> Vec { ..ServicePort::default() }]; - if kafka.client_tls_secret_class().is_some() { + if kafka.client_tls_secret_class().is_some() || kafka.client_authentication_class().is_some() { ports.push(ServicePort { name: Some(SECURE_CLIENT_PORT_NAME.to_string()), port: SECURE_CLIENT_PORT.into(), @@ -882,6 +894,7 @@ fn service_ports(kafka: &KafkaCluster) -> Vec { ports } +/// We only expose client HTTP / HTTPS and Metrics ports. fn container_ports(kafka: &KafkaCluster) -> Vec { let mut ports = vec![ContainerPort { name: Some(METRICS_PORT_NAME.to_string()), @@ -890,7 +903,7 @@ fn container_ports(kafka: &KafkaCluster) -> Vec { ..ContainerPort::default() }]; - if kafka.client_tls_secret_class().is_some() { + if kafka.client_tls_secret_class().is_some() || kafka.client_authentication_class().is_some() { ports.push(ContainerPort { name: Some(SECURE_CLIENT_PORT_NAME.to_string()), container_port: SECURE_CLIENT_PORT.into(), @@ -925,63 +938,3 @@ fn create_tls_volume(volume_name: &str, tls_secret_class: Option<&TlsSecretClass ) .build() } - -struct KafkaListener { - pub listener: String, - pub advertised_listener: String, - pub listener_security_protocol_map: String, -} - -/// Returns the `listener`, `advertised.listener` and `listener.security.protocol.map` properties -/// depending on internal and external TLS settings. -fn get_kafka_listener( - kafka: &KafkaCluster, - rolegroup_ref: &RoleGroupRef, -) -> Result { - let pod_fqdn = format!( - "$POD_NAME.{}.{}.svc.cluster.local", - rolegroup_ref.object_name(), - kafka.namespace().context(ObjectHasNoNamespaceSnafu)? - ); - - if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { - // If both client and internal TLS are set we do not need the HTTP port. - Ok(KafkaListener { - listener: format!("internal://0.0.0.0:{SECURE_INTERNAL_PORT},SSL://0.0.0.0:{SECURE_CLIENT_PORT}"), - advertised_listener: format!( - "internal://{pod}:{SECURE_INTERNAL_PORT},SSL://$NODE:$(cat {STACKABLE_TMP_DIR}/{SECURE_CLIENT_PORT_NAME}_nodeport)", pod = pod_fqdn - ), - listener_security_protocol_map: "internal:SSL,SSL:SSL".to_string() - }) - } else if kafka.client_tls_secret_class().is_some() { - // If only client TLS is required we need to set the HTTPS port and keep the HTTP port - // for internal communications. - Ok(KafkaListener { - listener: format!("internal://0.0.0.0:{INTERNAL_PORT},SSL://0.0.0.0:{SECURE_CLIENT_PORT}"), - advertised_listener: format!( - "internal://{pod}:{INTERNAL_PORT},SSL://$NODE:$(cat {STACKABLE_TMP_DIR}/{SECURE_CLIENT_PORT_NAME}_nodeport)", pod = pod_fqdn - ), - listener_security_protocol_map: "internal:PLAINTEXT,SSL:SSL".to_string() - }) - } else if kafka.internal_tls_secret_class().is_some() { - // If only internal TLS is required we need to set the HTTPS port and keep the HTTP port - // for client communications. - Ok(KafkaListener { - listener: format!("internal://0.0.0.0:{SECURE_INTERNAL_PORT},PLAINTEXT://0.0.0.0:{CLIENT_PORT}"), - advertised_listener: format!( - "internal://{pod}:{SECURE_INTERNAL_PORT},PLAINTEXT://$NODE:$(cat {STACKABLE_TMP_DIR}/{CLIENT_PORT_NAME}_nodeport)", pod = pod_fqdn - ), - listener_security_protocol_map: "internal:SSL,PLAINTEXT:PLAINTEXT".to_string() - }) - } else { - // If no is TLS specified the HTTP port is sufficient - Ok(KafkaListener { - listener: format!("internal://0.0.0.0:{INTERNAL_PORT},PLAINTEXT://0.0.0.0:{CLIENT_PORT}"), - advertised_listener: format!( - "internal://{pod}:{INTERNAL_PORT},PLAINTEXT://$NODE:$(cat {STACKABLE_TMP_DIR}/{CLIENT_PORT_NAME}_nodeport)", - pod = pod_fqdn - ), - listener_security_protocol_map: "internal:PLAINTEXT,PLAINTEXT:PLAINTEXT".to_string() - }) - } -} From 953efb1df92cdd59afad4c5475d54fb2d9846c25 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Aug 2022 11:51:43 +0200 Subject: [PATCH 18/37] added default for internal tls --- deploy/crd/kafkacluster.crd.yaml | 4 +++ deploy/helm/kafka-operator/crds/crds.yaml | 4 +++ deploy/manifests/crds.yaml | 4 +++ rust/crd/src/lib.rs | 21 ++++---------- tests/test-definition.yaml | 34 ++++++++++++----------- 5 files changed, 35 insertions(+), 32 deletions(-) diff --git a/deploy/crd/kafkacluster.crd.yaml b/deploy/crd/kafkacluster.crd.yaml index 326d334f..f243a061 100644 --- a/deploy/crd/kafkacluster.crd.yaml +++ b/deploy/crd/kafkacluster.crd.yaml @@ -279,6 +279,8 @@ spec: default: tls: secretClass: tls + internalTls: + secretClass: tls properties: clientAuthentication: description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" @@ -290,6 +292,8 @@ spec: - authenticationClass type: object internalTls: + default: + secretClass: tls description: "Only affects internal communication. Use mutual verification between Trino nodes This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" nullable: true properties: diff --git a/deploy/helm/kafka-operator/crds/crds.yaml b/deploy/helm/kafka-operator/crds/crds.yaml index 099a199e..3430144f 100644 --- a/deploy/helm/kafka-operator/crds/crds.yaml +++ b/deploy/helm/kafka-operator/crds/crds.yaml @@ -281,6 +281,8 @@ spec: default: tls: secretClass: tls + internalTls: + secretClass: tls properties: clientAuthentication: description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" @@ -292,6 +294,8 @@ spec: - authenticationClass type: object internalTls: + default: + secretClass: tls description: "Only affects internal communication. Use mutual verification between Trino nodes This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" nullable: true properties: diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index 8c8ec280..711e8376 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -282,6 +282,8 @@ spec: default: tls: secretClass: tls + internalTls: + secretClass: tls properties: clientAuthentication: description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" @@ -293,6 +295,8 @@ spec: - authenticationClass type: object internalTls: + default: + secretClass: tls description: "Only affects internal communication. Use mutual verification between Trino nodes This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" nullable: true properties: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index a25fb9db..96031426 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -38,10 +38,6 @@ pub const SERVER_PROPERTIES_FILE: &str = "server.properties"; pub const KAFKA_HEAP_OPTS: &str = "KAFKA_HEAP_OPTS"; // server_properties pub const LOG_DIRS_VOLUME_NAME: &str = "log-dirs"; -// - listener -pub const LISTENER_SECURITY_PROTOCOL_MAP: &str = "listener.security.protocol.map"; -pub const LISTENER: &str = "listeners"; -pub const ADVERTISED_LISTENER: &str = "advertised.listeners"; // - TLS global pub const TLS_DEFAULT_SECRET_CLASS: &str = "tls"; pub const SSL_KEYSTORE_LOCATION: &str = "ssl.keystore.location"; @@ -80,8 +76,6 @@ pub const INTER_SSL_TRUSTSTORE_LOCATION: &str = "listener.name.internal.ssl.trus pub const INTER_SSL_TRUSTSTORE_PASSWORD: &str = "listener.name.internal.ssl.truststore.password"; pub const INTER_SSL_TRUSTSTORE_TYPE: &str = "listener.name.internal.ssl.truststore.type"; pub const INTER_SSL_CLIENT_AUTH: &str = "listener.name.internal.ssl.client.auth"; -pub const INTER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: &str = - "listener.name.internal.ssl.endpoint.identification.algorithm"; // directories pub const STACKABLE_TMP_DIR: &str = "/stackable/tmp"; pub const STACKABLE_DATA_DIR: &str = "/stackable/data"; @@ -151,7 +145,10 @@ pub struct GlobalKafkaConfig { /// This setting controls: /// - Which cert the servers should use to authenticate themselves against other servers /// - Which ca.crt to use when validating the other server - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + default = "tls_secret_class_default", + skip_serializing_if = "Option::is_none" + )] pub internal_tls: Option, } @@ -160,7 +157,7 @@ impl Default for GlobalKafkaConfig { GlobalKafkaConfig { tls: tls_secret_class_default(), client_authentication: None, - internal_tls: None, + internal_tls: tls_secret_class_default(), } } } @@ -463,10 +460,6 @@ impl Configuration for KafkaConfig { CLIENT_AUTH_SSL_CLIENT_AUTH.to_string(), Some("required".to_string()), ); - // config.insert( - // SSL_ENDPOINT_IDENTIFICATION_ALGORITHM.to_string(), - // Some("HTTPS".to_string()), - // ); } // Client TLS else if resource.client_tls_secret_class().is_some() { @@ -526,10 +519,6 @@ impl Configuration for KafkaConfig { INTER_SSL_CLIENT_AUTH.to_string(), Some("required".to_string()), ); - config.insert( - INTER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM.to_string(), - Some("HTTPS".to_string()), - ); } // common diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 956b5aa7..60050c79 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -2,14 +2,14 @@ dimensions: - name: kafka values: - #- 2.7.1-stackable0 - #- 2.8.1-stackable0 - #- 3.1.0-stackable0 + - 2.7.1-stackable0 + - 2.8.1-stackable0 + - 3.1.0-stackable0 - 3.2.0-stackable0.1.0 - name: zookeeper values: - #- 3.6.3-stackable0.7.1 - #- 3.7.0-stackable0.7.1 + - 3.6.3-stackable0.7.1 + - 3.7.0-stackable0.7.1 - 3.8.0-stackable0.7.1 - name: upgrade_old values: @@ -22,21 +22,23 @@ dimensions: - name: use-client-tls values: - "true" - #- "false" + - "false" - name: use-internal-tls values: - #- "true" + - "true" - "false" tests: -# - name: smoke -# dimensions: -# - kafka -# - zookeeper -# - name: upgrade -# dimensions: -# - zookeeper -# - upgrade_new -# - upgrade_old + - name: smoke + dimensions: + - kafka + - zookeeper + - use-client-tls + - name: upgrade + dimensions: + - zookeeper + - upgrade_new + - upgrade_old + - use-client-tls - name: tls dimensions: - kafka From c0827fff384d52c41cc93873a65159db3b5616e0 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Aug 2022 11:54:55 +0200 Subject: [PATCH 19/37] adapted internal tls default in docs --- docs/modules/ROOT/pages/usage.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index 7d84008b..0fbd9911 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -141,7 +141,7 @@ spec: replicas: 3 ---- <1> The `tls.secretClass` refers to the client-to-server encryption. Defaults to the `tls` secret. It can be deactivated by setting `config.tls` to `null`. -<2> The `internalTls.secretClass` refers to the broker-to-broker internal encryption. This must be explicitly set. +<2> The `internalTls.secretClass` refers to the broker-to-broker internal encryption. This must be explicitly set or defaults to `tls`. Can be disabled by setting `config.internalTls` to `null`. The `tls` secret is deployed from the xref:secret-operator::index.adoc[Secret Operator] and looks like this: From bcf6c59268a65bcbeaffedeccd75c05c8ec190f9 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Aug 2022 12:14:50 +0200 Subject: [PATCH 20/37] fixed tests for default internal tls --- rust/crd/src/lib.rs | 25 ++++++++++++++++++++----- rust/crd/src/listener.rs | 7 ++++--- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 96031426..33bee2e4 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -552,7 +552,10 @@ mod tests { kafka.client_tls_secret_class().unwrap().secret_class, TLS_DEFAULT_SECRET_CLASS.to_string() ); - assert_eq!(kafka.internal_tls_secret_class(), None); + assert_eq!( + kafka.internal_tls_secret_class().unwrap().secret_class, + TLS_DEFAULT_SECRET_CLASS.to_string() + ); let input = r#" apiVersion: kafka.stackable.tech/v1alpha1 @@ -571,7 +574,10 @@ mod tests { kafka.client_tls_secret_class().unwrap().secret_class, "simple-kafka-client-tls".to_string() ); - assert_eq!(kafka.internal_tls_secret_class(), None); + assert_eq!( + kafka.internal_tls_secret_class().unwrap().secret_class, + TLS_DEFAULT_SECRET_CLASS + ); let input = r#" apiVersion: kafka.stackable.tech/v1alpha1 @@ -586,7 +592,10 @@ mod tests { "#; let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); assert_eq!(kafka.client_tls_secret_class(), None); - assert_eq!(kafka.internal_tls_secret_class(), None); + assert_eq!( + kafka.internal_tls_secret_class().unwrap().secret_class, + TLS_DEFAULT_SECRET_CLASS.to_string() + ); let input = r#" apiVersion: kafka.stackable.tech/v1alpha1 @@ -623,7 +632,10 @@ mod tests { zookeeperConfigMapName: xyz "#; let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); - assert_eq!(kafka.internal_tls_secret_class(), None); + assert_eq!( + kafka.internal_tls_secret_class().unwrap().secret_class, + TLS_DEFAULT_SECRET_CLASS.to_string() + ); assert_eq!( kafka.client_tls_secret_class().unwrap().secret_class, TLS_DEFAULT_SECRET_CLASS @@ -664,7 +676,10 @@ mod tests { secretClass: simple-kafka-client-tls "#; let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); - assert_eq!(kafka.internal_tls_secret_class(), None); + assert_eq!( + kafka.internal_tls_secret_class().unwrap().secret_class, + TLS_DEFAULT_SECRET_CLASS.to_string() + ); assert_eq!( kafka.client_tls_secret_class().unwrap().secret_class, "simple-kafka-client-tls" diff --git a/rust/crd/src/listener.rs b/rust/crd/src/listener.rs index d7d6c15e..2f03157b 100644 --- a/rust/crd/src/listener.rs +++ b/rust/crd/src/listener.rs @@ -282,7 +282,7 @@ mod tests { port = SECURE_CLIENT_PORT, internal_name = KafkaListenerName::Internal, internal_host = LISTENER_LOCAL_ADDRESS, - internal_port = INTERNAL_PORT, + internal_port = SECURE_INTERNAL_PORT, ) ); @@ -295,7 +295,7 @@ mod tests { port = node_port_cmd(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME), internal_name = KafkaListenerName::Internal, internal_host = pod_fqdn(&kafka, object_name).unwrap(), - internal_port = INTERNAL_PORT, + internal_port = SECURE_INTERNAL_PORT, ) ); @@ -306,7 +306,7 @@ mod tests { name = KafkaListenerName::Client, protocol = KafkaListenerProtocol::Ssl, internal_name = KafkaListenerName::Internal, - internal_protocol = KafkaListenerProtocol::Plaintext + internal_protocol = KafkaListenerProtocol::Ssl ) ); @@ -321,6 +321,7 @@ mod tests { zookeeperConfigMapName: xyz config: tls: null + internalTls: null "#; let kafka: KafkaCluster = serde_yaml::from_str(input).expect("illegal test input"); let config = get_kafka_listener_config(&kafka, object_name).unwrap(); From 45f3b72a5a7aafa022b79b0403a8243e334febd8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Aug 2022 15:12:27 +0200 Subject: [PATCH 21/37] client authentication tests working --- examples/tls/simple-kafka-cluster-tls.yaml | 4 +- rust/crd/src/lib.rs | 7 +-- rust/operator/src/command.rs | 35 ++++++----- .../kuttl/tls/20-install-kafka.yaml.j2 | 14 +++-- tests/templates/kuttl/tls/30-assert.yaml.j2 | 7 +-- .../kuttl/tls/30-prepare-test-kafka.yaml.j2 | 7 +-- .../kuttl/tls/test_client_auth_tls.sh | 62 +++++++++++++++++++ tests/templates/kuttl/tls/test_client_tls.sh | 6 +- .../templates/kuttl/tls/test_internal_tls.sh | 4 -- 9 files changed, 103 insertions(+), 43 deletions(-) create mode 100755 tests/templates/kuttl/tls/test_client_auth_tls.sh delete mode 100755 tests/templates/kuttl/tls/test_internal_tls.sh diff --git a/examples/tls/simple-kafka-cluster-tls.yaml b/examples/tls/simple-kafka-cluster-tls.yaml index f076fccf..f68cc074 100644 --- a/examples/tls/simple-kafka-cluster-tls.yaml +++ b/examples/tls/simple-kafka-cluster-tls.yaml @@ -38,7 +38,7 @@ spec: apiVersion: authentication.stackable.tech/v1alpha1 kind: AuthenticationClass metadata: - name: kafka-client-tls + name: kafka-client-auth-tls spec: provider: tls: @@ -68,7 +68,7 @@ spec: tls: secretClass: tls clientAuthentication: - authenticationClass: kafka-client-tls + authenticationClass: kafka-client-auth-tls internalTls: secretClass: kafka-internal-tls brokers: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 33bee2e4..3215282b 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -454,15 +454,12 @@ impl Configuration for KafkaConfig { CLIENT_AUTH_SSL_TRUSTSTORE_TYPE.to_string(), Some("PKCS12".to_string()), ); - - // Authentication + // client auth required config.insert( CLIENT_AUTH_SSL_CLIENT_AUTH.to_string(), Some("required".to_string()), ); - } - // Client TLS - else if resource.client_tls_secret_class().is_some() { + } else if resource.client_tls_secret_class().is_some() { config.insert( CLIENT_SSL_KEYSTORE_LOCATION.to_string(), Some(format!("{}/keystore.p12", STACKABLE_TLS_CLIENT_DIR)), diff --git a/rust/operator/src/command.rs b/rust/operator/src/command.rs index 242f51d5..03360e7d 100644 --- a/rust/operator/src/command.rs +++ b/rust/operator/src/command.rs @@ -38,21 +38,15 @@ pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { } pub fn get_svc_container_cmd_args(kafka: &KafkaCluster) -> String { - if kafka.client_tls_secret_class().is_some() && kafka.internal_tls_secret_class().is_some() { - get_node_port(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME) - } else if kafka.client_tls_secret_class().is_some() - || kafka.internal_tls_secret_class().is_some() + let port_name = if kafka.client_tls_secret_class().is_some() + || kafka.client_authentication_class().is_some() { - [ - get_node_port(STACKABLE_TMP_DIR, CLIENT_PORT_NAME), - get_node_port(STACKABLE_TMP_DIR, SECURE_CLIENT_PORT_NAME), - ] - .join(" && ") - } - // If no is TLS specified the HTTP port is sufficient - else { - get_node_port(STACKABLE_TMP_DIR, CLIENT_PORT_NAME) - } + SECURE_CLIENT_PORT_NAME + } else { + CLIENT_PORT_NAME + }; + + get_node_port(STACKABLE_TMP_DIR, port_name) } pub fn kcat_container_cmd_args(kafka: &KafkaCluster) -> Vec { @@ -61,7 +55,7 @@ pub fn kcat_container_cmd_args(kafka: &KafkaCluster) -> Vec { if kafka.client_authentication_class().is_some() { args.push("-b".to_string()); args.push(format!("localhost:{}", SECURE_CLIENT_PORT)); - args.extend(kcat_client_ssl(STACKABLE_TLS_CLIENT_AUTH_DIR)); + args.extend(kcat_client_auth_ssl(STACKABLE_TLS_CLIENT_AUTH_DIR)); } else if kafka.client_tls_secret_class().is_some() { args.push("-b".to_string()); args.push(format!("localhost:{}", SECURE_CLIENT_PORT)); @@ -75,7 +69,7 @@ pub fn kcat_container_cmd_args(kafka: &KafkaCluster) -> Vec { args } -fn kcat_client_ssl(cert_directory: &str) -> Vec { +fn kcat_client_auth_ssl(cert_directory: &str) -> Vec { vec![ "-X".to_string(), "security.protocol=SSL".to_string(), @@ -88,6 +82,15 @@ fn kcat_client_ssl(cert_directory: &str) -> Vec { ] } +fn kcat_client_ssl(cert_directory: &str) -> Vec { + vec![ + "-X".to_string(), + "security.protocol=SSL".to_string(), + "-X".to_string(), + format!("ssl.ca.location={}/ca.crt", cert_directory), + ] +} + /// Generates the shell script to create key and truststores from the certificates provided /// by the secret operator. fn create_key_and_trust_store(directory: &str, alias_name: &str) -> Vec { diff --git a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 index b3241c05..064675d1 100644 --- a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 +++ b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 @@ -11,16 +11,16 @@ spec: apiVersion: authentication.stackable.tech/v1alpha1 kind: AuthenticationClass metadata: - name: test-kafka-client-tls + name: test-kafka-client-auth-tls spec: provider: tls: - clientCertSecretClass: test-kafka-client-tls + clientCertSecretClass: test-kafka-client-auth-tls --- apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass metadata: - name: test-kafka-client-tls + name: test-kafka-client-auth-tls spec: backend: autoTls: @@ -57,14 +57,18 @@ spec: {% if test_scenario['values']['use-client-tls'] == 'true' %} tls: secretClass: tls - clientAuthentication: - authenticationClass: test-kafka-client-tls {% else %} tls: null {% endif %} +{% if test_scenario['values']['use-client-auth-tls'] == 'true' %} + clientAuthentication: + authenticationClass: test-kafka-client-auth-tls +{% endif %} {% if test_scenario['values']['use-internal-tls'] == 'true' %} internalTls: secretClass: test-kafka-internal-tls +{% else %} + internalTls: null {% endif %} brokers: roleGroups: diff --git a/tests/templates/kuttl/tls/30-assert.yaml.j2 b/tests/templates/kuttl/tls/30-assert.yaml.j2 index 799d975e..2a1cf896 100644 --- a/tests/templates/kuttl/tls/30-assert.yaml.j2 +++ b/tests/templates/kuttl/tls/30-assert.yaml.j2 @@ -4,9 +4,8 @@ kind: TestAssert metadata: name: test-tls commands: -{% if test_scenario['values']['use-client-tls'] == 'true' %} +{% if test_scenario['values']['use-client-auth-tls'] == 'true' %} + - script: kubectl exec -n $NAMESPACE test-kafka-broker-default-0 -- /tmp/test_client_auth_tls.sh $NAMESPACE +{% elif test_scenario['values']['use-client-tls'] == 'true' %} - script: kubectl exec -n $NAMESPACE test-kafka-broker-default-0 -- /tmp/test_client_tls.sh $NAMESPACE {% endif %} -{% if test_scenario['values']['use-internal-tls'] == 'true' %} - - script: kubectl exec -n $NAMESPACE test-kafka-broker-default-0 -- /tmp/test_internal_tls.sh $NAMESPACE -{% endif %} diff --git a/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml.j2 b/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml.j2 index 75569f35..79921ba7 100644 --- a/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml.j2 +++ b/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml.j2 @@ -2,9 +2,8 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: -{% if test_scenario['values']['use-client-tls'] == 'true' %} +{% if test_scenario['values']['use-client-auth-tls'] == 'true' %} + - script: kubectl cp -n $NAMESPACE ./test_client_auth_tls.sh test-kafka-broker-default-0:/tmp +{% elif test_scenario['values']['use-client-tls'] == 'true' %} - script: kubectl cp -n $NAMESPACE ./test_client_tls.sh test-kafka-broker-default-0:/tmp {% endif %} -{% if test_scenario['values']['use-internal-tls'] == 'true' %} - - script: kubectl cp -n $NAMESPACE ./test_internal_tls.sh test-kafka-broker-default-0:/tmp -{% endif %} diff --git a/tests/templates/kuttl/tls/test_client_auth_tls.sh b/tests/templates/kuttl/tls/test_client_auth_tls.sh new file mode 100755 index 00000000..3607d5dc --- /dev/null +++ b/tests/templates/kuttl/tls/test_client_auth_tls.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Usage: test_tls.sh namespace + +NAMESPACE=$1 + +# to be safe +unset TOPIC +unset BAD_TOPIC + +SERVER="test-kafka-broker-default-0.test-kafka-broker-default.${NAMESPACE}.svc.cluster.local:9093" + +echo "Start client auth TLS testing..." +############################################################################ +# Test the secured connection +############################################################################ +# create random topics +TOPIC=$(tr -dc A-Za-z0-9 /tmp/client.config + +if /stackable/kafka/bin/kafka-topics.sh --create --topic $TOPIC --bootstrap-server $SERVER --command-config /tmp/client.config +then + echo "[SUCCESS] Secure client topic created!" +else + echo "[ERROR] Secure client topic creation failed!" + exit 1 +fi + +if /stackable/kafka/bin/kafka-topics.sh --list --topic $TOPIC --bootstrap-server $SERVER --command-config /tmp/client.config | grep $TOPIC +then + echo "[SUCCESS] Secure client topic read!" +else + echo "[ERROR] Secure client topic read failed!" + exit 1 +fi + +############################################################################ +# Test the connection without certificates +############################################################################ +if /stackable/kafka/bin/kafka-topics.sh --create --topic $BAD_TOPIC --bootstrap-server $SERVER &> /dev/null +then + echo "[ERROR] Secure client topic created without certificates!" + exit 1 +else + echo "[SUCCESS] Secure client topic creation failed without certificates!" +fi + +############################################################################ +# Test the connection with bad host name +############################################################################ +if /stackable/kafka/bin/kafka-topics.sh --create --topic $BAD_TOPIC --bootstrap-server localhost:9093 --command-config /tmp/client.config &> /dev/null +then + echo "[ERROR] Secure client topic created with bad host name!" + exit 1 +else + echo "[SUCCESS] Secure client topic creation failed with bad host name!" +fi + +echo "All client auth TLS tests successful!" +exit 0 diff --git a/tests/templates/kuttl/tls/test_client_tls.sh b/tests/templates/kuttl/tls/test_client_tls.sh index f5178eb6..fc7b436f 100755 --- a/tests/templates/kuttl/tls/test_client_tls.sh +++ b/tests/templates/kuttl/tls/test_client_tls.sh @@ -9,7 +9,7 @@ unset BAD_TOPIC SERVER="test-kafka-broker-default-0.test-kafka-broker-default.${NAMESPACE}.svc.cluster.local:9093" -echo "Start TLS testing..." +echo "Start client TLS testing..." ############################################################################ # Test the secured connection ############################################################################ @@ -18,7 +18,7 @@ TOPIC=$(tr -dc A-Za-z0-9 /tmp/client.config +echo $'security.protocol=SSL\nssl.truststore.location=/stackable/tls_client/truststore.p12\nssl.truststore.password=changeit' > /tmp/client.config if /stackable/kafka/bin/kafka-topics.sh --create --topic $TOPIC --bootstrap-server $SERVER --command-config /tmp/client.config then @@ -58,5 +58,5 @@ else echo "[SUCCESS] Secure client topic creation failed with bad host name!" fi -echo "All TLS tests successful!" +echo "All client TLS tests successful!" exit 0 diff --git a/tests/templates/kuttl/tls/test_internal_tls.sh b/tests/templates/kuttl/tls/test_internal_tls.sh deleted file mode 100755 index 03640117..00000000 --- a/tests/templates/kuttl/tls/test_internal_tls.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -# Usage: test_tls.sh namespace - -NAMESPACE=$1 From d6f00ac723141948ba36862b298dc9027bfa12c8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Aug 2022 17:10:00 +0200 Subject: [PATCH 22/37] all tests working --- rust/operator/src/discovery.rs | 4 ++- .../kuttl/tls/20-install-kafka.yaml.j2 | 4 +-- .../kuttl/upgrade/01-install-kafka.yaml.j2 | 28 +++++++++++++++++ .../kuttl/upgrade/02-write-data.yaml.j2 | 19 +++++++----- .../kuttl/upgrade/04-read-data.yaml.j2 | 30 ++++++++----------- tests/test-definition.yaml | 6 ++++ 6 files changed, 64 insertions(+), 27 deletions(-) diff --git a/rust/operator/src/discovery.rs b/rust/operator/src/discovery.rs index 0f87c83d..228c47ff 100644 --- a/rust/operator/src/discovery.rs +++ b/rust/operator/src/discovery.rs @@ -48,7 +48,9 @@ pub async fn build_discovery_configmaps( svc: &Service, ) -> Result, Error> { let name = owner.name(); - let port_name = if kafka.client_tls_secret_class().is_some() { + let port_name = if kafka.client_tls_secret_class().is_some() + || kafka.client_authentication_class().is_some() + { SECURE_CLIENT_PORT_NAME } else { CLIENT_PORT_NAME diff --git a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 index 064675d1..93e8ce34 100644 --- a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 +++ b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 @@ -6,7 +6,7 @@ metadata: spec: clusterRef: name: test-zk -{% if test_scenario['values']['use-client-tls'] == 'true' %} +{% if test_scenario['values']['use-client-auth-tls'] == 'true' %} --- apiVersion: authentication.stackable.tech/v1alpha1 kind: AuthenticationClass @@ -26,7 +26,7 @@ spec: autoTls: ca: secret: - name: secret-provisioner-tls-kafka-client-ca + name: secret-provisioner-tls-kafka-client-auth-ca namespace: default autoGenerate: true {% endif %} diff --git a/tests/templates/kuttl/upgrade/01-install-kafka.yaml.j2 b/tests/templates/kuttl/upgrade/01-install-kafka.yaml.j2 index 362ae0a7..393de25d 100644 --- a/tests/templates/kuttl/upgrade/01-install-kafka.yaml.j2 +++ b/tests/templates/kuttl/upgrade/01-install-kafka.yaml.j2 @@ -2,6 +2,30 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep timeout: 300 +{% if test_scenario['values']['use-client-auth-tls'] == 'true' %} +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: test-kafka-client-auth-tls +spec: + provider: + tls: + clientCertSecretClass: test-kafka-client-auth-tls +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: test-kafka-client-auth-tls +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-kafka-client-auth-ca + namespace: default + autoGenerate: true +{% endif %} --- apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaCluster @@ -16,6 +40,10 @@ spec: secretClass: tls {% else %} tls: null +{% endif %} +{% if test_scenario['values']['use-client-auth-tls'] == 'true' %} + clientAuthentication: + authenticationClass: test-kafka-client-auth-tls {% endif %} brokers: roleGroups: diff --git a/tests/templates/kuttl/upgrade/02-write-data.yaml.j2 b/tests/templates/kuttl/upgrade/02-write-data.yaml.j2 index 730f0117..8d642452 100644 --- a/tests/templates/kuttl/upgrade/02-write-data.yaml.j2 +++ b/tests/templates/kuttl/upgrade/02-write-data.yaml.j2 @@ -16,28 +16,34 @@ spec: command: [sh, -euo, pipefail, -c] args: - | - echo "message written before upgrade" > message -{% if test_scenario['values']['use-client-tls'] == 'true' %} - kcat -b $KAFKA -X security.protocol=SSL -X ssl.key.location=/stackable/certificates/tls.key -X ssl.certificate.location=/stackable/certificates/tls.crt -X ssl.ca.location=/stackable/certificates/ca.crt -t upgrade-test-data -P message +{% if test_scenario['values']['use-client-auth-tls'] == 'true' %} + export SSL_OPTIONS="-X security.protocol=SSL -X ssl.key.location=/stackable/tls_client/tls.key -X ssl.certificate.location=/stackable/tls_client/tls.crt -X ssl.ca.location=/stackable/tls_client/ca.crt" +{% elif test_scenario['values']['use-client-tls'] == 'true' %} + export SSL_OPTIONS="-X security.protocol=SSL -X ssl.ca.location=/stackable/tls_client/ca.crt" {% else %} - kcat -b $KAFKA -t upgrade-test-data -P message + export SSL_OPTIONS="" {% endif %} + echo "message written before upgrade" > message + kcat -b $KAFKA $SSL_OPTIONS -t upgrade-test-data -P message env: - name: KAFKA valueFrom: configMapKeyRef: name: simple-kafka key: KAFKA -{% if test_scenario['values']['use-client-tls'] == 'true' %} volumeMounts: - - mountPath: /stackable/certificates + - mountPath: /stackable/tls_client name: tls volumes: - ephemeral: volumeClaimTemplate: metadata: annotations: +{% if test_scenario['values']['use-client-auth-tls'] == 'true' %} + secrets.stackable.tech/class: test-kafka-client-auth-tls +{% else %} secrets.stackable.tech/class: tls +{% endif %} secrets.stackable.tech/scope: pod,node creationTimestamp: null spec: @@ -49,5 +55,4 @@ spec: storageClassName: secrets.stackable.tech volumeMode: Filesystem name: tls -{% endif %} restartPolicy: Never diff --git a/tests/templates/kuttl/upgrade/04-read-data.yaml.j2 b/tests/templates/kuttl/upgrade/04-read-data.yaml.j2 index 31d967ac..c3b60550 100644 --- a/tests/templates/kuttl/upgrade/04-read-data.yaml.j2 +++ b/tests/templates/kuttl/upgrade/04-read-data.yaml.j2 @@ -16,45 +16,42 @@ spec: command: [sh, -euo, pipefail, -c] args: - | -{% if test_scenario['values']['use-client-tls'] == 'true' %} - echo "message written after upgrade" > message - kcat -b $KAFKA -X security.protocol=SSL -X ssl.key.location=/stackable/certificates/tls.key -X ssl.certificate.location=/stackable/certificates/tls.crt -X ssl.ca.location=/stackable/certificates/ca.crt -t upgrade-test-data -P message - - echo "message written before upgrade" > expected-messages - echo >> expected-messages - cat message >> expected-messages - echo >> expected-messages - kcat -b $KAFKA -X security.protocol=SSL -X ssl.key.location=/stackable/certificates/tls.key -X ssl.certificate.location=/stackable/certificates/tls.crt -X ssl.ca.location=/stackable/certificates/ca.crt -t upgrade-test-data -C -e > read-messages - diff read-messages expected-messages - cmp read-messages expected-messages +{% if test_scenario['values']['use-client-auth-tls'] == 'true' %} + export SSL_OPTIONS="-X security.protocol=SSL -X ssl.key.location=/stackable/tls_client/tls.key -X ssl.certificate.location=/stackable/tls_client/tls.crt -X ssl.ca.location=/stackable/tls_client/ca.crt" +{% elif test_scenario['values']['use-client-tls'] == 'true' %} + export SSL_OPTIONS="-X security.protocol=SSL -X ssl.ca.location=/stackable/tls_client/ca.crt" {% else %} + export SSL_OPTIONS="" +{% endif %} echo "message written after upgrade" > message - kcat -b $KAFKA -t upgrade-test-data -P message + kcat -b $KAFKA $SSL_OPTIONS -t upgrade-test-data -P message echo "message written before upgrade" > expected-messages echo >> expected-messages cat message >> expected-messages echo >> expected-messages - kcat -b $KAFKA -t upgrade-test-data -C -e > read-messages + kcat -b $KAFKA $SSL_OPTIONS -t upgrade-test-data -C -e > read-messages diff read-messages expected-messages cmp read-messages expected-messages -{% endif %} env: - name: KAFKA valueFrom: configMapKeyRef: name: simple-kafka key: KAFKA -{% if test_scenario['values']['use-client-tls'] == 'true' %} volumeMounts: - - mountPath: /stackable/certificates + - mountPath: /stackable/tls_client name: tls volumes: - ephemeral: volumeClaimTemplate: metadata: annotations: +{% if test_scenario['values']['use-client-auth-tls'] == 'true' %} + secrets.stackable.tech/class: test-kafka-client-auth-tls +{% else %} secrets.stackable.tech/class: tls +{% endif %} secrets.stackable.tech/scope: pod,node creationTimestamp: null spec: @@ -66,5 +63,4 @@ spec: storageClassName: secrets.stackable.tech volumeMode: Filesystem name: tls -{% endif %} restartPolicy: Never diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 60050c79..6b7e5805 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -23,6 +23,10 @@ dimensions: values: - "true" - "false" + - name: use-client-auth-tls + values: + - "true" + - "false" - name: use-internal-tls values: - "true" @@ -39,9 +43,11 @@ tests: - upgrade_new - upgrade_old - use-client-tls + - use-client-auth-tls - name: tls dimensions: - kafka - zookeeper - use-client-tls + - use-client-auth-tls - use-internal-tls From f854531efd76e41415199f1f0a2e5f52ed183b7b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Aug 2022 09:02:08 +0200 Subject: [PATCH 23/37] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- tests/templates/kuttl/tls/test_client_auth_tls.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/tls/test_client_auth_tls.sh b/tests/templates/kuttl/tls/test_client_auth_tls.sh index 3607d5dc..db0d5836 100755 --- a/tests/templates/kuttl/tls/test_client_auth_tls.sh +++ b/tests/templates/kuttl/tls/test_client_auth_tls.sh @@ -20,7 +20,7 @@ BAD_TOPIC=$(tr -dc A-Za-z0-9 /tmp/client.config -if /stackable/kafka/bin/kafka-topics.sh --create --topic $TOPIC --bootstrap-server $SERVER --command-config /tmp/client.config +if /stackable/kafka/bin/kafka-topics.sh --create --topic "$TOPIC" --bootstrap-server "$SERVER" --command-config /tmp/client.config then echo "[SUCCESS] Secure client topic created!" else @@ -28,7 +28,7 @@ else exit 1 fi -if /stackable/kafka/bin/kafka-topics.sh --list --topic $TOPIC --bootstrap-server $SERVER --command-config /tmp/client.config | grep $TOPIC +if /stackable/kafka/bin/kafka-topics.sh --list --topic "$TOPIC" --bootstrap-server "$SERVER" --command-config /tmp/client.config | grep "$TOPIC" then echo "[SUCCESS] Secure client topic read!" else @@ -39,7 +39,7 @@ fi ############################################################################ # Test the connection without certificates ############################################################################ -if /stackable/kafka/bin/kafka-topics.sh --create --topic $BAD_TOPIC --bootstrap-server $SERVER &> /dev/null +if /stackable/kafka/bin/kafka-topics.sh --create --topic "$BAD_TOPIC" --bootstrap-server "$SERVER" &> /dev/null then echo "[ERROR] Secure client topic created without certificates!" exit 1 @@ -50,7 +50,7 @@ fi ############################################################################ # Test the connection with bad host name ############################################################################ -if /stackable/kafka/bin/kafka-topics.sh --create --topic $BAD_TOPIC --bootstrap-server localhost:9093 --command-config /tmp/client.config &> /dev/null +if /stackable/kafka/bin/kafka-topics.sh --create --topic "$BAD_TOPIC" --bootstrap-server localhost:9093 --command-config /tmp/client.config &> /dev/null then echo "[ERROR] Secure client topic created with bad host name!" exit 1 From 04a69b3e425278b6131eb352c79c902beeeb8943 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Aug 2022 09:06:05 +0200 Subject: [PATCH 24/37] added quotations to env variables in test script --- tests/templates/kuttl/tls/test_client_tls.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/tls/test_client_tls.sh b/tests/templates/kuttl/tls/test_client_tls.sh index fc7b436f..2428d38c 100755 --- a/tests/templates/kuttl/tls/test_client_tls.sh +++ b/tests/templates/kuttl/tls/test_client_tls.sh @@ -20,7 +20,7 @@ BAD_TOPIC=$(tr -dc A-Za-z0-9 /tmp/client.config -if /stackable/kafka/bin/kafka-topics.sh --create --topic $TOPIC --bootstrap-server $SERVER --command-config /tmp/client.config +if /stackable/kafka/bin/kafka-topics.sh --create --topic "$TOPIC" --bootstrap-server "$SERVER" --command-config /tmp/client.config then echo "[SUCCESS] Secure client topic created!" else @@ -28,7 +28,7 @@ else exit 1 fi -if /stackable/kafka/bin/kafka-topics.sh --list --topic $TOPIC --bootstrap-server $SERVER --command-config /tmp/client.config | grep $TOPIC +if /stackable/kafka/bin/kafka-topics.sh --list --topic "$TOPIC" --bootstrap-server "$SERVER" --command-config /tmp/client.config | grep $TOPIC then echo "[SUCCESS] Secure client topic read!" else @@ -39,7 +39,7 @@ fi ############################################################################ # Test the connection without certificates ############################################################################ -if /stackable/kafka/bin/kafka-topics.sh --create --topic $BAD_TOPIC --bootstrap-server $SERVER &> /dev/null +if /stackable/kafka/bin/kafka-topics.sh --create --topic "$BAD_TOPIC" --bootstrap-server "$SERVER" &> /dev/null then echo "[ERROR] Secure client topic created without certificates!" exit 1 @@ -50,7 +50,7 @@ fi ############################################################################ # Test the connection with bad host name ############################################################################ -if /stackable/kafka/bin/kafka-topics.sh --create --topic $BAD_TOPIC --bootstrap-server localhost:9093 --command-config /tmp/client.config &> /dev/null +if /stackable/kafka/bin/kafka-topics.sh --create --topic "$BAD_TOPIC" --bootstrap-server localhost:9093 --command-config /tmp/client.config &> /dev/null then echo "[ERROR] Secure client topic created with bad host name!" exit 1 From bf08f3438c3fd4035aee45d3e65812a5bd41dd34 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Aug 2022 09:07:07 +0200 Subject: [PATCH 25/37] Update tests/templates/kuttl/tls/test_client_tls.sh Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- tests/templates/kuttl/tls/test_client_tls.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/tls/test_client_tls.sh b/tests/templates/kuttl/tls/test_client_tls.sh index 2428d38c..cc675b0d 100755 --- a/tests/templates/kuttl/tls/test_client_tls.sh +++ b/tests/templates/kuttl/tls/test_client_tls.sh @@ -28,7 +28,7 @@ else exit 1 fi -if /stackable/kafka/bin/kafka-topics.sh --list --topic "$TOPIC" --bootstrap-server "$SERVER" --command-config /tmp/client.config | grep $TOPIC +if /stackable/kafka/bin/kafka-topics.sh --list --topic "$TOPIC" --bootstrap-server "$SERVER" --command-config /tmp/client.config | grep "$TOPIC" then echo "[SUCCESS] Secure client topic read!" else From c097319d6b15a6b21fbb04fa02e0b82c30085c72 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Aug 2022 13:07:02 +0200 Subject: [PATCH 26/37] Apply suggestions from code review Co-authored-by: Sebastian Bernauer --- rust/crd/src/lib.rs | 2 +- rust/crd/src/listener.rs | 4 ++-- rust/operator/src/command.rs | 8 ++++---- tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 | 10 ---------- 4 files changed, 7 insertions(+), 17 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 3215282b..2bc85b87 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -141,7 +141,7 @@ pub struct GlobalKafkaConfig { /// Defaults to `None` #[serde(skip_serializing_if = "Option::is_none")] pub client_authentication: Option, - /// Only affects internal communication. Use mutual verification between Trino nodes + /// Only affects internal communication. Use mutual verification between Kafka nodes /// This setting controls: /// - Which cert the servers should use to authenticate themselves against other servers /// - Which ca.crt to use when validating the other server diff --git a/rust/crd/src/listener.rs b/rust/crd/src/listener.rs index 2f03157b..13b79cde 100644 --- a/rust/crd/src/listener.rs +++ b/rust/crd/src/listener.rs @@ -69,7 +69,7 @@ impl KafkaListenerConfig { pub fn listener_security_protocol_map(&self) -> String { self.listener_security_protocol_map .iter() - .map(|(name, protocol)| format!("{}:{}", name, protocol)) + .map(|(name, protocol)| format!("{name}:{protocol}")) .collect::>() .join(",") } @@ -181,7 +181,7 @@ pub fn get_kafka_listener_config( } fn node_port_cmd(directory: &str, port_name: &str) -> String { - format!("$(cat {}/{}_nodeport)", directory, port_name) + format!("$(cat {directory}/{port_name}_nodeport)") } fn pod_fqdn(kafka: &KafkaCluster, object_name: &str) -> Result { diff --git a/rust/operator/src/command.rs b/rust/operator/src/command.rs index 03360e7d..e971fe67 100644 --- a/rust/operator/src/command.rs +++ b/rust/operator/src/command.rs @@ -74,11 +74,11 @@ fn kcat_client_auth_ssl(cert_directory: &str) -> Vec { "-X".to_string(), "security.protocol=SSL".to_string(), "-X".to_string(), - format!("ssl.key.location={}/tls.key", cert_directory), + format!("ssl.key.location={cert_directory}/tls.key"), "-X".to_string(), - format!("ssl.certificate.location={}/tls.crt", cert_directory), + format!("ssl.certificate.location={cert_directory}/tls.crt"), "-X".to_string(), - format!("ssl.ca.location={}/ca.crt", cert_directory), + format!("ssl.ca.location={cert_directory}/ca.crt"), ] } @@ -87,7 +87,7 @@ fn kcat_client_ssl(cert_directory: &str) -> Vec { "-X".to_string(), "security.protocol=SSL".to_string(), "-X".to_string(), - format!("ssl.ca.location={}/ca.crt", cert_directory), + format!("ssl.ca.location={cert_directory}/ca.crt"), ] } diff --git a/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 b/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 index d2dce8b8..83d9fdc9 100644 --- a/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 @@ -6,17 +6,7 @@ metadata: spec: version: {{ test_scenario['values']['zookeeper'] }} config: -{% if test_scenario['values']['use-client-tls'] == 'true' %} - tls: - secretClass: tls - clientAuthentication: - authenticationClass: zk-client-tls -{% else %} tls: null -{% endif %} -{% if test_scenario['values']['use-internal-tls'] == 'true' %} - internalTlsSecretClass: tls -{% endif %} servers: roleGroups: default: From cae1ba6a57a5b8cb2f1414d47293b853bfc94aa3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Aug 2022 14:02:11 +0200 Subject: [PATCH 27/37] adapted to pr review --- deploy/crd/kafkacluster.crd.yaml | 2 +- deploy/helm/kafka-operator/crds/crds.yaml | 2 +- deploy/manifests/crds.yaml | 2 +- rust/crd/src/lib.rs | 29 ++++++++++++++++++++--- rust/operator/src/command.rs | 16 ++++--------- rust/operator/src/discovery.rs | 12 ++-------- rust/operator/src/kafka_controller.rs | 1 + 7 files changed, 36 insertions(+), 28 deletions(-) diff --git a/deploy/crd/kafkacluster.crd.yaml b/deploy/crd/kafkacluster.crd.yaml index f243a061..5e709021 100644 --- a/deploy/crd/kafkacluster.crd.yaml +++ b/deploy/crd/kafkacluster.crd.yaml @@ -294,7 +294,7 @@ spec: internalTls: default: secretClass: tls - description: "Only affects internal communication. Use mutual verification between Trino nodes This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" + description: "Only affects internal communication. Use mutual verification between Kafka nodes This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" nullable: true properties: secretClass: diff --git a/deploy/helm/kafka-operator/crds/crds.yaml b/deploy/helm/kafka-operator/crds/crds.yaml index 3430144f..e9e2858c 100644 --- a/deploy/helm/kafka-operator/crds/crds.yaml +++ b/deploy/helm/kafka-operator/crds/crds.yaml @@ -296,7 +296,7 @@ spec: internalTls: default: secretClass: tls - description: "Only affects internal communication. Use mutual verification between Trino nodes This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" + description: "Only affects internal communication. Use mutual verification between Kafka nodes This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" nullable: true properties: secretClass: diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index 711e8376..19abd657 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -297,7 +297,7 @@ spec: internalTls: default: secretClass: tls - description: "Only affects internal communication. Use mutual verification between Trino nodes This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" + description: "Only affects internal communication. Use mutual verification between Kafka nodes This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" nullable: true properties: secretClass: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 2bc85b87..38b95e35 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -298,7 +298,7 @@ impl KafkaCluster { .transpose() } - /// Returns the provided docker image e.g. 2.8.1-stackable0.1.0 + /// Returns the provided docker image e.g. 2.8.1-stackable0.1.0. pub fn image_version(&self) -> Result<&str, Error> { self.spec .version @@ -306,7 +306,7 @@ impl KafkaCluster { .context(ObjectHasNoVersionSnafu) } - /// Returns our semver representation for product config e.g. 2.8.1 + /// Returns our semver representation for product config e.g. 2.8.1. pub fn product_version(&self) -> Result<&str, Error> { let image_version = self.image_version()?; image_version @@ -332,11 +332,31 @@ impl KafkaCluster { .map(|tls| tls.authentication_class.as_ref()) } - /// Returns the secret class for internal server encryption + /// Returns the secret class for internal server encryption. pub fn internal_tls_secret_class(&self) -> Option<&TlsSecretClass> { let spec: &KafkaClusterSpec = &self.spec; spec.config.internal_tls.as_ref() } + + /// Returns the client port based on the security (tls) settings. + pub fn client_port(&self) -> u16 { + if self.client_tls_secret_class().is_some() || self.client_authentication_class().is_some() + { + SECURE_CLIENT_PORT + } else { + CLIENT_PORT + } + } + + /// Returns the client port name based on the security (tls) settings. + pub fn client_port_name(&self) -> &str { + if self.client_tls_secret_class().is_some() || self.client_authentication_class().is_some() + { + SECURE_CLIENT_PORT_NAME + } else { + CLIENT_PORT_NAME + } + } } /// Reference to a single `Pod` that is a component of a [`KafkaCluster`] @@ -429,6 +449,9 @@ impl Configuration for KafkaConfig { ); } + // We set either client tls with authentication or client tls without authentication + // If authentication is explicitly required we do not want to have any other CAs to + // be trusted. if resource.client_authentication_class().is_some() { config.insert( CLIENT_AUTH_SSL_KEYSTORE_LOCATION.to_string(), diff --git a/rust/operator/src/command.rs b/rust/operator/src/command.rs index e971fe67..4643277e 100644 --- a/rust/operator/src/command.rs +++ b/rust/operator/src/command.rs @@ -1,7 +1,7 @@ use stackable_kafka_crd::{ - KafkaCluster, CLIENT_PORT, CLIENT_PORT_NAME, SECURE_CLIENT_PORT, SECURE_CLIENT_PORT_NAME, - SSL_STORE_PASSWORD, STACKABLE_DATA_DIR, STACKABLE_TLS_CLIENT_AUTH_DIR, - STACKABLE_TLS_CLIENT_DIR, STACKABLE_TLS_INTERNAL_DIR, STACKABLE_TMP_DIR, + KafkaCluster, CLIENT_PORT, SECURE_CLIENT_PORT, SSL_STORE_PASSWORD, STACKABLE_DATA_DIR, + STACKABLE_TLS_CLIENT_AUTH_DIR, STACKABLE_TLS_CLIENT_DIR, STACKABLE_TLS_INTERNAL_DIR, + STACKABLE_TMP_DIR, }; pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { @@ -38,15 +38,7 @@ pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { } pub fn get_svc_container_cmd_args(kafka: &KafkaCluster) -> String { - let port_name = if kafka.client_tls_secret_class().is_some() - || kafka.client_authentication_class().is_some() - { - SECURE_CLIENT_PORT_NAME - } else { - CLIENT_PORT_NAME - }; - - get_node_port(STACKABLE_TMP_DIR, port_name) + get_node_port(STACKABLE_TMP_DIR, kafka.client_port_name()) } pub fn kcat_container_cmd_args(kafka: &KafkaCluster) -> Vec { diff --git a/rust/operator/src/discovery.rs b/rust/operator/src/discovery.rs index 228c47ff..a1734dce 100644 --- a/rust/operator/src/discovery.rs +++ b/rust/operator/src/discovery.rs @@ -1,9 +1,7 @@ use std::{collections::BTreeSet, convert::TryInto, num::TryFromIntError}; use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_kafka_crd::{ - KafkaCluster, KafkaRole, APP_NAME, CLIENT_PORT_NAME, SECURE_CLIENT_PORT_NAME, -}; +use stackable_kafka_crd::{KafkaCluster, KafkaRole, APP_NAME}; use stackable_operator::{ builder::{ConfigMapBuilder, ObjectMetaBuilder}, k8s_openapi::api::core::v1::{ConfigMap, Endpoints, Service, ServicePort}, @@ -48,13 +46,7 @@ pub async fn build_discovery_configmaps( svc: &Service, ) -> Result, Error> { let name = owner.name(); - let port_name = if kafka.client_tls_secret_class().is_some() - || kafka.client_authentication_class().is_some() - { - SECURE_CLIENT_PORT_NAME - } else { - CLIENT_PORT_NAME - }; + let port_name = kafka.client_port_name(); Ok(vec![ build_discovery_configmap(&name, owner, kafka, service_hosts(svc, port_name)?)?, build_discovery_configmap( diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index 058ebcae..70c84693 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -253,6 +253,7 @@ pub async fn reconcile_kafka(kafka: Arc, ctx: Arc) -> Result< .map(Cow::Borrowed) .unwrap_or_default(); + // TODO: switch to AuthenticationClass::resolve if operator-rs is updated let client_authentication_class = if let Some(auth_class) = kafka.client_authentication_class() { Some( From a18b972d58914766047fd56a3d024601542cc152 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Aug 2022 14:06:36 +0200 Subject: [PATCH 28/37] adapted to pr review --- rust/operator/src/kafka_controller.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rust/operator/src/kafka_controller.rs b/rust/operator/src/kafka_controller.rs index 70c84693..0369f5e1 100644 --- a/rust/operator/src/kafka_controller.rs +++ b/rust/operator/src/kafka_controller.rs @@ -7,6 +7,7 @@ use stackable_kafka_crd::{ METRICS_PORT_NAME, SECURE_CLIENT_PORT, SECURE_CLIENT_PORT_NAME, SERVER_PROPERTIES_FILE, STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_TLS_CLIENT_AUTH_DIR, STACKABLE_TLS_CLIENT_DIR, STACKABLE_TLS_INTERNAL_DIR, STACKABLE_TMP_DIR, + TLS_DEFAULT_SECRET_CLASS, }; use stackable_operator::{ builder::{ @@ -924,11 +925,9 @@ fn container_ports(kafka: &KafkaCluster) -> Vec { } fn create_tls_volume(volume_name: &str, tls_secret_class: Option<&TlsSecretClass>) -> Volume { - let mut secret_class_name = "tls"; - - if let Some(tls) = tls_secret_class { - secret_class_name = &tls.secret_class; - } + let secret_class_name = tls_secret_class + .map(|t| t.secret_class.as_ref()) + .unwrap_or(TLS_DEFAULT_SECRET_CLASS); VolumeBuilder::new(volume_name) .ephemeral( From b64b045978b7fa094434b6880eb35578993eb617 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Aug 2022 14:29:16 +0200 Subject: [PATCH 29/37] added test with wrong certificates when using authenticated client tls. --- rust/operator/src/command.rs | 4 +-- .../kuttl/tls/10-install-zookeeper.yaml.j2 | 25 ------------------ .../kuttl/tls/30-prepare-test-kafka.yaml.j2 | 2 ++ .../kuttl/tls/test_client_auth_tls.sh | 12 +++++++++ tests/templates/kuttl/tls/wrong_keystore.p12 | Bin 0 -> 3573 bytes .../templates/kuttl/tls/wrong_truststore.p12 | Bin 0 -> 1223 bytes 6 files changed, 16 insertions(+), 27 deletions(-) create mode 100644 tests/templates/kuttl/tls/wrong_keystore.p12 create mode 100644 tests/templates/kuttl/tls/wrong_truststore.p12 diff --git a/rust/operator/src/command.rs b/rust/operator/src/command.rs index 4643277e..2cf8b322 100644 --- a/rust/operator/src/command.rs +++ b/rust/operator/src/command.rs @@ -1,7 +1,7 @@ use stackable_kafka_crd::{ KafkaCluster, CLIENT_PORT, SECURE_CLIENT_PORT, SSL_STORE_PASSWORD, STACKABLE_DATA_DIR, STACKABLE_TLS_CLIENT_AUTH_DIR, STACKABLE_TLS_CLIENT_DIR, STACKABLE_TLS_INTERNAL_DIR, - STACKABLE_TMP_DIR, + STACKABLE_TMP_DIR, SYSTEM_TRUST_STORE_DIR, }; pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { @@ -15,7 +15,7 @@ pub fn prepare_container_cmd_args(kafka: &KafkaCluster) -> String { args.extend(chown_and_chmod(STACKABLE_TLS_CLIENT_AUTH_DIR)); } else if kafka.client_tls_secret_class().is_some() { // Copy system truststore to stackable truststore - //args.push(format!("keytool -importkeystore -srckeystore {SYSTEM_TRUST_STORE_DIR} -srcstoretype jks -srcstorepass {SSL_STORE_PASSWORD} -destkeystore {STACKABLE_TLS_CLIENT_DIR}/truststore.p12 -deststoretype pkcs12 -deststorepass {SSL_STORE_PASSWORD} -noprompt")); + args.push(format!("keytool -importkeystore -srckeystore {SYSTEM_TRUST_STORE_DIR} -srcstoretype jks -srcstorepass {SSL_STORE_PASSWORD} -destkeystore {STACKABLE_TLS_CLIENT_DIR}/truststore.p12 -deststoretype pkcs12 -deststorepass {SSL_STORE_PASSWORD} -noprompt")); args.extend(create_key_and_trust_store( STACKABLE_TLS_CLIENT_DIR, "stackable-tls-client-ca-cert", diff --git a/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 b/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 index 83d9fdc9..151ae01d 100644 --- a/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 @@ -11,28 +11,3 @@ spec: roleGroups: default: replicas: 1 - -{% if test_scenario['values']['use-client-tls'] == 'true' %} ---- -apiVersion: authentication.stackable.tech/v1alpha1 -kind: AuthenticationClass -metadata: - name: zk-client-tls -spec: - provider: - tls: - clientCertSecretClass: zk-client-auth-secret ---- -apiVersion: secrets.stackable.tech/v1alpha1 -kind: SecretClass -metadata: - name: zk-client-auth-secret -spec: - backend: - autoTls: - ca: - secret: - name: secret-provisioner-tls-zk-client-ca - namespace: default - autoGenerate: true -{% endif %} diff --git a/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml.j2 b/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml.j2 index 79921ba7..27ac66ab 100644 --- a/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml.j2 +++ b/tests/templates/kuttl/tls/30-prepare-test-kafka.yaml.j2 @@ -4,6 +4,8 @@ kind: TestStep commands: {% if test_scenario['values']['use-client-auth-tls'] == 'true' %} - script: kubectl cp -n $NAMESPACE ./test_client_auth_tls.sh test-kafka-broker-default-0:/tmp + - script: kubectl cp -n $NAMESPACE ./wrong_keystore.p12 test-kafka-broker-default-0:/tmp + - script: kubectl cp -n $NAMESPACE ./wrong_truststore.p12 test-kafka-broker-default-0:/tmp {% elif test_scenario['values']['use-client-tls'] == 'true' %} - script: kubectl cp -n $NAMESPACE ./test_client_tls.sh test-kafka-broker-default-0:/tmp {% endif %} diff --git a/tests/templates/kuttl/tls/test_client_auth_tls.sh b/tests/templates/kuttl/tls/test_client_auth_tls.sh index db0d5836..eece2a5e 100755 --- a/tests/templates/kuttl/tls/test_client_auth_tls.sh +++ b/tests/templates/kuttl/tls/test_client_auth_tls.sh @@ -58,5 +58,17 @@ else echo "[SUCCESS] Secure client topic creation failed with bad host name!" fi +############################################################################ +# Test the connection with bad certificate +############################################################################ +echo $'security.protocol=SSL\nssl.keystore.location=/tmp/wrong_keystore.p12\nssl.keystore.password=changeit\nssl.truststore.location=/tmp/wrong_truststore.p12\nssl.truststore.password=changeit' > /tmp/client.config +if /stackable/kafka/bin/kafka-topics.sh --create --topic "$BAD_TOPIC" --bootstrap-server "$SERVER" --command-config /tmp/client.config &> /dev/null +then + echo "[ERROR] Secure client topic created with wrong certificate!" + exit 1 +else + echo "[SUCCESS] Secure client topic creation failed with wrong certificate!" +fi + echo "All client auth TLS tests successful!" exit 0 diff --git a/tests/templates/kuttl/tls/wrong_keystore.p12 b/tests/templates/kuttl/tls/wrong_keystore.p12 new file mode 100644 index 0000000000000000000000000000000000000000..98df5a84ee55eaba98e488f47dbfe28c52c04587 GIT binary patch literal 3573 zcmVD zv+*qVuFB-&XLp82Z4!)Gp^EZT1Hl_b#q#N(qu2HdZ=+@REVA3Do{Y7XC2w$sWqEMU zPdIx70OksqigO}W5fCaUcC;f#ly0WHowj{JE@n<++|Y*|Lz&b@e_~;YDVpxGz?K#u zR9!QrBX>RK+AN&GZ-^=0|JIM*=}@etKn#q5O$z71_$G$I&N_935K)9)W0qPnYU5Cz2KyA_vK^+>jX76k z(1l&=2*4At&;H%zrgyxRFdTAjV}NlR%0gHHdKpor=B^3cVR@z9m|%F!*NoH=CQeFr zvx_Pqp$f0q8V*3J=)DH~Z)NPFAV4;(#S+GQ(7g`GJe&PX5U&;_14;$tJVs^{Bc{%- ze+H+Et;*h$?4|6dXaTJmR`-&(#aI$rU}}7C5U9+?IPU#_Az$t830cIBH*!1yb#)D* z2AdKvbXc^T%Qsa3)lNm?{BD>|vbJx7Tay!zBKatXxFRuYa(Icp=^cA)2zL4`-4 ztghjdvsnvjvZ)^5E7E}ET2rwCpN14ldM1BOC$vgQeI?K%#tydfsJ8xrv~fbsrBX-= z%th;D8@B*>!-0Feivg^H7D#l>$%L*PfJeP2X3;}`A^;op&T}Y5gu&oh^abtHRX#?q z(K657d8%fcs-x^xNPl)PscsUPG0B~!HVc|Av5(bJU$}Kt|1Y_E5{G-%TUhc_Lpsv^ z#%!EbY)wqx5XjNsP+bjs8qw$74tsB0I*X6AKacZN=U-<`nK_w~&)_otm#z%>yKrEi ztvrjRg*aOqoSuJflA~ss%$W0NF$-wI{Gal5#DaxN=!O3icFp#|gPE_ng+}L;RH}~v zhVIqgAfg;>27!QVc*~(S0y_E-(l&in+b}9JB;C;!V=r_@y$f26X^|sXC%sPs&vq<+ zxYV0zwp~B9E#nXAV!b_0kZ{h*VhDO0NS-G>>^7&@z%O2T`NX|mlz0{dBnhc zz08#~vrP!LyN`Bea=&t-I65cdOMQ1u{N);@KI8pH(jwn)&12+Z$+AO@+fdx`_?`EUmCgMS)t!8E9Cd{Z{ zIo_2Ct(fg|*#llF3q<{kG8OgnjHfk}Bt<zFm?vlU3Kv0SX^xAQce%yn(Z+)@ftwrJ@swDvY;V1OKAQ9Ddnmib5uIscqYb^ zbz2ofn*M;=qsr6>{H?}4L5rXS57W6&nae;Npy>@~){@1C#30(~-@mvJmDSUmT%~9+ zphfbYJYoFsX2VtaC1JoBK0^;%i^}i+LRI&b7VdW-%`Y9-q>0c^cMW6Y8yRW9sUewW z7j%>MwGARzQN$IgcE>96|H?O^od{q-s(V9z*w*m-0b&=6m->!gVr& zJ7H;MM7>{3sLk7K%ySB50p5rbgcGbtK5`B*LCClk=UBgo_U3(>g?!K#6iiPT0)_5M z=kUhuuZ$t$U^P4WAUb#tflomg!bD3CfMwS4zBlSZcpi<`PL55=`?rhPs$;AW>+3ez zd^(>1Y_dIHG$hUHuBkM&K{pO>u>b?gn2CqEAGj(=2kZ?`=SepJ*W`z8;GXkp`?>TZ z$eBqVO56IhUy2pes*ERHds&kw-Y+2VKHZpC5V10BMewl^8Q`5_a8Pu@T(L&_JaCm1 z16ewwIGTDDi?aq-h?M41an5fUxUeLz3bYS6wf$Nc!VxBDkk|;w!qT%P>g7tsV1nY! z6E?pRzG=uI~tl<3+G>qeKnoT@{t`Vs{6{)_-@Jlqt1d)nTHxPIzB=?Eu?lDqR$^ z0GTs`F{7{>!mM5{Eo+YANgB4`1I1CDA>t#71ojCqCQ(=Xisx{EsI6k1S_VkK8EE<) zsD)ns8&UgV&~G(&LNQUl8)qw2zNFFm&Kwx9jTd?YIR-mg_xoi0 z3Da6XzF!tZ8D_bc-ub7RM2SaJoC`p4TfmJBBMCd}?%qPUH_?w%k?om~{;UBhQA*PU zX_984|0r6wI1mR13>$WRWI!MpN_oL8Vc=c!O9lyZGt+u@W*bv**!O!7YXfsChVP>_v^#+E2^PHlz$3I0w zLEVH+mNSc9Q%P$XwA#PrD`<@FjU7WeBIdG!hm`6!=qWh`3}EYH8~hCVkix>@w+p)xep<+K_#h&}KpjfCbyK zfJH?8s|r_=NEvN|QzRiD-R&THy}BHkdpI)$PW3pGq9qn^06j{qk-o*ktF6=$lWy&} za+o;W>2PMX8lk9^u7Gk4n%pCdM^ZDER;=T{j31NW0IyE0+Tx{9Y%`*y^0V%jr;oRs zg=R!0EPnR*l>oRFr0G^I4qlNbTZ@34WZA)aeG1Ku2>9g(a!+B)c!SJ``K1q8n$73i8v>QbM_hIYfWy^2Y0!e#MY zFeG-|Q}fQRV;b2kck{GK_YTsvp+P5_hjf(gsa$uMDSc!<6+;|Tetd#{AvBzT9=wZV zh^t!CXcu8nM(h))V`>~IZ$PH)*DGnvN^fX*Jo_{Qhw)zHd=vKF5g5BLMCpT(go9nB zRm23d&;1qK9!;;b4jh;Xxm`^Qua_-GJ1zKR=zOPG>Zx{%{bj zYv=ZG=$R&!_?m);WR^#T+-UD59u|juvTKtdB}$iIonVvxfK!Fdl?9n9`FN&O@eDd- z(9*G29Bk-;ZSHd2PG`}xTh*+n9m*X`z9=B*m1>p$x(Oetuv8VY-h~H$fQMmqo4)8C zJ}>u>bE2}LJ5a6ouGec>CH&W3p`WuIjb(yRu@(_@QPeMpJ^L(E4I5+tziUH&!C2&b zAQ^-7NGO-eLA3+6U`DI3J$+D9Rc+5~O!`K$+-q0o2-_8-68l!&dK(|ELYPMzmMb+! zW7OX{_=h3uTv|AgD4wnJsw2tGNCq4QL>W4bvA%8u11V17-ghb^?Wa$t4AJ`vek#Vn7XGA75-u4c{-DV`t^E7*WH+6WZKPvsN-k79<@iE}J8MD> z?4V5(4*xMFFe3&DDuzgg_YDCF6)_eB6!WIWjy?br>eY0eM<4~ANgtq8AMq<>pcVrhT-zKf4FQf0s;sCxaphq literal 0 HcmV?d00001 diff --git a/tests/templates/kuttl/tls/wrong_truststore.p12 b/tests/templates/kuttl/tls/wrong_truststore.p12 new file mode 100644 index 0000000000000000000000000000000000000000..db2170cbcede2ef5730d1c9e0ce0a435afd20124 GIT binary patch literal 1223 zcmV;&1UUOJf&{|?0Ru3C1bhYwDuzgg_YDCD0ic2eZ3Kb@X)uBWWiWySVFn2*hDe6@ z4FLxRpn?QaFoFb50s#Opf&@nf2`Yw2hW8Bt2LUi<1_>&LNQU+thDZTr0|Wso1Q2o{5X2fN@mdfH^o0pLnvH;h1JIe2f}bVK&rzr8N(%8!!y6#=O1B@>M5O~)fvZuDhY6Mr7$q1*0w8^=G6+Zos-OgG)}(%|9u)&H z^}=Xed(*C{dWWTS=m^VG>J}+L2K>PV-r%aUpAbbzwT(zN?>V;>no{)X|Is{?@S>q_ z@{||Ad-v5)SpY$%`gV=z#|@Dr*lq8dcDU8~xe7rQI*ch6W5sdTn8N${uI3N>uC2z3 zJwC~h{*QZBf#T{Eg!=L8X>p&IU@&SQ$H+0X1vAD$X?{}oX-Gux5s_qnNTk?(oG&k& zy#w!swr_IDssS1G6jv{e*LHT6OH%>?Lt9*Z&;3}tY8`X7*70Yf8wKlVw0L!eEb-Ly zeeA%W=321`0fc4VdX^=bHeMyK?riwa+W^q)VPBIaME}B)Q$+#EQ4$Mvi+iwgv(BXj zi}P_5O=4Pmsc$RmpxiJV&_sf2Zrg2GZ%|pSe=#w=^ML@l;D?QNkU#Bz@*8b>Mmp4$E{xTAD7v!nYBg9V@@R{l)?t))zz!9_HrKy6o_PS^u%0{(1$M7hfz>M zto5{bIbfV-l3#UwsKqbfjzMxpAfpGK<~{hHM0-kewdYuH)P`6~7YqBZu<491rabdo z4+#}=p`T{$1^QS^szdQ`+@ga_mX7MF)sXG-Bo1EUd`q=j>u-@NQYVb${C+MXH{&T` z8-K}?k$GIpiA{~JqA#kFR63W!&A+(tPFymLwT7MXyIkY1}8~M>^yN8vGBbV6I zOS1Br71YK=G?#2#S=ljN!Hqblln5|BFd;Ar1_dh)0|FWa00b0E4lzcAU}3-L-^3ug l6FHr{&{NR_6r%6ndc!NZq%zoYylq_MG0_Ijf&v2phM)r-D;od+ literal 0 HcmV?d00001 From fb0689808328667221d9dd39b3d4dccc3526cab2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Aug 2022 15:14:40 +0200 Subject: [PATCH 30/37] fix typo in comment --- rust/crd/src/listener.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crd/src/listener.rs b/rust/crd/src/listener.rs index 13b79cde..abb00fd8 100644 --- a/rust/crd/src/listener.rs +++ b/rust/crd/src/listener.rs @@ -19,7 +19,7 @@ pub enum KafkaListenerError { #[derive(strum::Display, Debug, EnumString)] pub enum KafkaListenerProtocol { - /// Unencrypted and authenticated HTTP connections + /// Unencrypted and unauthenticated HTTP connections #[strum(serialize = "PLAINTEXT")] Plaintext, /// Encrypted and server-authenticated HTTPS connections From 789327e55f8b5ce37fffc167b6573e035620b300 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Aug 2022 16:04:46 +0200 Subject: [PATCH 31/37] Update tests/templates/kuttl/tls/20-install-kafka.yaml.j2 Co-authored-by: Sebastian Bernauer --- tests/templates/kuttl/tls/20-install-kafka.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 index 93e8ce34..6952f3f7 100644 --- a/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 +++ b/tests/templates/kuttl/tls/20-install-kafka.yaml.j2 @@ -63,6 +63,8 @@ spec: {% if test_scenario['values']['use-client-auth-tls'] == 'true' %} clientAuthentication: authenticationClass: test-kafka-client-auth-tls +{% else %} + clientAuthentication: null {% endif %} {% if test_scenario['values']['use-internal-tls'] == 'true' %} internalTls: From ec3edf837b34e526f063343d1591bc33389822a3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Aug 2022 11:24:55 +0200 Subject: [PATCH 32/37] set wrong key/truststore cert enddate 100y --- tests/templates/kuttl/tls/wrong_keystore.p12 | Bin 3573 -> 3533 bytes .../templates/kuttl/tls/wrong_truststore.p12 | Bin 1223 -> 1351 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/tests/templates/kuttl/tls/wrong_keystore.p12 b/tests/templates/kuttl/tls/wrong_keystore.p12 index 98df5a84ee55eaba98e488f47dbfe28c52c04587..e5dc3a42ed431b53b82ca1d0e197610bbdad685b 100644 GIT binary patch delta 3474 zcmV;D4Q=xE8_gR)FoF%q0s#Xsf(?%b2`Yw2hW8Bt2LYgh4S)oK4SX2N!?wn6-r-*jxW(9pCs_f{Mj>1zM1&XzJ^)YUR5C*{eA?M)dHWTmI=ERURH80f`UN4YizX8ab-HDQ`IjnDoMtPsxW}Jxu4x9=m>skcdpzjcn zRD)~anK=o68I*LYl0``O2Rj42gSV}JI=xMzL6cXj#Ig2vXkc2?=U{K2_<8SYJ?%-V zpCInZd}IAbK5e$9E1}c4UhT7Q(|G2|w*r=*HD#76a7%-Bat(Hf9y@vWCdX7_o$5WW zBWEboQ##-UM58v9$Y#EBa<$aVV7RV+LymqnHF|`r29+`_pk>gqrsM)2VJz)`%FPNM zr=L>yKlYh;m{WRoRv%Y@Uysa;L-qTu{-D4+z~E3EB4MIXn7~7Y@Y>^~USrH@MP*?| zT#sz-v09+-zs5tWGQJg;N24y#f3|R1_)pF%u-$j`G`8;x%CUrZdx(-jj&%8(y%lDn* zz~H&uotJNSv^|lO)&csuL{h}L>_whv%-d0`6Xk&O&vnQYZ-|)84;LMBQK89N%K7na z>d?Z+fjEN0A9l4yG!`4Mp%HP%ZXWRSYpeSdad15#0h)deUO+Cw8t|ciCffjQ?fIoJ zkAB-DJ95#JbiJ$;N%M0M0|#s-|ALJ|7)(l;I@5<6Mr~SU!^GB=93;alib38vtHd41I1uO|0Qmo<}6c}@-$oN~~_`fY(H zYrNKstm!p;L5jU|nXS8jO}o3)3}uVAe`f3tlyKu$EW!A{cVzS5y%Y-(yW9$SssaI6 zBdw0Q6WV9VdhSU!iw@M1w?Lb6aq{M-d3Rb zwUGPh7L3I|&-JtY*sxLWan#735Ak9)Te(=Ym(gaZ>dkLMQYo5$`L`Jsfk?4q6SNNM zKFVf8afAI4cX-_3zm^E2?eLuw4}q5vT+yntHYoGaQ7-w^m z>V2W}2#$h41|39y=Gu|aYpDFXXR*GlL#3=xG*$?n()|r zDAW!f=qd2IaeWTIPqXEz!G*0?g|k0Rd@SncSm0YEKSIUTM8dUqyaSoV7N=qNK0)_) ze44kRdmMq~kTIr)gt`VSi=;g0K`+0@TT!v3pX7@IAY`YI zc<`pyPCvbWUAo(%8is&>Z-uiXfl$G=;<^kEM6mTNlIzcnw7&P{xAv46&OC zR%gnqA>gS3*5rFWzR6I%p>oRRJ!1RbOTDp~a*XEf4q(6j zI~3J5T4^VaZKBawImkkyCK~hQ*cb>fe-z+cFGkzq2iieBz2qMDUY-q*_Vp7WT+>v4 zD_VcA_!Xwj2VKw+j~UF&Qw+O-9+@JN{|Z$>PV78nVR!74*qJt9 zIsWW(5TW4Up8yKdTNNNzVO4$n|Mm5DinXg!l9ysT@x4@SZj#71t#iV?Uf{ip{#G~B zYWldb?s4clHbH%ODQ{kX#>NRp*#0Dc5zw)w;J|(LS%zkljZyM1bvZ~$J z3RHWgu&}SMIYK+0^`W!jvhiBWK8Nau=*_g@+QQsJirQZCQt2B7=XU06T7i2?QT`VQ zCTSBy!X9~_zayR0D9GUDZRXd?HN=%^x*hfsM{@Czh{FvIr8$TD$JauCeV29!X5KfA zm@U5BlYIzAe_+QnPPnHAL;?Z`00e>r$ZcPr!1dU&hy@D%cokwC9~g&1_0UsGAjL%I zfg^_l@J^M1jke%4&*vnOzF0~hRs{&^miz0GXBTp#f9fT{*)&dye3nl^)J}IZ)w^47 z8#S0Pj=Pl2VdBdE&Ah3x1aDyi6~eBFV@a&Wfwsn075jNhqOSMNr)u@gspCPxE!+<~ zXR^pkx#d)syQjixXHBKe)`7lSPMl$+`gc_dvF?$$Ecl>H-2 z#Ob6`a3KqT0CG6~TmgMb;q!o^n+pLdxQGZ1e@E!GcW=B0oQy8f>Y^jzB;SJGp=g6e z(=U=wM$C`-@zC}21Ed=Jf>aAm?NbBrsIOR^5iX%M?mHQF*Yy)5HA_+Ls1>eDw-!gl z%Zj{S8vHsHKuyOfcjSi?(YAWfl7Si=$2urMrOB&ujfdNqSpNj;*p03unMiAs^ORfA ze^NRwuL$W$U;+&X$`&`mn6#Rj<1S_>$@{|Q=qB9FW{T}~WTl-jBT=OlXdMYN=$A3J z*D^W}?9Sit7NMz47NymoI>dk=S1WftwA=3h7ZefOHHTd4fp6H{4U#$F)!CI@^Sa7( zfgJTg-+}nU3`y?Jf-|AHC&uYX=F%MDe}O(#S-)16eV%W?3GIWIjd>Ft&}Mj@wPeROn-PXX{FBnUD2)1K|0HEUk1SuPeOS5WZu>hctjen6N}qYC>bQ^!N=EMv zGFC9;{1`axwL9$qG})#2Uu<{Vt~>tvrk~ZpLL(01Rzc-3vfzQ?JkmSd<=$*CyR&a* z6-zh7(eYu);kix|Y&ji#)Qn=ee|x2D_9K^VP$^Gwr@67|xVYKQ^$!m1$VyuJNhm!I zUw8W{LAS{cHn-~GnNK6(K9>Q#=W7wM%PYUo?7f^dfBOhmv5mce$m7j3B6j|8VA^At z4(6GFZ7ep)}-TPeMD!NOqUFTnTw zUs&cwBOA8av-Lct*w}S+Va)*>e3$tFex{)VR+xsz{GaHDFF<_QF`(-p)ZVt?X=elx zZ2T8;f={TeQc2sf;9)W%x z7whWb>yr10`;svwFe3&DDuzgg_YDCF6)_eB6x>WwUomPTOXM3>JeYKY9snL*J;X3E zFd;Ar1_dh)0|FWa00b0mhg#8{4q7V(gFeq090MS Aga7~l delta 3514 zcmV;r4MpDv+*qVuFB-&XLp82Z4!)Gp^EZT z1Hl_b#q#N(qu2HdZ=+@REVA3Do{Y7XC2w$sWqEMUPdIx70OksqigO}W5fCaUcC;gZ zMwD)*yq&gvK`v%aW8Bb(9YdMaMt@>qi7A@yvcQ%WAXHs5r6YGe=GrWr!EcBu-v8E* z-sw=Rr9cdfflUhM!T2VI!Q?){ceTqkjoX=O2|r>}g7?)Aob%=I<@_PT8!vcGKVyKZ z>}O@UkHR25TS=XON|IgI^8h|GgB(GBVzXrzOnNbwsFR#PSR8h_-z@!7VGsMZ^tuh<$6K&t3}y$1Ym zW$dCLKsKz!62^Pby$;AcoBc}=uNEW&N(JORMrIQurp~T^2B(Xy%HEUgrR=9@0j(KU z_ma28SQ1)bYJ6`HsLaMV?)`ruU+wM*S;USvay$Wbbq%5hn-VZ|ShSnVH&p=DPDSGU zZkSH8wr_)5lM|34`6!3DCqaLIRuYa(Icp=^cA)2zL4`-4tghjdvsnvjvZ)^5E7E}E zT2rwCpN14ldM1BOC$vgQeI?K%#tydfsJ8xrv~fbsrBX-=%th;D8@B*>!-0Feivg^H z7D#l>$%L*PfJeP2X3;}`A^;op&T}Y5gu&oh^abtHRX#?q(K657d8%fAo2sMiRY-q! zFsW`5nK8+or8WziF0qf*QD3-qRR1rzdJ>0w)>~NeQ$sq^{l;vZRcuX6-w?>r;80x+ zdm7Q_-41(iTsn)7v_FsYROer3Oqn^ElF#5W{+F%{_`7gmpRGKLrG+?K8=RhhZjz&B znar5;XfX?D!u+4|b;N>yg-Ym!{}Xo2_Q8XhuepUr=af{cj{t`5)!rbY9Bc-GfNXfn zp*8|K`Vi7KeO22qDl#P9(G_DabVj`kT8(LuBUmTBPXf<&EPlAun`yRPKea9659wmP zJx!2s&dOsUjcy!U-eS5Q1=?d^CR@$8C+4h}SLhk74{j;l3$7V|YwthAjt!B^$Fs50 zVs$foy+S+FG1FI?u^r~!JvXb2ohjmTnL`=S++EtYHk!FIC zWau?d^b4Y21M7d^OanIt0NO0P*MS96cJF4R;3nchq^)LU58;~aF*WfTK_ubd&bA4I)_N zg?!K#6iiPT0)_5M=kUhuuZ$t$ zU^P4WAUb#tflomg!bD3CfMwS4zBlSZcpi<`PL55=`?rhPs$;AW>+3ezd^(>1Y_dIH zG$hUHuBkM&K{pO>u>b?gn2CqEAGj(=2kZ@hPv=QD0N3P)Zs4BtYWunLBgmOa9!lH# zv|ox9)2fUoU3*!RC*Ch0@IKv`R}ir>Y(?;~5gFi}VsKD&!d$UN`aE!z6a!g0qd1y+ z6^pY5SBRA6QgP028Mv?{uL`seIJNy+7{U=IXpqgPEjF>2>^3Jc5^{@M#JMbcZC!2Pz?EPr)L8Lg(uqS7Bt@^JbI zHOA2*ctf2v#t zMW;0Ys{#TD00e>r$cL}J+VG=R@r4*&PORtXBIw`00wpscY`pa|R7`HX&gPPz00VK4 zvG}MFej1A|M7C%gn!`IOpXDd{`q|y4$e;gRFjTd?YIR-mg_xoi03Da6XzF!tZ8D_bc-ub7R zM2SaJoC`p4TfmJBBMCd}?%qPUH_?w%k?om~{;UBhQA*PUX_984|0r6wI1mR13>$WR zWI!MpN_oL8Vc=c!O9lyZGt+u@W*bv**!O5^+{a5M zB86%es-QiveZ1>dW}a7)L})Wc^I>MSyY&W!fAgH8w8uY1LqXkyO_no@UQT`E$VrbeOY8? z)lk3KuZK<^bOIViE!6~KBlM4*X_H!xdAq-!@KhJt*WhK_>AS=VS=5z!MO>lP+`*c9wC>3`M0nZw$UeV5Q?LJ5Ec+p>T~e?9g(a!+B)c!SJ``K1q8n$73i8v>QbM_hIYfWy^2Y0!e#MYFeG-|Q}fQRV;b2k zck{GK_YTsvp+P5_hjf(gsa$uMDSc!<6+;|Tetd#{AvBzTe;&MxV~DF-(r6c9P)6(% zsAFmzC~rWf?bj=5%t~))cs%lm38Hh0T=(nJW2srd06^I%Lq&u~!^y=z(qS za@|g6(X(6Ctf(Ez9ZS9_An289mH)a4AE~fZ6|&xi2Y-NvVRf6n=pH^V_mFd5y*5p+@1FNi(+EK>~|e`El^YeRm)Smb*k8H4mlD3{7X zwF9`bM(cYggw8+ZCe{`&Qk08y~Jhm`59yD>X-B)ZaPyhau}+ zS~!p>o~`q$BgxH31{?)M89I%zzHS5qDNft z4AJ`vek#Vn7XGA75-u4c{-DV`t^E7*WH+6WZKPvsN-k79<@iE}J8MD>?4V5(4*xMF zFe3&DDuzgg_YDCF6)_eB6!WIWjy?b1m!S-1mOk= zDuzgg_YDCD2B3lj(lCMq&H@1dFoFcfkw7aHGfBLj$f*7}ZzT6@W-kiX75`hqk$)tA ziDlE! zP&RN``{Er6tP?TAxx>-Sb7>PUU&K=s3Td45sm$|c!H66lOCl!1tgg72Ui4%SB-C}d zBfurk-dC%-b{a;wZlbK^0;O`Kyb{zZBK##Gj2IMuW90 zp?6EC9Ds!cwgXIWtJ#47`xZa7EQW?6Y>Uku%ikwhEd)9NWhb^; zbxiC+|LXA%Vg}v7sDT+n6B`tA%|SLGlst}fM?!>%vn_IjlF^L{9XJeTBO9yKeLH$C zl+Ao#Acq=h+X+GupGMM)J%2)ffzRGw8gEsh`^;(j|A~DKefo82V;9(-^pJDsPtJNN zw3z3fn@wH*C74?RF$>8iep&||k5V9}aW6@;)1>4baV*Uz9CA;r2q``T2xGeiOzJ+I z2N@r9{dw|nywVpM7SOBydRYn2mEehn|3~p{#|SGpI0w?YH|QE{*>MDatAK6KOLyaD zRe45s(jKh8b?6*!wIGPhT=7KgC0R>~KqAxp`9W8H8+ZTrtFeZfS}%==%%Tm=v>&;h z1x`XBoyf|xZdt#Yi0!u1Dq*Wxj0F5}<#Y{>wt*eeW^V^e7{5U_uA}2HbHZr>>J#|| zindm7m}y%bTu_A&&tO-7EuF1$>NO-??aQdp7aA>q<7YZwZ2Fck9MR}epb`{b<5~R4 zKF##q$1`kfW`QzG*(2WHZ2>!m^N;dgs{A1Vn4bWC9|HprxSCC50i-Yah0R_tyd4SI z+EZyhFxn_O1{GqcdZz|09+pkXLWW^fA8Ch)^gfTsuyGiY{Rl)Z9nI^Pa4{ z&e)Zs_am`TY%S$iqGH0n(R)I8(aN~}OZ<2?=h7y}Vwwv73CDJ0gN?2(8gF}zKMn83 z2x-`?Xncmh$>w&7bwe;ky>5RTnOCL*W!2efdZ{d!PR9u1^6fO8Z@YFV^F2tqDeh?< z*buov&xgR~lzfqYnY{Csmxd!hnHgph18Bir;`aD0BKX=ho9FyHy z!OpanA(1=FigtJhjhW#8FX-0s!0d*FoX#{-)uFw$)ZQM5!tI;OCFQI~d~UnQD-Oy2 zIUkHQg)MQjU2C|s&8|IZ&RRtKt!a0ks#gLLvVy&U=I8@|51SeUg{g)Ysg3VTEfXK| z2wB`Z1dUx0wRu{UMV1@&y39xjG?DD23T`8f(A2Q}hnQ?AoG?W?`d{m5f3*F@tneaY zK~P9ec<}rek&6&V7Ujff_E9%+iAxbaGaeIjb@v&{GlgTE!<-q5Fg`FLFbM_)D-Ht! z8U+9Z6kbOypX?OG9#1N&oNE*O0Vf_m`uhYFZyB0L)gNPcq`A<+0#=r}rZ$qO0s{et Epc~p=x&QzG delta 1165 zcmV;81akYw3dadRFoFcb0s#Xsf&_d92`Yw2hW8Bt2LYgh1Z@O@1ZgmW1Z6OS1YrgV zDuzgg_YDCD2B3ljQZRx9P67b{FoFa}kw7aHnSV4$XoLW5_^ai%=m%@+Y_9>yk$)tA zav%`I8YuBv5DN5#2|SvOfPw?inUsQ`CCtxJr|C)x@mPV^J4|1zd*pd`0_RmT^rjr3 zE#X@Dl3*C+u$T3GBMC&*iCs!$Zqzs5Oz&J*l;~c+UEE>|*R%#Hha9;E@)ptdjz|om zGyO&4;|cFh%Cs`RbQbOX5gEPby(yD_vlchH2eE3auhY;=V?K&`+Up}H7Z07AQpUo4 zj@K*rbL~!8^OG-Bd)Gk7{L7mUx?8!X6_ALv&v}O!o+x~cN+?Ws9Pp40n`AM=8zA*c zw;$9*r2|)it5J@J36>5RB^X5lAbqMb2uKC0pag5yq<*a)6$3E!!f0H3)2^t0dWWTS z=m^VG>J}+L2K>PV-r%aUpAbbzwT(zN?>V;>no{)X|Is{?@S>q_@{||Ad-v5)SpY$% z`gV=z#|@Dr*lq8dcDU8~xe7rQI*ch6W5sdTn8N${uI3N>uC2z3JwC~h{*QZBf#T{E zg!=L8X>p&IU@&SQ$H+0X1vAEfL1}(c_i0E(?-7w?e@LX*eVi{ZoV^3@g|=^U$*KVv z^%Pexjn{T|mP=Ct0Yh6{eb4<^yJ{VCw$|}yqZ|tM%B}D(il2b(i$x#vub&Gqja;QXVInR;kb-@i zr=;lh;DglXIaXdhIR-D}T6ZfrFiM)JMmp4 z$E{xTAD7v!nYBg9V@@R{l)?t))zz!9_HrKy6o_PS^u%0{(1$M7hfz>Mto5{bIbfV- zl3#UwsKqbfjzMxpAfpGK<~{hHM0-kewdYuH)P`6~7YqBZu<491rabdo4+#}=p`T{$ z1^QS^szdQ`+@gbjOqP!7snw9}@+1yk;(SZBTI+9-DpDtm)Z|1jfM&RTyMpE Date: Fri, 5 Aug 2022 11:47:07 +0200 Subject: [PATCH 33/37] added authentication class to roles --- deploy/helm/kafka-operator/templates/roles.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deploy/helm/kafka-operator/templates/roles.yaml b/deploy/helm/kafka-operator/templates/roles.yaml index 65cbdba6..3a7205b7 100644 --- a/deploy/helm/kafka-operator/templates/roles.yaml +++ b/deploy/helm/kafka-operator/templates/roles.yaml @@ -89,3 +89,11 @@ rules: - {{ include "operator.name" . }}clusters/status verbs: - patch + - apiGroups: + - authentication.stackable.tech + resources: + - authenticationclasses + verbs: + - get + - list + - watch From 54e4e7036cb5551c1b5fc7d393ed7cfabfce4a4f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Aug 2022 11:47:16 +0200 Subject: [PATCH 34/37] reduced number of tls tests --- tests/test-definition.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 6b7e5805..5df23d98 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -11,6 +11,9 @@ dimensions: - 3.6.3-stackable0.7.1 - 3.7.0-stackable0.7.1 - 3.8.0-stackable0.7.1 + - name: zookeeper_latest + values: + - 3.8.0-stackable0.7.1 - name: upgrade_old values: - 2.7.1-stackable0 @@ -47,7 +50,7 @@ tests: - name: tls dimensions: - kafka - - zookeeper + - zookeeper_latest - use-client-tls - use-client-auth-tls - use-internal-tls From 8695185ae6fb15f0210aa891bb83254da4e7f106 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Aug 2022 11:48:44 +0200 Subject: [PATCH 35/37] fixed zookeeper latest --- tests/test-definition.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 5df23d98..8c9af1b1 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -11,7 +11,7 @@ dimensions: - 3.6.3-stackable0.7.1 - 3.7.0-stackable0.7.1 - 3.8.0-stackable0.7.1 - - name: zookeeper_latest + - name: zookeeper-latest values: - 3.8.0-stackable0.7.1 - name: upgrade_old @@ -50,7 +50,7 @@ tests: - name: tls dimensions: - kafka - - zookeeper_latest + - zookeeper-latest - use-client-tls - use-client-auth-tls - use-internal-tls From 6534debb1549ea46817c536b00fe3fdd7a374595 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Aug 2022 11:53:40 +0200 Subject: [PATCH 36/37] regenerated charts --- deploy/manifests/roles.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deploy/manifests/roles.yaml b/deploy/manifests/roles.yaml index 77c8d30a..fa65d3f5 100644 --- a/deploy/manifests/roles.yaml +++ b/deploy/manifests/roles.yaml @@ -89,3 +89,11 @@ rules: - kafkaclusters/status verbs: - patch + - apiGroups: + - authentication.stackable.tech + resources: + - authenticationclasses + verbs: + - get + - list + - watch From 2846b8348159600684460ea22e6354f33f89fa05 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 5 Aug 2022 13:14:50 +0200 Subject: [PATCH 37/37] Fix scenario name in tls test --- tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 b/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 index 151ae01d..72acb99a 100644 --- a/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/tls/10-install-zookeeper.yaml.j2 @@ -4,7 +4,7 @@ kind: ZookeeperCluster metadata: name: test-zk spec: - version: {{ test_scenario['values']['zookeeper'] }} + version: {{ test_scenario['values']['zookeeper-latest'] }} config: tls: null servers: