percona · hors · Mar 26, 2024 · Nov 29, 2023 · Jan 30, 2024 · Mar 7, 2024
@@ -1371,3 +1371,68 @@ deploy_pmm_server() {
 
 	retry 10 60 helm install monitoring --set imageTag=${IMAGE_PMM_SERVER#*:} --set imageRepo=${IMAGE_PMM_SERVER%:*} $additional_params https://percona-charts.storage.googleapis.com/pmm-server-$PMM_SERVER_VER.tgz
 }
+
+renew-certificate() {
+	certificate="$1"
+
+	desc "renew $certificate"
+
+	local pod_name
+	pod_name=$(kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}')
+
+	local revision
+	revision=$(kubectl_bin get certificate "$certificate" -o 'jsonpath={.status.revision}')
+
+	kubectl_bin exec "$pod_name" -- /tmp/cmctl renew "$certificate"
+
+	# wait for new revision
+	for i in {1..10}; do
+		local new_revision
+		new_revision=$(kubectl_bin get certificate "$certificate" -o 'jsonpath={.status.revision}')
+		if [ "$((revision + 1))" == "$new_revision" ]; then
+			break
+		fi
+		sleep 1
+	done
+}
+
+deploy_cmctl() {
+	local service_account="cmctl"
+
+	$sed -e "s/percona-server-mongodb-operator/$service_account/g" "${src_dir}/deploy/rbac.yaml" \
+		| yq '(select(.rules).rules[] | select(contains({"apiGroups": ["cert-manager.io"]}))).resources += "certificates/status"' \
+		| kubectl_bin apply -f -
+	kubectl_bin apply -f "$conf_dir/cmctl.yml"
+}
+
+stop_cluster() {
+	local cluster_name=$1
+	local max_wait_time=${2:-120}
+
+	local passed_time=0
+	local sleep_time=1
+	kubectl_bin patch psmdb "${cluster_name}" --type json -p='[{"op":"add","path":"/spec/pause","value":true}]'
+	set +x
+	echo -n 'Waiting for cluster stop'
+	until [[ $(kubectl_bin get psmdb "${cluster_name}" -o jsonpath='{.status.mongos.ready}') -le 0 ]] \
+		&& [[ $(kubectl_bin get deployment "${cluster_name}-mongos" -o jsonpath='{.status.replicas}') -le 0 ]] \
+		&& [[ $(kubectl_bin get psmdb "${cluster_name}" -o jsonpath='{.status.replsets.cfg.ready}') -le 0 ]] \
+		&& [[ $(kubectl_bin get psmdb "${cluster_name}" -o jsonpath='{.status.replsets.rs0.ready}') -le 0 ]]; do
+		echo -n .
+		passed_time=$((passed_time + sleep_time))
+		sleep ${passed_time}
+		if [[ ${passed_time} -gt ${max_wait_time} ]]; then
+			echo "We've been waiting for cluster stop for too long. Exiting..."
+			exit 1
+		fi
+	done
+	echo
+	set -x
+}
+
+start_cluster() {
+	local cluster_name=$1
+
+	kubectl_bin patch psmdb "${cluster_name}" --type json -p='[{"op":"add","path":"/spec/pause","value":false}]'
+	wait_cluster_consistency "${cluster_name}"
+}
@@ -14,6 +14,6 @@ security-context
 self-healing-chaos
 smart-update
 upgrade-consistency
-upgrade-consistency-sharded
+upgrade-consistency-sharded-tls
 users
 version-service
@@ -39,7 +39,7 @@ storage
 tls-issue-cert-manager
 upgrade
 upgrade-consistency
-upgrade-consistency-sharded
+upgrade-consistency-sharded-tls
 upgrade-sharded
 users
 version-service
@@ -40,7 +40,7 @@ storage
 tls-issue-cert-manager
 upgrade
 upgrade-consistency
-upgrade-consistency-sharded
+upgrade-consistency-sharded-tls
 upgrade-sharded
 users
 version-service
@@ -6,30 +6,6 @@ test_dir=$(realpath $(dirname $0))
 . "${test_dir}/../functions"
 set_debug
 
-renew-certificate() {
-	certificate="$1"
-
-	desc "renew $certificate"
-
-	local pod_name
-	pod_name=$(kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}')
-
-	local revision
-	revision=$(kubectl_bin get certificate "$certificate" -o 'jsonpath={.status.revision}')
-
-	kubectl_bin exec "$pod_name" -- /tmp/cmctl renew "$certificate"
-
-	# wait for new revision
-	for i in {1..10}; do
-		local new_revision
-		new_revision=$(kubectl_bin get certificate "$certificate" -o 'jsonpath={.status.revision}')
-		if [ "$((revision + 1))" == "$new_revision" ]; then
-			break
-		fi
-		sleep 1
-	done
-}
-
 check_tls_secret() {
 	local secret_name=$1
 	check_secret_data_key "$secret_name" 'ca.crt'
@@ -48,15 +24,6 @@ check_secret_data_key() {
 	fi
 }
 
-deploy_cmctl() {
-	local service_account="cmctl"
-
-	$sed -e "s/percona-server-mongodb-operator/$service_account/g" "${src_dir}/deploy/rbac.yaml" \
-		| yq '(select(.rules).rules[] | select(contains({"apiGroups": ["cert-manager.io"]}))).resources += "certificates/status"' \
-		| kubectl_bin apply -f -
-	kubectl_bin apply -f "$conf_dir/cmctl.yml"
-}
-
 main() {
 	create_infra "$namespace"
 	deploy_cert_manager

@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   annotations: {}
-  generation: 2
+  generation: 8
   labels:
     app.kubernetes.io/component: mongod
     app.kubernetes.io/instance: some-name

@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   annotations: {}
-  generation: 2
+  generation: 8
   labels:
     app.kubernetes.io/component: cfg
     app.kubernetes.io/instance: some-name

@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   annotations: {}
-  generation: 2
+  generation: 10
   labels:
     app.kubernetes.io/component: mongod
     app.kubernetes.io/instance: some-name

@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   annotations: {}
-  generation: 2
+  generation: 10
   labels:
     app.kubernetes.io/component: cfg
     app.kubernetes.io/instance: some-name

@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   annotations: {}
-  generation: 2
+  generation: 9
   labels:
     app.kubernetes.io/component: mongod
     app.kubernetes.io/instance: some-name

@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   annotations: {}
-  generation: 2
+  generation: 9
   labels:
     app.kubernetes.io/component: mongod
     app.kubernetes.io/instance: some-name

@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   annotations: {}
-  generation: 2
+  generation: 11
   labels:
     app.kubernetes.io/component: mongod
     app.kubernetes.io/instance: some-name

@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   annotations: {}
-  generation: 2
+  generation: 11
   labels:
     app.kubernetes.io/component: mongod
     app.kubernetes.io/instance: some-name

@@ -2,27 +2,39 @@
 
 set -o errexit
 
-test_dir=$(realpath $(dirname $0))
-. ${test_dir}/../functions
+test_dir=$(realpath "$(dirname "$0")")
+. "${test_dir}/../functions"
 set_debug
 
 CLUSTER='some-name'
 
+wait_cluster() {
+	wait_for_running ${CLUSTER}-rs0 3
+	wait_for_running ${CLUSTER}-cfg 3
+	wait_for_running ${CLUSTER}-mongos 3
+	wait_cluster_consistency ${CLUSTER}
+}
+
 main() {
-	create_infra $namespace
+	create_infra "$namespace"
+	deploy_cert_manager
 
 	desc 'create secrets and start client'
-	kubectl_bin apply -f "${conf_dir}/client.yml" \
-		-f "${conf_dir}/secrets.yml"
+	kubectl_bin apply -f "$conf_dir/secrets.yml"
+	kubectl_bin apply -f "$conf_dir/client_with_tls.yml"
+	deploy_cmctl
 
 	desc "create first PSMDB cluster 1.14.0 $CLUSTER"
 	apply_cluster "$test_dir/conf/${CLUSTER}.yml"
 
 	desc 'check if Pod started'
-	wait_for_running ${CLUSTER}-rs0 3
-	wait_for_running ${CLUSTER}-cfg 3
-	wait_for_running ${CLUSTER}-mongos 3
-	wait_cluster_consistency ${CLUSTER}
+	wait_cluster
+
+	# TODO: uncomment when 1.14.0 will be removed,
+	#       renewal doesn't work on "1.14.0" version
+	#renew-certificate "some-name-ssl"
+	#renew-certificate "some-name-ssl-internal"
+	#wait_cluster
 
 	desc 'check if service and statefulset created with expected config'
 	compare_kubectl service/${CLUSTER}-rs0 "-1140"
@@ -31,15 +43,29 @@ main() {
 	compare_kubectl statefulset/${CLUSTER}-cfg "-1140"
 
 	desc 'test 1.15.0'
+	# workaround to switch to updated certificate structure
+	# more details: https://github.com/percona/percona-server-mongodb-operator/pull/1287
+	# TODO: remove the workaround when 1.14.0 will be removed
+	stop_cluster $CLUSTER
 	kubectl_bin patch psmdb "${CLUSTER}" --type=merge --patch '{
         "spec": {"crVersion":"1.15.0"}
     }'
+	kubectl_bin delete secret "$CLUSTER"-ssl "$CLUSTER"-ssl-internal
+	kubectl_bin delete certificate "$CLUSTER"-ssl "$CLUSTER"-ssl-internal
+	kubectl_bin delete issuer "$CLUSTER-psmdb-ca"
+	start_cluster $CLUSTER
+
 	# Wait for at least one reconciliation
-	sleep 10
+	sleep 20
 	desc 'check if Pod started'
-	wait_for_running "${CLUSTER}-cfg" 3
-	wait_for_running "${CLUSTER}-rs0" 3
-	wait_for_running "${CLUSTER}-mongos" 3
+	wait_cluster
+
+	renew-certificate "some-name-ssl"
+	sleep 20
+	wait_cluster
+	renew-certificate "some-name-ssl-internal"
+	sleep 20
+	wait_cluster
 
 	desc 'check if service and statefulset created with expected config'
 	compare_kubectl service/${CLUSTER}-rs0 "-1150"
@@ -52,19 +78,24 @@ main() {
         "spec": {"crVersion":"1.16.0"}
     }'
 	# Wait for at least one reconciliation
-	sleep 10
+	sleep 20
 	desc 'check if Pod started'
-	wait_for_running "${CLUSTER}-cfg" 3
-	wait_for_running "${CLUSTER}-rs0" 3
-	wait_for_running "${CLUSTER}-mongos" 3
+	wait_cluster
+
+	renew-certificate "some-name-ssl"
+	sleep 20
+	wait_cluster
+	renew-certificate "some-name-ssl-internal"
+	sleep 20
+	wait_cluster
 
 	desc 'check if service and statefulset created with expected config'
 	compare_kubectl service/${CLUSTER}-rs0 "-1160"
 	compare_kubectl service/${CLUSTER}-cfg "-1160"
 	compare_kubectl statefulset/${CLUSTER}-rs0 "-1160"
 	compare_kubectl statefulset/${CLUSTER}-cfg "-1160"
 
-	destroy $namespace
+	destroy "$namespace"
 
 	desc 'test passed'
 }