Skip to content

Commit 07886d3

Browse files
inelpandzichors
inelpandzic
and
hors
authored
K8SPSMDB-913: Fix getting mongos host when Expose is LoadBalancer (percona#1292)
* Fix getting mongos host when Expose is LoadBalancer. * Delete mongos sts without deleting its services. Fix mongos ready status setting. * Properly cleanup mongos services when needed. * fix demand-backup-sharded test * logs * Update geting mongos status. * Cleanup logs. * Update test. * Test exported mongos service endpoint withing demand-backup-physical-sharded test. --------- Co-authored-by: Viacheslav Sarzhan <[email protected]>
1 parent e74b37e commit 07886d3

File tree

6 files changed

+72
-40
lines changed

6 files changed

+72
-40
lines changed

‎e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ spec:
6262
cpu: 100m
6363
memory: 0.1G
6464
expose:
65-
exposeType: ClusterIP
65+
exposeType: LoadBalancer
6666
configsvrReplSet:
6767
affinity:
6868
antiAffinityTopologyKey: none

‎e2e-tests/demand-backup-physical-sharded/run‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,15 @@ run_recovery_check() {
4848
set -o xtrace
4949
}
5050

51+
check_exported_mongos_service_endpoint() {
52+
local host=$1
53+
54+
if [ "$host" != "$(kubectl_bin get psmdb $cluster -o=jsonpath='{.status.host}')" ]; then
55+
echo "Exported host is not correct after the restore"
56+
exit 1
57+
fi
58+
}
59+
5160
create_infra "${namespace}"
5261

5362
deploy_minio
@@ -68,6 +77,13 @@ wait_for_running ${cluster}-cfg 3
6877
wait_for_running ${cluster}-mongos 3
6978
wait_cluster_consistency ${cluster}
7079

80+
lbEndpoint=$(kubectl_bin get svc $cluster-mongos -o=jsonpath='{.status}' \
81+
| jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0].ip')
82+
if [ -z $lbEndpoint ]; then
83+
echo "mongos service not exported correctly"
84+
exit 1
85+
fi
86+
7187
run_mongos \
7288
'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' \
7389
"userAdmin:userAdmin123456@${cluster}-mongos.${namespace}"
@@ -102,18 +118,21 @@ if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
102118
echo 'check backup and restore -- aws-s3'
103119
run_restore ${backup_name_aws} "_restore_sharded"
104120
run_recovery_check ${backup_name_aws} "_restore_sharded"
121+
check_exported_mongos_service_endpoint "$lbEndpoint"
105122

106123
echo "drop collection"
107124
run_mongos 'use myApp\n db.test.drop()' "myApp:myPass@${cluster}-mongos.${namespace}"
108125
echo 'check backup and restore -- gcp-cs'
109126
run_restore ${backup_name_gcp} "_restore_sharded"
110127
run_recovery_check ${backup_name_gcp} "_restore_sharded"
128+
check_exported_mongos_service_endpoint "$lbEndpoint"
111129

112130
echo "drop collection"
113131
run_mongos 'use myApp\n db.test.drop()' "myApp:myPass@${cluster}-mongos.${namespace}"
114132
echo 'check backup and restore -- azure-blob'
115133
run_restore ${backup_name_azure} "_restore_sharded"
116134
run_recovery_check ${backup_name_azure} "_restore_sharded"
135+
check_exported_mongos_service_endpoint "$lbEndpoint"
117136
fi
118137

119138
echo "drop collection"

‎e2e-tests/demand-backup-sharded/run‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
131131
insert_data "100501"
132132
check_data "-2nd"
133133
run_restore "$backup_name_gcp"
134-
wait_restore "$backup_name_aws" "$cluster"
134+
wait_restore "$backup_name_gcp" "$cluster"
135135
check_data
136136

137137
desc 'check backup and restore -- azure-blob'
@@ -142,7 +142,7 @@ if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
142142
insert_data "100501"
143143
check_data "-2nd"
144144
run_restore "$backup_name_azure"
145-
wait_restore "$backup_name_aws" "$cluster"
145+
wait_restore "$backup_name_azure" "$cluster"
146146
check_data
147147
fi
148148

@@ -155,7 +155,7 @@ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \
155155
insert_data "100501"
156156
check_data "-2nd"
157157
run_restore "$backup_name_minio"
158-
wait_restore "$backup_name_aws" "$cluster"
158+
wait_restore "$backup_name_minio" "$cluster"
159159
check_data
160160

161161
desc 'delete backup and check if it is removed from bucket -- minio'

‎pkg/controller/perconaservermongodb/psmdb_controller.go‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -864,29 +864,18 @@ func (r *ReconcilePerconaServerMongoDB) upgradeFCVIfNeeded(ctx context.Context,
864864
}
865865

866866
func (r *ReconcilePerconaServerMongoDB) deleteMongos(ctx context.Context, cr *api.PerconaServerMongoDB) error {
867-
svcList, err := psmdb.GetMongosServices(ctx, r.client, cr)
868-
if err != nil {
869-
return errors.Wrap(err, "failed to list mongos services")
870-
}
871-
872867
var mongos client.Object
873868
if cr.CompareVersion("1.12.0") >= 0 {
874869
mongos = psmdb.MongosStatefulset(cr)
875870
} else {
876871
mongos = psmdb.MongosDeployment(cr)
877872
}
878-
err = r.client.Delete(ctx, mongos)
873+
874+
err := r.client.Delete(ctx, mongos)
879875
if err != nil && !k8serrors.IsNotFound(err) {
880876
return errors.Wrap(err, "failed to delete mongos statefulset")
881877
}
882878

883-
for _, svc := range svcList.Items {
884-
err = r.client.Delete(ctx, &svc)
885-
if err != nil && !k8serrors.IsNotFound(err) {
886-
return errors.Wrap(err, "failed to delete mongos services")
887-
}
888-
}
889-
890879
return nil
891880
}
892881

@@ -904,6 +893,18 @@ func (r *ReconcilePerconaServerMongoDB) deleteMongosIfNeeded(ctx context.Context
904893
return nil
905894
}
906895

896+
ss, err := psmdb.GetMongosServices(ctx, r.client, cr)
897+
if err != nil {
898+
return errors.Wrap(err, "failed to list mongos services")
899+
}
900+
901+
for _, svc := range ss.Items {
902+
err = r.client.Delete(ctx, &svc)
903+
if err != nil && !k8serrors.IsNotFound(err) {
904+
return errors.Wrap(err, "failed to delete mongos services")
905+
}
906+
}
907+
907908
return r.deleteMongos(ctx, cr)
908909
}
909910

‎pkg/controller/perconaservermongodb/status.go‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,7 @@ func (r *ReconcilePerconaServerMongoDB) updateStatus(ctx context.Context, cr *ap
204204
}
205205
case !inProgress && replsetsReady == len(repls) && clusterState == api.AppStateReady && cr.Status.Host != "":
206206
state = api.AppStateReady
207+
207208
if cr.Spec.Sharding.Enabled && cr.Status.Mongos.Status != api.AppStateReady {
208209
state = cr.Status.Mongos.Status
209210
}
@@ -307,15 +308,26 @@ func (r *ReconcilePerconaServerMongoDB) rsStatus(ctx context.Context, cr *api.Pe
307308
}
308309

309310
func (r *ReconcilePerconaServerMongoDB) mongosStatus(ctx context.Context, cr *api.PerconaServerMongoDB) (api.MongosStatus, error) {
311+
status := api.MongosStatus{
312+
Status: api.AppStateInit,
313+
}
314+
315+
sts := psmdb.MongosStatefulset(cr)
316+
err := r.client.Get(ctx, types.NamespacedName{Name: sts.Name, Namespace: sts.Namespace}, sts)
317+
if err != nil && k8serrors.IsNotFound(err) {
318+
return status, nil
319+
}
320+
if err != nil && !k8serrors.IsNotFound(err) {
321+
return api.MongosStatus{}, errors.Wrapf(err, "get statefulset %s", sts.Name)
322+
}
323+
310324
list, err := r.getMongosPods(ctx, cr)
311325
if err != nil {
326+
312327
return api.MongosStatus{}, fmt.Errorf("get list: %v", err)
313328
}
314329

315-
status := api.MongosStatus{
316-
Size: len(list.Items),
317-
Status: api.AppStateInit,
318-
}
330+
status.Size = len(list.Items)
319331

320332
for _, pod := range list.Items {
321333
for _, cntr := range pod.Status.ContainerStatuses {
@@ -344,7 +356,7 @@ func (r *ReconcilePerconaServerMongoDB) mongosStatus(ctx context.Context, cr *ap
344356
status.Status = api.AppStateStopping
345357
case cr.Spec.Pause:
346358
status.Status = api.AppStatePaused
347-
case status.Size == status.Ready:
359+
case status.Size > 0 && status.Size == status.Ready:
348360
status.Status = api.AppStateReady
349361
}
350362

‎pkg/psmdb/service.go‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ func GetServiceAddr(ctx context.Context, svc corev1.Service, pod corev1.Pod, cl
145145
}
146146

147147
case corev1.ServiceTypeLoadBalancer:
148-
host, err := getIngressPoint(ctx, pod, cl)
148+
host, err := getIngressPoint(ctx, cl, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace})
149149
if err != nil {
150150
return nil, errors.Wrap(err, "get ingress endpoint")
151151
}
@@ -171,7 +171,7 @@ func GetServiceAddr(ctx context.Context, svc corev1.Service, pod corev1.Pod, cl
171171

172172
var ErrNoIngressPoints = errors.New("ingress points not found")
173173

174-
func getIngressPoint(ctx context.Context, pod corev1.Pod, cl client.Client) (string, error) {
174+
func getIngressPoint(ctx context.Context, cl client.Client, serviceNN types.NamespacedName) (string, error) {
175175
var retries uint64 = 0
176176

177177
var ip string
@@ -188,7 +188,7 @@ func getIngressPoint(ctx context.Context, pod corev1.Pod, cl client.Client) (str
188188
}
189189

190190
svc := &corev1.Service{}
191-
err := cl.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, svc)
191+
err := cl.Get(ctx, serviceNN, svc)
192192
if err != nil {
193193
return "", errors.Wrap(err, "failed to fetch service")
194194
}
@@ -302,12 +302,14 @@ func MongosHost(ctx context.Context, cl client.Client, cr *api.PerconaServerMong
302302
if cr.Spec.Sharding.Mongos.Expose.ServicePerPod {
303303
svcName = pod.Name
304304
}
305+
306+
nn := types.NamespacedName{
307+
Namespace: cr.Namespace,
308+
Name: svcName,
309+
}
310+
305311
svc := new(corev1.Service)
306-
err := cl.Get(ctx,
307-
types.NamespacedName{
308-
Namespace: cr.Namespace,
309-
Name: svcName,
310-
}, svc)
312+
err := cl.Get(ctx, nn, svc)
311313
if err != nil {
312314
if k8serrors.IsNotFound(err) {
313315
return "", nil
@@ -316,18 +318,16 @@ func MongosHost(ctx context.Context, cl client.Client, cr *api.PerconaServerMong
316318
return "", errors.Wrap(err, "failed to get mongos service")
317319
}
318320

319-
var host string
320-
if mongos := cr.Spec.Sharding.Mongos; mongos.Expose.ExposeType == corev1.ServiceTypeLoadBalancer {
321-
for _, i := range svc.Status.LoadBalancer.Ingress {
322-
host = i.IP
323-
if len(i.Hostname) > 0 {
324-
host = i.Hostname
325-
}
321+
mongos := cr.Spec.Sharding.Mongos
322+
if mongos.Expose.ExposeType == corev1.ServiceTypeLoadBalancer && svc.Spec.Type == corev1.ServiceTypeLoadBalancer {
323+
host, err := getIngressPoint(ctx, cl, nn)
324+
if err != nil {
325+
return "", errors.Wrap(err, "get ingress endpoint")
326326
}
327-
} else {
328-
host = svc.Name + "." + cr.Namespace + "." + cr.Spec.ClusterServiceDNSSuffix
327+
return host, nil
329328
}
330-
return host, nil
329+
330+
return svc.Name + "." + cr.Namespace + "." + cr.Spec.ClusterServiceDNSSuffix, nil
331331
}
332332

333333
func getExtAddr(ctx context.Context, cl client.Client, namespace string, pod corev1.Pod) (string, error) {

0 commit comments

Comments
 (0)