Enable frontend NLB

Author: wweiwei-li

Diff for: controllers/ingress/group_controller.go

@@ -151,7 +151,7 @@ func (r *groupReconciler) reconcile(ctx context.Context, req reconcile.Request)
 		return errmetrics.NewErrorWithMetrics(controllerName, "add_group_finalizer_error", err, r.metricsCollector)
 	}
 
-	_, lb, err := r.buildAndDeployModel(ctx, ingGroup)
+	_, lb, frontendNlb, err := r.buildAndDeployModel(ctx, ingGroup)
 	if err != nil {
 		return err
 	}
@@ -164,7 +164,14 @@ func (r *groupReconciler) reconcile(ctx context.Context, req reconcile.Request)
 			if statusErr != nil {
 				return
 			}
-			statusErr = r.updateIngressGroupStatus(ctx, ingGroup, lbDNS)
+			var frontendNlbDNS string
+			if frontendNlb != nil {
+				frontendNlbDNS, statusErr = frontendNlb.DNSName().Resolve(ctx)
+				if statusErr != nil {
+					return
+				}
+			}
+			statusErr = r.updateIngressGroupStatus(ctx, ingGroup, lbDNS, frontendNlbDNS)
 			if statusErr != nil {
 				r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedUpdateStatus,
 					fmt.Sprintf("Failed update status due to %v", statusErr))
@@ -191,38 +198,40 @@ func (r *groupReconciler) reconcile(ctx context.Context, req reconcile.Request)
 	return nil
 }
 
-func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingress.Group) (core.Stack, *elbv2model.LoadBalancer, error) {
+func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingress.Group) (core.Stack, *elbv2model.LoadBalancer, *elbv2model.LoadBalancer, error) {
 	var stack core.Stack
 	var lb *elbv2model.LoadBalancer
 	var secrets []types.NamespacedName
 	var backendSGRequired bool
 	var err error
+	var frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState
+	var frontendNlb *elbv2model.LoadBalancer
 	buildModelFn := func() {
-		stack, lb, secrets, backendSGRequired, err = r.modelBuilder.Build(ctx, ingGroup, r.metricsCollector)
+		stack, lb, secrets, backendSGRequired, frontendNlbTargetGroupDesiredState, frontendNlb, err = r.modelBuilder.Build(ctx, ingGroup, r.metricsCollector)
 	}
 	r.metricsCollector.ObserveControllerReconcileLatency(controllerName, "build_model", buildModelFn)
 	if err != nil {
 		r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
-		return nil, nil, errmetrics.NewErrorWithMetrics(controllerName, "build_model_error", err, r.metricsCollector)
+		return nil, nil, nil, errmetrics.NewErrorWithMetrics(controllerName, "build_model_error", err, r.metricsCollector)
 	}
 	stackJSON, err := r.stackMarshaller.Marshal(stack)
 	if err != nil {
 		r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	r.logger.Info("successfully built model", "model", stackJSON)
 
 	deployModelFn := func() {
-		err = r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "ingress")
+		err = r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "ingress", frontendNlbTargetGroupDesiredState)
 	}
 	r.metricsCollector.ObserveControllerReconcileLatency(controllerName, "deploy_model", deployModelFn)
 	if err != nil {
 		var requeueNeededAfter *runtime.RequeueNeededAfter
 		if errors.As(err, &requeueNeededAfter) {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		r.recordIngressGroupEvent(ctx, ingGroup, corev1.EventTypeWarning, k8s.IngressEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %v", err))
-		return nil, nil, errmetrics.NewErrorWithMetrics(controllerName, "deploy_model_error", err, r.metricsCollector)
+		return nil, nil, nil, errmetrics.NewErrorWithMetrics(controllerName, "deploy_model_error", err, r.metricsCollector)
 	}
 	r.logger.Info("successfully deployed model", "ingressGroup", ingGroup.ID)
 	r.secretsManager.MonitorSecrets(ingGroup.ID.String(), secrets)
@@ -232,9 +241,9 @@ func (r *groupReconciler) buildAndDeployModel(ctx context.Context, ingGroup ingr
 		inactiveResources = append(inactiveResources, k8s.ToSliceOfNamespacedNames(ingGroup.Members)...)
 	}
 	if err := r.backendSGProvider.Release(ctx, networkingpkg.ResourceTypeIngress, inactiveResources); err != nil {
-		return nil, nil, errmetrics.NewErrorWithMetrics(controllerName, "release_auto_generated_backend_sg_error", err, r.metricsCollector)
+		return nil, nil, nil, errmetrics.NewErrorWithMetrics(controllerName, "release_auto_generated_backend_sg_error", err, r.metricsCollector)
 	}
-	return stack, lb, nil
+	return stack, lb, frontendNlb, nil
 }
 
 func (r *groupReconciler) recordIngressGroupEvent(_ context.Context, ingGroup ingress.Group, eventType string, reason string, message string) {
@@ -243,16 +252,16 @@ func (r *groupReconciler) recordIngressGroupEvent(_ context.Context, ingGroup in
 	}
 }
 
-func (r *groupReconciler) updateIngressGroupStatus(ctx context.Context, ingGroup ingress.Group, lbDNS string) error {
+func (r *groupReconciler) updateIngressGroupStatus(ctx context.Context, ingGroup ingress.Group, lbDNS string, frontendNLBDNS string) error {
 	for _, member := range ingGroup.Members {
-		if err := r.updateIngressStatus(ctx, lbDNS, member.Ing); err != nil {
+		if err := r.updateIngressStatus(ctx, lbDNS, frontendNLBDNS, member.Ing); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (r *groupReconciler) updateIngressStatus(ctx context.Context, lbDNS string, ing *networking.Ingress) error {
+func (r *groupReconciler) updateIngressStatus(ctx context.Context, lbDNS string, frontendNLBDNS string, ing *networking.Ingress) error {
 	if len(ing.Status.LoadBalancer.Ingress) != 1 ||
 		ing.Status.LoadBalancer.Ingress[0].IP != "" ||
 		ing.Status.LoadBalancer.Ingress[0].Hostname != lbDNS {
@@ -262,6 +271,11 @@ func (r *groupReconciler) updateIngressStatus(ctx context.Context, lbDNS string,
 				Hostname: lbDNS,
 			},
 		}
+		if frontendNLBDNS != "" {
+			ing.Status.LoadBalancer.Ingress = append(ing.Status.LoadBalancer.Ingress, networking.IngressLoadBalancerIngress{
+				Hostname: frontendNLBDNS,
+			})
+		}
 		if err := r.k8sClient.Status().Patch(ctx, ing, client.MergeFrom(ingOld)); err != nil {
 			return errors.Wrapf(err, "failed to update ingress status: %v", k8s.NamespacedName(ing))
 		}

Diff for: controllers/service/service_controller.go

@@ -152,7 +152,8 @@ func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service)
 }
 
 func (r *serviceReconciler) deployModel(ctx context.Context, svc *corev1.Service, stack core.Stack) error {
-	if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "service"); err != nil {
+	albTargetGroupDesiredState := core.NewFrontendNlbTargetGroupDesiredState()
+	if err := r.stackDeployer.Deploy(ctx, stack, r.metricsCollector, "service", albTargetGroupDesiredState); err != nil {
 		var requeueNeededAfter *runtime.RequeueNeededAfter
 		if errors.As(err, &requeueNeededAfter) {
 			return err

Diff for: pkg/annotations/constants.go

@@ -59,6 +59,19 @@ const (
 	IngressLBSuffixMultiClusterTargetGroup       = "multi-cluster-target-group"
 	IngressSuffixLoadBalancerCapacityReservation = "minimum-load-balancer-capacity"
 	IngressSuffixIPAMIPv4PoolId                  = "ipam-ipv4-pool-id"
+	IngressSuffixEnableFrontendNlb                             = "enable-frontend-nlb"
+	IngressSuffixFrontendNlbScheme                             = "frontend-nlb-scheme"
+	IngressSuffixFrontendNlbSubnets                            = "frontend-nlb-subnets"
+	IngressSuffixFrontendNlbSecurityGroups                     = "frontend-nlb-security-groups"
+	IngressSuffixFrontendNlbListenerPortMapping                = "frontend-nlb-listener-port-mapping"
+	IngressSuffixFrontendNlbHealthCheckPort                    = "frontend-nlb-healthcheck-port"
+	IngressSuffixFrontendNlbHealthCheckProtocol                = "frontend-nlb-healthcheck-protocol"
+	IngressSuffixFrontendNlbHealthCheckPath                    = "frontend-nlb-healthcheck-path"
+	IngressSuffixFrontendNlbHealthCheckIntervalSeconds         = "frontend-nlb-healthcheck-interval-seconds"
+	IngressSuffixFrontendNlbHealthCheckTimeoutSeconds          = "frontend-nlb-healthcheck-timeout-seconds"
+	IngressSuffixFrontendNlbHealthCheckHealthyThresholdCount   = "frontend-nlb-healthcheck-healthy-threshold-count"
+	IngressSuffixFrontendNlHealthCheckbUnhealthyThresholdCount = "frontend-nlb-healthcheck-unhealthy-threshold-count"
+	IngressSuffixFrontendNlbHealthCheckSuccessCodes            = "frontend-nlb-healthcheck-success-codes"
 
 	// NLB annotation suffixes
 	// prefixes service.beta.kubernetes.io, service.kubernetes.io

Diff for: pkg/deploy/elbv2/frontend_nlb_target_synthesizer.go

@@ -0,0 +1,169 @@
+package elbv2
+
+import (
+	"context"
+
+	awssdk "github.com/aws/aws-sdk-go-v2/aws"
+	elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
+	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/model/core"
+	elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func NewFrontendNlbTargetSynthesizer(k8sClient client.Client, trackingProvider tracking.Provider, taggingManager TaggingManager, frontendNlbTargetsManager FrontendNlbTargetsManager, logger logr.Logger, featureGates config.FeatureGates, stack core.Stack, frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState) *frontendNlbTargetSynthesizer {
+	return &frontendNlbTargetSynthesizer{
+		k8sClient:                          k8sClient,
+		trackingProvider:                   trackingProvider,
+		taggingManager:                     taggingManager,
+		frontendNlbTargetsManager:          frontendNlbTargetsManager,
+		featureGates:                       featureGates,
+		logger:                             logger,
+		stack:                              stack,
+		frontendNlbTargetGroupDesiredState: frontendNlbTargetGroupDesiredState,
+		unmatchedResTGs:                    nil,
+	}
+}
+
+type frontendNlbTargetSynthesizer struct {
+	k8sClient                          client.Client
+	trackingProvider                   tracking.Provider
+	taggingManager                     TaggingManager
+	frontendNlbTargetsManager          FrontendNlbTargetsManager
+	featureGates                       config.FeatureGates
+	logger                             logr.Logger
+	stack                              core.Stack
+	frontendNlbTargetGroupDesiredState *core.FrontendNlbTargetGroupDesiredState
+	unmatchedResTGs                    []*elbv2model.TargetGroup
+}
+
+// Synthesize processes AWS target groups and deregisters ALB targets based on the desired state.
+func (s *frontendNlbTargetSynthesizer) Synthesize(ctx context.Context) error {
+	var resTGs []*elbv2model.TargetGroup
+	s.stack.ListResources(&resTGs)
+	sdkTGs, err := s.findSDKTargetGroups(ctx)
+	if err != nil {
+		return err
+	}
+	_, _, unmatchedSDKTGs, err := matchResAndSDKTargetGroups(resTGs, sdkTGs,
+		s.trackingProvider.ResourceIDTagKey(), s.featureGates)
+	if err != nil {
+		return err
+	}
+
+	for _, sdkTG := range unmatchedSDKTGs {
+		if sdkTG.TargetGroup.TargetType != "alb" {
+			continue
+		}
+
+		err := s.deregisterCurrentTarget(ctx, sdkTG)
+		if err != nil {
+			return errors.Wrapf(err, "failed to deregister target for the target group: %s", *sdkTG.TargetGroup.TargetGroupArn)
+		}
+	}
+
+	return nil
+
+}
+
+func (s *frontendNlbTargetSynthesizer) deregisterCurrentTarget(ctx context.Context, sdkTG TargetGroupWithTags) error {
+	// Retrieve the current targets for the target group
+	currentTargets, err := s.frontendNlbTargetsManager.ListTargets(ctx, *sdkTG.TargetGroup.TargetGroupArn)
+	if err != nil {
+		return errors.Wrapf(err, "failed to list current target for target group: %s", *sdkTG.TargetGroup.TargetGroupArn)
+	}
+
+	// If there is no target, nothing to deregister
+	if len(currentTargets) == 0 {
+		s.logger.Info("No targets to deregister",
+			"targetGroupARN", *sdkTG.TargetGroup.TargetGroupArn)
+		return nil
+	}
+
+	// Deregister current target
+	s.logger.Info("Deregistering current target",
+		"targetGroupARN", *sdkTG.TargetGroup.TargetGroupArn,
+		"target", currentTargets[0].Target.Id,
+		"port", currentTargets[0].Target.Port,
+	)
+
+	err = s.frontendNlbTargetsManager.DeregisterTargets(ctx, *sdkTG.TargetGroup.TargetGroupArn, elbv2types.TargetDescription{
+		Id:   awssdk.String(*currentTargets[0].Target.Id),
+		Port: awssdk.Int32(*currentTargets[0].Target.Port),
+	})
+
+	if err != nil {
+		return errors.Wrapf(err, "failed to deregister targets for target group: %s", *sdkTG.TargetGroup.TargetGroupArn)
+	}
+
+	return nil
+}
+
+func (s *frontendNlbTargetSynthesizer) PostSynthesize(ctx context.Context) error {
+	var resTGs []*elbv2model.TargetGroup
+	s.stack.ListResources(&resTGs)
+
+	// Filter desired target group to include only ALB type target group
+	albResTGs := filterALBTargetGroups(resTGs)
+
+	for _, resTG := range albResTGs {
+
+		// Skip target group that are not yet created
+		if resTG.Status.TargetGroupARN == "" {
+			continue
+		}
+
+		// List current targets
+		currentTargets, err := s.frontendNlbTargetsManager.ListTargets(ctx, resTG.Status.TargetGroupARN)
+
+		if err != nil {
+			return err
+		}
+
+		desiredTarget, err := s.frontendNlbTargetGroupDesiredState.TargetGroups[resTG.Spec.Name].TargetARN.Resolve(ctx)
+		desiredTargetPort := s.frontendNlbTargetGroupDesiredState.TargetGroups[resTG.Spec.Name].TargetPort
+
+		if err != nil {
+			return errors.Wrapf(err, "failed to resolve the desiredTarget for target group: %s", desiredTarget)
+		}
+
+		if len(currentTargets) == 0 ||
+			currentTargets[0].Target == nil ||
+			currentTargets[0].Target.Id == nil ||
+			*currentTargets[0].Target.Id != desiredTarget {
+			err = s.frontendNlbTargetsManager.RegisterTargets(ctx, resTG.Status.TargetGroupARN, elbv2types.TargetDescription{
+				Id:   awssdk.String(desiredTarget),
+				Port: awssdk.Int32(desiredTargetPort),
+			})
+
+			if err != nil {
+				return err
+			}
+
+		}
+
+	}
+
+	return nil
+}
+
+func filterALBTargetGroups(targetGroups []*elbv2model.TargetGroup) []*elbv2model.TargetGroup {
+	var filteredTargetGroups []*elbv2model.TargetGroup
+	for _, tg := range targetGroups {
+		if tg.Spec.TargetType == "alb" {
+			filteredTargetGroups = append(filteredTargetGroups, tg)
+		}
+	}
+	return filteredTargetGroups
+}
+
+func (s *frontendNlbTargetSynthesizer) findSDKTargetGroups(ctx context.Context) ([]TargetGroupWithTags, error) {
+	stackTags := s.trackingProvider.StackTags(s.stack)
+	stackTagsLegacy := s.trackingProvider.StackTagsLegacy(s.stack)
+	return s.taggingManager.ListTargetGroups(ctx,
+		tracking.TagsAsTagFilter(stackTags),
+		tracking.TagsAsTagFilter(stackTagsLegacy))
+}