[feat aga] Add cross-namespace reference support for AGA #4547
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,177 @@ | ||
| package eventhandlers | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "github.com/go-logr/logr" | ||
| "k8s.io/client-go/util/workqueue" | ||
| agaapi "sigs.k8s.io/aws-load-balancer-controller/apis/aga/v1beta1" | ||
| "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" | ||
| "sigs.k8s.io/aws-load-balancer-controller/pkg/shared_constants" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
| "sigs.k8s.io/controller-runtime/pkg/event" | ||
| "sigs.k8s.io/controller-runtime/pkg/handler" | ||
| "sigs.k8s.io/controller-runtime/pkg/reconcile" | ||
| gwbeta1 "sigs.k8s.io/gateway-api/apis/v1beta1" | ||
| ) | ||
|
|
||
| // NewEnqueueRequestsForReferenceGrantEvent creates handler for ReferenceGrant resources | ||
| func NewEnqueueRequestsForReferenceGrantEvent( | ||
| k8sClient client.Client, | ||
| logger logr.Logger, | ||
| ) handler.TypedEventHandler[*gwbeta1.ReferenceGrant, reconcile.Request] { | ||
| return &enqueueRequestsForReferenceGrantEvent{ | ||
| k8sClient: k8sClient, | ||
| logger: logger, | ||
| } | ||
| } | ||
|
|
||
| var _ handler.TypedEventHandler[*gwbeta1.ReferenceGrant, reconcile.Request] = (*enqueueRequestsForReferenceGrantEvent)(nil) | ||
|
|
||
| // enqueueRequestsForReferenceGrantEvent handles ReferenceGrant events | ||
| type enqueueRequestsForReferenceGrantEvent struct { | ||
| k8sClient client.Client | ||
| logger logr.Logger | ||
| } | ||
|
|
||
| func (h *enqueueRequestsForReferenceGrantEvent) Create(ctx context.Context, e event.TypedCreateEvent[*gwbeta1.ReferenceGrant], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { | ||
| refGrant := e.Object | ||
| h.logger.V(1).Info("enqueue reference grant create event", "reference grant", refGrant.Name) | ||
| h.enqueueImpactedGlobalAccelerators(ctx, refGrant, nil, queue) | ||
| } | ||
|
|
||
| func (h *enqueueRequestsForReferenceGrantEvent) Update(ctx context.Context, e event.TypedUpdateEvent[*gwbeta1.ReferenceGrant], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { | ||
| refGrantNew := e.ObjectNew | ||
| refGrantOld := e.ObjectOld | ||
| h.logger.V(1).Info("enqueue reference grant update event", "reference grant", refGrantNew.Name) | ||
| h.enqueueImpactedGlobalAccelerators(ctx, refGrantNew, refGrantOld, queue) | ||
| } | ||
|
|
||
| func (h *enqueueRequestsForReferenceGrantEvent) Delete(ctx context.Context, e event.TypedDeleteEvent[*gwbeta1.ReferenceGrant], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { | ||
| refGrant := e.Object | ||
| h.logger.V(1).Info("enqueue reference grant delete event", "reference grant", refGrant.Name) | ||
| h.enqueueImpactedGlobalAccelerators(ctx, refGrant, nil, queue) | ||
| } | ||
|
|
||
| func (h *enqueueRequestsForReferenceGrantEvent) Generic(ctx context.Context, e event.TypedGenericEvent[*gwbeta1.ReferenceGrant], queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { | ||
| refGrant := e.Object | ||
| h.logger.V(1).Info("enqueue reference grant generic event", "reference grant", refGrant.Name) | ||
| h.enqueueImpactedGlobalAccelerators(ctx, refGrant, nil, queue) | ||
| } | ||
|
|
||
| // enqueueImpactedGlobalAccelerators finds and enqueues GlobalAccelerators impacted by a ReferenceGrant change | ||
| func (h *enqueueRequestsForReferenceGrantEvent) enqueueImpactedGlobalAccelerators( | ||
| ctx context.Context, | ||
| newRefGrant *gwbeta1.ReferenceGrant, | ||
| oldRefGrant *gwbeta1.ReferenceGrant, | ||
| queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { | ||
|
|
||
| // Collect all relevant namespaces from both old and new ReferenceGrant | ||
| impactedFroms := make(map[string]gwbeta1.ReferenceGrantFrom) | ||
|
|
||
| // Process new reference grant | ||
| for i, from := range newRefGrant.Spec.From { | ||
| if from.Group == shared_constants.GlobalAcceleratorResourcesGroup && from.Kind == shared_constants.GlobalAcceleratorKind { | ||
| key := generateGrantFromKey(from) | ||
| impactedFroms[key] = newRefGrant.Spec.From[i] | ||
| } | ||
| } | ||
|
|
||
| // Also process old reference grant if it exists (for updates) | ||
| if oldRefGrant != nil { | ||
| for i, from := range oldRefGrant.Spec.From { | ||
| if from.Group == shared_constants.GlobalAcceleratorResourcesGroup && from.Kind == shared_constants.GlobalAcceleratorKind { | ||
| key := generateGrantFromKey(from) | ||
| impactedFroms[key] = oldRefGrant.Spec.From[i] | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // If no GlobalAccelerator references found, nothing to do | ||
| if len(impactedFroms) == 0 { | ||
| h.logger.V(1).Info("ReferenceGrant doesn't reference GlobalAccelerators, skipping", | ||
| "referenceGrant", k8s.NamespacedName(newRefGrant)) | ||
| return | ||
| } | ||
|
|
||
| totalMatched := 0 | ||
|
|
||
| // Process each impacted GlobalAccelerator namespace | ||
| for _, from := range impactedFroms { | ||
|
|
||
| var gaList agaapi.GlobalAcceleratorList | ||
| if err := h.k8sClient.List(ctx, &gaList, &client.ListOptions{Namespace: string(from.Namespace)}); err != nil { | ||
| h.logger.Error(err, "Failed to list GlobalAccelerators for ReferenceGrant", | ||
| "referenceGrant", k8s.NamespacedName(newRefGrant), | ||
| "from namespace", from.Namespace) | ||
| continue | ||
| } | ||
|
|
||
| // Check each GA to see if it references resources in the target namespace | ||
| for i := range gaList.Items { | ||
| ga := &gaList.Items[i] | ||
|
|
||
| // Only check GAs that reference resources in the ReferenceGrant's namespace | ||
| hasRelevantCrossNamespaceRef := h.hasCrossNamespaceReferences( | ||
| ga, | ||
| newRefGrant.Namespace, | ||
| ) | ||
|
|
||
| if hasRelevantCrossNamespaceRef { | ||
| totalMatched++ | ||
|
|
||
| // Enqueue reconcile request for this GA | ||
| request := reconcile.Request{ | ||
| NamespacedName: k8s.NamespacedName(ga), | ||
| } | ||
|
|
||
| h.logger.V(1).Info("Enqueueing GlobalAccelerator for reconcile due to ReferenceGrant change", | ||
| "globalAccelerator", request.NamespacedName, | ||
| "referenceGrant", k8s.NamespacedName(newRefGrant)) | ||
|
|
||
| queue.Add(request) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| h.logger.V(1).Info("ReferenceGrant event processing completed", | ||
| "referenceGrant", k8s.NamespacedName(newRefGrant), | ||
| "totalMatchedGAs", totalMatched) | ||
| } | ||
|
|
||
| // hasCrossNamespaceReferences checks if a GlobalAccelerator has cross-namespace references to resources in the target namespace | ||
| func (h *enqueueRequestsForReferenceGrantEvent) hasCrossNamespaceReferences( | ||
| ga *agaapi.GlobalAccelerator, | ||
| targetNamespace string) bool { | ||
|
|
||
| // Go through all endpoints in the GA spec | ||
| if ga.Spec.Listeners == nil { | ||
| return false | ||
| } | ||
|
|
||
| for _, listener := range *ga.Spec.Listeners { | ||
| if listener.EndpointGroups == nil { | ||
| continue | ||
| } | ||
|
|
||
| for _, endpointGroup := range *listener.EndpointGroups { | ||
| if endpointGroup.Endpoints == nil { | ||
| continue | ||
| } | ||
|
|
||
| for _, endpoint := range *endpointGroup.Endpoints { | ||
| // Check for cross-namespace references | ||
| if endpoint.Namespace != nil && *endpoint.Namespace == targetNamespace && *endpoint.Namespace != ga.Namespace { | ||
| return true // Found a cross-namespace reference to target namespace | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return false | ||
| } | ||
|
|
||
| // generateGrantFromKey creates a unique key for a ReferenceGrantFrom | ||
| func generateGrantFromKey(from gwbeta1.ReferenceGrantFrom) string { | ||
| return fmt.Sprintf("%s-%s", from.Kind, from.Namespace) | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -177,7 +177,60 @@ endpointGroups: | |
|
|
||
| For more information about when and how to use port overrides, see [AWS Global Accelerator Port Overrides](https://docs.aws.amazon.com/global-accelerator/latest/dg/about-endpoint-groups-port-override.html) in the AWS documentation. | ||
|
|
||
| !!!note "Note" | ||
| The AWS Global Accelerator Controller handles all port override constraints automatically, ensuring your configuration is valid. | ||
|
|
||
| ## Cross-Namespace Endpoint References | ||
|
|
||
|
|
||
| The AWS Global Accelerator controller supports cross-namespace references for endpoints using the [Gateway API ReferenceGrant](https://gateway-api.sigs.k8s.io/api-types/referencegrant/) approach, which is a common pattern for secure cross-namespace references in Kubernetes. Cross-namespace references allow a GlobalAccelerator resource in one namespace (e.g., `accelerator-ns`) to reference resources in another namespace (e.g., `web-ns`), provided that a ReferenceGrant exists in the target namespace explicitly allowing the reference. | ||
|
|
||
| ### Using Cross-Namespace References | ||
|
|
||
| #### Step 1: Create a ReferenceGrant in the Target Namespace | ||
|
|
||
| To allow a GlobalAccelerator in namespace A to reference a resource in namespace B, create a ReferenceGrant in namespace B: | ||
|
|
||
| ```yaml | ||
| apiVersion: gateway.networking.k8s.io/v1beta1 | ||
| kind: ReferenceGrant | ||
| metadata: | ||
| name: allow-accelerator-references | ||
| namespace: web-ns # Target namespace containing the service | ||
| spec: | ||
| from: | ||
| - group: aga.k8s.aws | ||
| kind: GlobalAccelerator | ||
| namespace: accelerator-ns # Source namespace containing the GlobalAccelerator | ||
| to: | ||
| - group: "" | ||
| kind: Service | ||
| name: web-service | ||
| ``` | ||
|
|
||
| #### Step 2: Reference the Resource in your GlobalAccelerator | ||
|
|
||
| Once the ReferenceGrant is in place, you can reference the resource in your GlobalAccelerator: | ||
|
|
||
| ```yaml | ||
| apiVersion: aga.k8s.aws/v1beta1 | ||
| kind: GlobalAccelerator | ||
| metadata: | ||
| name: my-accelerator | ||
| namespace: accelerator-ns # Source namespace | ||
| spec: | ||
| listeners: | ||
| - endpointGroups: | ||
| - endpoints: | ||
| - namespace: web-ns # Target namespace | ||
| name: web-service # Service in target namespace | ||
| type: Service | ||
| ``` | ||
|
|
||
| !!!note "To use cross-namespace references" | ||
|
|
||
| 1. The Gateway API CRDs must be installed in your cluster (specifically the ReferenceGrant CRD) | ||
|
Collaborator
There was a problem hiding this comment. You can be explicit here and say that Reference grant is installed via "standard crds" https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/gateway/gateway/#prerequisites |
||
| 2. The controller must be granted permission to read ReferenceGrants cluster-wide | ||
|
|
||
| ## Sample CRDs | ||
|
|
||
|
|
@@ -283,11 +336,6 @@ Ensure your Service/Ingress/Gateway resources: | |
| 2. Have been successfully provisioned with actual AWS load balancers | ||
| 3. Are in the same namespace as specified in the endpoint | ||
|
|
||
|
|
||
| ## References | ||
|
|
||
|
|
||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is there any caching you can do here to prevent calling List() on the same namespace multiple times? I think the client should handle this case, but adding an explicit cache might make things cleaner here.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ohh are you concerned they might have multiple same namespace references in single grant or you thinking across grants?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I took a look at the Reference Grant structure, and the comment I made doesn't make sense (sorry). I see the existing reference grant logic also does this same listing.