From f461e74dfdb21cc9c8ee6428e41677d6021004e6 Mon Sep 17 00:00:00 2001 From: abasitt Date: Sun, 6 Jul 2025 21:14:55 -0700 Subject: [PATCH 01/66] adding logic to inject bgp sidecar to egress gateway --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 40 +++++++++ pkg/apis/kubeovn/v1/vpc-egress-gateway.go | 22 +++++ pkg/controller/vpc_egress_gateway.go | 95 +++++++++++++++++++++ 3 files changed, 157 insertions(+) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index 1f45c50c160..987a679a705 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -1192,6 +1192,46 @@ spec: required: - key - operator + bgp: + type: object + properties: + enabled: + type: boolean + default: false + image: + type: string + asn: + type: integer + format: int32 + minimum: 0 + remoteAsn: + type: integer + format: int32 + minimum: 0 + neighbors: + type: array + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + holdTime: + type: string + pattern: ^[0-9]+[smhd]$ + routerId: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + password: + type: string + enableGracefulRestart: + type: boolean + default: false + extraArgs: + type: array + items: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/pkg/apis/kubeovn/v1/vpc-egress-gateway.go b/pkg/apis/kubeovn/v1/vpc-egress-gateway.go index 3404804a88a..a0685f96310 100644 --- a/pkg/apis/kubeovn/v1/vpc-egress-gateway.go +++ b/pkg/apis/kubeovn/v1/vpc-egress-gateway.go @@ -95,6 +95,12 @@ type VpcEgressGatewaySpec struct { Policies []VpcEgressGatewayPolicy `json:"policies,omitempty"` // optional node selector used to select the nodes where the workload will be running NodeSelector []VpcEgressGatewayNodeSelector `json:"nodeSelector,omitempty"` + + // Add BGP configuration + BGP VpcEgressGatewayBGPConfig `json:"bgp,omitempty"` + + // TODO, subnet to access kube-apiserver. this will be used to reconcile routes to vpc + //KubeApiSubnet string `json:"kubeApiSubnet,omitempty"` } type VpcEgressGatewaySelector struct { @@ -151,3 +157,19 @@ type VpcEgressWorkload struct { // nodes where the workload is running Nodes []string `json:"nodes,omitempty"` } + +type VpcEgressGatewayBGPConfig struct { + // whether to enable BGP for the egress gateway + Enabled bool `json:"enabled"` + // optional bgp image used by the workload + // if not specified, the default image passed in by kube-ovn-controller will be used + Image string `json:"image,omitempty"` + ASN uint32 `json:"asn"` + RemoteASN uint32 `json:"remoteAsn"` + Neighbors []string `json:"neighbors"` + HoldTime metav1.Duration `json:"holdTime"` + RouterID string `json:"routerId"` + Password string `json:"password"` + EnableGracefulRestart bool `json:"enableGracefulRestart"` + ExtraArgs []string `json:"extraArgs"` +} \ No newline at end of file diff --git a/pkg/controller/vpc_egress_gateway.go b/pkg/controller/vpc_egress_gateway.go index 33846965f42..9a2147da582 100644 --- a/pkg/controller/vpc_egress_gateway.go +++ b/pkg/controller/vpc_egress_gateway.go @@ -240,9 +240,13 @@ func (c *Controller) updateVpcEgressGatewayStatus(gw *kubeovnv1.VpcEgressGateway // create or update vpc egress gateway workload func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGateway, vpc *kubeovnv1.Vpc, bfdIP, bfdIPv4, bfdIPv6 string) (string, set.Set[string], set.Set[string], *appsv1.Deployment, error) { image := c.config.Image + bgpImage := c.config.Image if gw.Spec.Image != "" { image = gw.Spec.Image } + if gw.Spec.BGP.Image != "" { + bgpImage = gw.Spec.BGP.Image + } if image == "" { err := fmt.Errorf("no image specified for vpc egress gateway %s/%s", gw.Namespace, gw.Name) klog.Error(err) @@ -476,6 +480,17 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa deploy.Spec.Template.Spec.Containers[0] = container } + // bgp sidecar container logic + if gw.Spec.BGP.Enabled { + // run BGP in the gateway container + bgpContainer, err := vpcEgressGatewayContainerBGP(bgpImage, gw.Name, &gw.Spec.BGP) + if err != nil { + klog.Errorf("failed to create a BGP speaker container for gateway %s: %v", gw.Name, err) + return "", nil, nil, nil, err + } + deploy.Spec.Template.Spec.Containers = append(deploy.Spec.Template.Spec.Containers, *bgpContainer) + } + // generate hash for the workload to determine whether to update the existing workload or not hash, err := util.Sha256HashObject(deploy) if err != nil { @@ -1051,3 +1066,83 @@ func (c *Controller) handlePodEventForVpcEgressGateway(pod *corev1.Pod) error { } return nil } + + +func vpcEgressGatewayContainerBGP(speakerImage, gatewayName string, speakerParams *kubeovnv1.VpcEgressGatewayBGPConfig) (*corev1.Container, error) { + if speakerImage == "" { + return nil, fmt.Errorf("BGP speaker image must be specified") + } + if speakerParams == nil { + return nil, fmt.Errorf("BGP config must not be nil") + } + if speakerParams.ASN == 0 { + return nil, fmt.Errorf("ASN not set, but must be non-zero value") + } + if speakerParams.RemoteASN == 0 { + return nil, fmt.Errorf("remote ASN not set, but must be non-zero value") + } + if len(speakerParams.Neighbors) == 0 { + return nil, fmt.Errorf("no BGP neighbors specified") + } + + args := []string{} + if speakerParams.RouterID != "" { + args = append(args, "--router-id="+speakerParams.RouterID) + } + if speakerParams.Password != "" { + args = append(args, "--auth-password="+speakerParams.Password) + } + if speakerParams.EnableGracefulRestart { + args = append(args, "--graceful-restart") + } + if speakerParams.HoldTime != (metav1.Duration{}) { + args = append(args, "--holdtime="+speakerParams.HoldTime.Duration.String()) + } + + args = append(args, fmt.Sprintf("--cluster-as=%d", speakerParams.ASN)) + args = append(args, fmt.Sprintf("--neighbor-as=%d", speakerParams.RemoteASN)) + + var neighIPv4, neighIPv6 []string + for _, neighbor := range speakerParams.Neighbors { + switch util.CheckProtocol(neighbor) { + case kubeovnv1.ProtocolIPv4: + neighIPv4 = append(neighIPv4, neighbor) + case kubeovnv1.ProtocolIPv6: + neighIPv6 = append(neighIPv6, neighbor) + default: + return nil, fmt.Errorf("unsupported protocol for peer %s", neighbor) + } + } + if len(neighIPv4) > 0 { + args = append(args, "--neighbor-address="+strings.Join(neighIPv4, ",")) + } + if len(neighIPv6) > 0 { + args = append(args, "--neighbor-ipv6-address="+strings.Join(neighIPv6, ",")) + } + + args = append(args, speakerParams.ExtraArgs...) + + container := &corev1.Container{ + Name: "vpc-egress-gw-speaker", + Image: speakerImage, + Command: []string{"/kube-ovn/kube-ovn-speaker"}, + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + { + Name: "EGRESS_GATEWAY_NAME", + Value: gatewayName, + }, + { + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + }, + Args: args, + } + + return container, nil +} \ No newline at end of file From 3d03196cc72d334d5ac1790f91fc6d590badace8 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Thu, 10 Jul 2025 21:36:53 -0700 Subject: [PATCH 02/66] [FEAT] bgp-speaker to watch event / sync local routing table --- pkg/speaker/controller.go | 10 ++ pkg/speaker/route_syncer.go | 326 ++++++++++++++++++++++++++++++++++++ 2 files changed, 336 insertions(+) create mode 100644 pkg/speaker/route_syncer.go diff --git a/pkg/speaker/controller.go b/pkg/speaker/controller.go index 294e124b34e..5561e8ef393 100644 --- a/pkg/speaker/controller.go +++ b/pkg/speaker/controller.go @@ -44,6 +44,8 @@ type Controller struct { informerFactory kubeinformers.SharedInformerFactory kubeovnInformerFactory kubeovninformer.SharedInformerFactory recorder record.EventRecorder + + routerSyncer *RouteSyncer } func NewController(config *Configuration) *Controller { @@ -69,6 +71,8 @@ func NewController(config *Configuration) *Controller { eipInformer := kubeovnInformerFactory.Kubeovn().V1().IptablesEIPs() natgatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcNatGateways() + routerSyncer := NewRouteSyncer(time.Second*60, config) + controller := &Controller{ config: config, @@ -86,6 +90,8 @@ func NewController(config *Configuration) *Controller { informerFactory: informerFactory, kubeovnInformerFactory: kubeovnInformerFactory, recorder: recorder, + + routerSyncer: routerSyncer, } return controller @@ -104,6 +110,10 @@ func (c *Controller) Run(stopCh <-chan struct{}) { klog.Info("Started workers") go wait.Until(c.Reconcile, 5*time.Second, stopCh) + // Start route syncer work + // run every c.routerSyncer.injectedRoutesSyncPeriod + c.routerSyncer.Run(stopCh) + <-stopCh klog.Info("Shutting down workers") } diff --git a/pkg/speaker/route_syncer.go b/pkg/speaker/route_syncer.go new file mode 100644 index 00000000000..456ff24ac84 --- /dev/null +++ b/pkg/speaker/route_syncer.go @@ -0,0 +1,326 @@ +package speaker + +import ( + "context" + "errors" + "fmt" + "net" + "strconv" + "sync" + "time" + + "k8s.io/klog/v2" + + gobgpapi "github.com/osrg/gobgp/v3/api" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netlink/nl" +) + +// RTPROT_BGP is the protocol number for BGP routes in the kernel's routing table. +// Get from https://github.com/torvalds/linux/blob/master/include/uapi/linux/rtnetlink.h#L312 +const ( + rtprotKernel = 2 // Route installed by kernel + rtprotBgp = 186 // BGP Routes +) + +// RouteSyncer is responsible for watching BGP events and handling them. +// It listens for BGP updates, injects routes into the local route table, and syncs them to the kernel's routing table. +// RouteSyncer is a struct that holds all of the information needed for syncing routes to the kernel's routing table +type RouteSyncer struct { + config *Configuration + routeTableStateMap map[string]*netlink.Route + injectedRoutesSyncPeriod time.Duration + mutex sync.Mutex + routeReplacer func(route *netlink.Route) error +} + +// NewRouteSyncer creates a new routeSyncer that, when run, +// will sync routes kept in its local state table every syncPeriod +func NewRouteSyncer(syncPeriod time.Duration, config *Configuration) *RouteSyncer { + rs := RouteSyncer{} + rs.config = config + rs.routeTableStateMap = make(map[string]*netlink.Route) + rs.injectedRoutesSyncPeriod = syncPeriod + rs.mutex = sync.Mutex{} + // We substitute the RouteReplace function here so that we can easily monkey patch it in our unit tests + rs.routeReplacer = netlink.RouteReplace + // Start to watch updates from the GoBGP server + rs.watchBgpUpdates() + + return &rs +} + +// update path cache when a new path is added or an existing path is updated from gobgp server +func (rs *RouteSyncer) watchBgpUpdates() { + pathWatch := func(r *gobgpapi.WatchEventResponse) { + if table := r.GetTable(); table != nil { + for _, path := range table.Paths { + if path.Family.Afi == gobgpapi.Family_AFI_IP || + path.Family.Afi == gobgpapi.Family_AFI_IP6 || + path.Family.Safi == gobgpapi.Family_SAFI_UNICAST { + if path.NeighborIp == "" { + return + } + klog.V(2).Infof("Processing bgp route advertisement from peer: %s", path.NeighborIp) + if err := rs.injectRoute(path); err != nil { + klog.Errorf("failed to inject routes due to: %v", err) + } + } + } + } + } + err := rs.config.BgpServer.WatchEvent(context.Background(), &gobgpapi.WatchEventRequest{ + Table: &gobgpapi.WatchEventRequest_Table{ + Filters: []*gobgpapi.WatchEventRequest_Table_Filter{ + { + Type: gobgpapi.WatchEventRequest_Table_Filter_BEST, + }, + }, + }, + }, pathWatch) + if err != nil { + klog.Errorf("failed to register monitor global routing table callback due to: %v", err) + } +} + +// addInjectedRoute adds a route to the route map that is regularly synced to the kernel's routing table +func (rs *RouteSyncer) AddInjectedRoute(dst *net.IPNet, route *netlink.Route) { + rs.mutex.Lock() + defer rs.mutex.Unlock() + klog.V(3).Infof("Adding route for destination: %s", dst) + rs.routeTableStateMap[dst.String()] = route +} + +// delInjectedRoute delete a route from the route map that is regularly synced to the kernel's routing table +func (rs *RouteSyncer) DelInjectedRoute(dst *net.IPNet) { + rs.mutex.Lock() + defer rs.mutex.Unlock() + if _, ok := rs.routeTableStateMap[dst.String()]; ok { + klog.V(3).Infof("Removing route for destination: %s", dst) + delete(rs.routeTableStateMap, dst.String()) + } +} + +// syncLocalRouteTable iterates over the local route state map and syncs all routes to the kernel's routing table +func (rs *RouteSyncer) SyncLocalRouteTable() (*netlink.Route, error) { + rs.mutex.Lock() + defer rs.mutex.Unlock() + klog.V(2).Infof("Running local route table synchronization") + for _, route := range rs.routeTableStateMap { + klog.V(3).Infof("Syncing route: %s -> %s via %s", route.Src, route.Dst, route.Gw) + err := rs.routeReplacer(route) + if err != nil { + return route, err + } + } + return nil, nil +} + +// run starts a goroutine that calls syncLocalRouteTable on interval injectedRoutesSyncPeriod +func (rs *RouteSyncer) Run(stopCh <-chan struct{}) { + // Start route synchronization routine + go func(stopCh <-chan struct{}) { + t := time.NewTicker(rs.injectedRoutesSyncPeriod) + defer t.Stop() + for { + select { + case <-t.C: + _, err := rs.SyncLocalRouteTable() + if err != nil { + klog.Errorf("route could not be replaced due to: %v", err) + } + case <-stopCh: + klog.Infof("Shutting down local route synchronization") + return + } + } + }(stopCh) +} + +// Delete the route from the kernel's routing table, and remove it from the local state map +func (rs *RouteSyncer) injectRoute(path *gobgpapi.Path) error { + klog.V(2).Infof("injectRoute Path Looks Like: %s", path.String()) + var route *netlink.Route + var link netlink.Link + + // TODO: This is a hardcoded link name, which is not ideal. + // Find the link by name, which is hardcoded to "net1" in this example. + // In a real-world scenario, you might want to make this configurable or discover it dynamically + // based on your network setup. + link, linkErr := netlink.LinkByName("net1") + if linkErr != nil { + klog.Fatalf("Failed to find interface: %v", linkErr) + return linkErr + } + + dst, nextHop, pathErr := rs.ParsePath(path) + if pathErr != nil { + return pathErr + } + + // TO-DO + // If path is same with external interface subnet do not add + if err := rs.checkExistingKernelRoute(dst); err != nil { + // If it's already a kernel route, just log and skip + klog.V(2).Infof("Skipping BGP route injection for %s: %v", dst.String(), err) + return nil + } + + // If the path we've received from GoBGP is a withdrawal, we should clean up any lingering routes that may exist + // on the host (rather than creating a new one or updating an existing one), and then return. + if path.IsWithdraw { + klog.V(2).Infof("Removing route: '%s via %s' from peer in the routing table", dst, nextHop) + + // Delete route from state map so that it doesn't get re-synced after deletion + rs.DelInjectedRoute(dst) + return rs.DeleteByDestination(dst) + } + + selectBestAddress := func(addrs []netlink.Addr) net.IP { + var bestAddr net.IP + bestScope := 1000 // Set a high initial scope value + for _, addr := range addrs { + // Scope values are defined as follows: + // RT_SCOPE_UNIVERSE(0) > RT_SCOPE_SITE(200) > RT_SCOPE_LINK(253) > RT_SCOPE_HOST(254) + if addr.Scope < bestScope { + bestScope = addr.Scope + bestAddr = addr.IP + } + } + // If no best address was found, return the first address in the list + if bestAddr == nil && len(addrs) > 0 { + bestAddr = addrs[0].IP + } + return bestAddr + } + // Use external interface for destination routing + // Determine the address family based on the destination IP + family := netlink.FAMILY_V4 + if dst.IP.To4() == nil { + family = netlink.FAMILY_V6 + } + // Get the list of addresses for the link + addrs, addrsErr := netlink.AddrList(link, family) + if addrsErr != nil { + klog.Errorf("failed to get addresses for interface %s: %s", + link.Attrs().Name, addrsErr) + return addrsErr + } + if len(addrs) == 0 { + klog.Errorf("no addresses found on interface %s", + link.Attrs().Name) + return errors.New("no addresses found on interface") + } + // If we have multiple addresses, we need to select the best one + bestIPForFamily := selectBestAddress(addrs) + + route = &netlink.Route{ + LinkIndex: link.Attrs().Index, + Src: bestIPForFamily, + Dst: dst, + Protocol: rtprotBgp, + Gw: nextHop, + } + + // We have our route configured, let's add it to the host's routing table + klog.V(2).Infof("Inject route: '%s via %s' from peer to routing table", dst, nextHop) + rs.AddInjectedRoute(dst, route) + // Immediately sync the local route table regardless of timer + _, syncLocalRouteTableErr := rs.SyncLocalRouteTable() + return syncLocalRouteTableErr +} + +// ParseNextHop takes in a GoBGP Path and parses out the destination's next hop from its attributes. If it +// can't parse a next hop IP from the GoBGP Path, it returns an error. +func (rs *RouteSyncer) ParseNextHop(path *gobgpapi.Path) (net.IP, error) { + for _, pAttr := range path.GetPattrs() { + unmarshalNew, err := pAttr.UnmarshalNew() + if err != nil { + return nil, fmt.Errorf("failed to unmarshal path attribute: %w", err) + } + switch t := unmarshalNew.(type) { + case *gobgpapi.NextHopAttribute: + // This is the primary way that we receive NextHops and happens when both the client and the server exchange + // next hops on the same IP family that they negotiated BGP on + nextHopIP := net.ParseIP(t.NextHop) + if nextHopIP != nil && (nextHopIP.To4() != nil || nextHopIP.To16() != nil) { + return nextHopIP, nil + } + return nil, fmt.Errorf("invalid nextHop address: %s", t.NextHop) + case *gobgpapi.MpReachNLRIAttribute: + // in the case where the server and the client are exchanging next-hops that don't relate to their primary + // IP family, we get MpReachNLRIAttribute instead of NextHopAttributes + // TODO: here we only take the first next hop, at some point in the future it would probably be best to + // consider multiple next hops + nextHopIP := net.ParseIP(t.NextHops[0]) + if nextHopIP != nil && (nextHopIP.To4() != nil || nextHopIP.To16() != nil) { + return nextHopIP, nil + } + return nil, fmt.Errorf("invalid nextHop address: %s", t.NextHops[0]) + } + } + return nil, fmt.Errorf("could not parse next hop received from GoBGP for path: %s", path) +} + +// ParsePath takes in a GoBGP Path and parses out the destination subnet and the next hop from its attributes. +// If successful, it will return the destination of the BGP path as a subnet form and the next hop. If it +// can't parse the destination or the next hop IP, it returns an error. +func (rs *RouteSyncer) ParsePath(path *gobgpapi.Path) (*net.IPNet, net.IP, error) { + nextHop, err := rs.ParseNextHop(path) + if err != nil { + return nil, nil, err + } + + nlri := path.GetNlri() + var prefix gobgpapi.IPAddressPrefix + err = nlri.UnmarshalTo(&prefix) + if err != nil { + return nil, nil, errors.New("invalid nlri in advertised path") + } + dstSubnet, err := netlink.ParseIPNet(prefix.Prefix + "/" + strconv.FormatUint(uint64(prefix.PrefixLen), 10)) + if err != nil { + return nil, nil, errors.New("couldn't parse IP subnet from nlri advertised path") + } + return dstSubnet, nextHop, nil +} + +// DeleteByDestination attempts to safely find all routes based upon its destination subnet and delete them +func (rs *RouteSyncer) DeleteByDestination(destinationSubnet *net.IPNet) error { + routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, &netlink.Route{ + Dst: destinationSubnet, Protocol: rtprotBgp, + }, netlink.RT_FILTER_DST|netlink.RT_FILTER_PROTOCOL) + if err != nil { + return fmt.Errorf("failed to get routes from netlink: %w", err) + } + for i, r := range routes { + klog.V(2).Infof("Found route to remove: %s", r.String()) + if err = netlink.RouteDel(&routes[i]); err != nil { + return fmt.Errorf("failed to remove route due to %w", err) + } + } + return nil +} + +// checks if a route with the same destination already exists +// with protocol "kernel" and returns an error if found +func (rs *RouteSyncer) checkExistingKernelRoute(dst *net.IPNet) error { + // Get existing routes for the destination + routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, &netlink.Route{ + Dst: dst, + Protocol: rtprotKernel, + }, netlink.RT_FILTER_DST|netlink.RT_FILTER_PROTOCOL) + if err != nil { + klog.Errorf("Failed to get existing routes for destination %s: %v", dst.String(), err) + return nil // Don't block route injection on query failure + } + + // If we found any kernel routes with the same destination, skip injection + if len(routes) > 0 { + for _, existingRoute := range routes { + klog.V(2).Infof("Found existing kernel route: %s", existingRoute.String()) + } + return fmt.Errorf("destination %s already exists as kernel protocol route", dst.String()) + } + + return nil +} From 72fae289c745e36c9f1c64a088047dd632306a0d Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Wed, 16 Jul 2025 02:04:21 -0700 Subject: [PATCH 03/66] set edge router flag to bgp speaker --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 3 + cmd/speaker/speaker.go | 3 +- pkg/apis/kubeovn/v1/vpc-egress-gateway.go | 11 +- pkg/controller/vpc_egress_gateway.go | 172 +++++++++++--------- pkg/speaker/config.go | 7 +- pkg/speaker/controller.go | 12 +- pkg/speaker/route_syncer.go | 23 ++- 7 files changed, 125 insertions(+), 106 deletions(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index 987a679a705..c5a1c013572 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -1195,6 +1195,9 @@ spec: bgp: type: object properties: + edgeRouterMode: + type: boolean + default: false enabled: type: boolean default: false diff --git a/cmd/speaker/speaker.go b/cmd/speaker/speaker.go index 55267cebecf..60ec54c968d 100644 --- a/cmd/speaker/speaker.go +++ b/cmd/speaker/speaker.go @@ -26,9 +26,8 @@ func CmdMain() { if err != nil { util.LogFatalAndExit(err, "failed to parse config") } - // Do not try to redirect the logs on the node if we're running in a NAT gateway - if !config.NatGwMode { + if !config.NatGwMode && !config.EdgeRouterMode { perm, err := strconv.ParseUint(config.LogPerm, 8, 32) if err != nil { util.LogFatalAndExit(err, "failed to parse log-perm") diff --git a/pkg/apis/kubeovn/v1/vpc-egress-gateway.go b/pkg/apis/kubeovn/v1/vpc-egress-gateway.go index a0685f96310..06889a67e30 100644 --- a/pkg/apis/kubeovn/v1/vpc-egress-gateway.go +++ b/pkg/apis/kubeovn/v1/vpc-egress-gateway.go @@ -97,10 +97,10 @@ type VpcEgressGatewaySpec struct { NodeSelector []VpcEgressGatewayNodeSelector `json:"nodeSelector,omitempty"` // Add BGP configuration - BGP VpcEgressGatewayBGPConfig `json:"bgp,omitempty"` + BGP VpcEgressGatewayBGPConfig `json:"bgp"` // TODO, subnet to access kube-apiserver. this will be used to reconcile routes to vpc - //KubeApiSubnet string `json:"kubeApiSubnet,omitempty"` + // KubeApiSubnet string `json:"kubeApiSubnet,omitempty"` } type VpcEgressGatewaySelector struct { @@ -159,11 +159,12 @@ type VpcEgressWorkload struct { } type VpcEgressGatewayBGPConfig struct { + EdgeRouterMode bool `json:"edgeRouterMode"` // whether to enable BGP for the egress gateway - Enabled bool `json:"enabled"` + Enabled bool `json:"enabled"` // optional bgp image used by the workload // if not specified, the default image passed in by kube-ovn-controller will be used - Image string `json:"image,omitempty"` + Image string `json:"image,omitempty"` ASN uint32 `json:"asn"` RemoteASN uint32 `json:"remoteAsn"` Neighbors []string `json:"neighbors"` @@ -172,4 +173,4 @@ type VpcEgressGatewayBGPConfig struct { Password string `json:"password"` EnableGracefulRestart bool `json:"enableGracefulRestart"` ExtraArgs []string `json:"extraArgs"` -} \ No newline at end of file +} diff --git a/pkg/controller/vpc_egress_gateway.go b/pkg/controller/vpc_egress_gateway.go index bbf980e6c87..20a8e99ff06 100644 --- a/pkg/controller/vpc_egress_gateway.go +++ b/pkg/controller/vpc_egress_gateway.go @@ -2,6 +2,7 @@ package controller import ( "context" + "errors" "fmt" "maps" "reflect" @@ -366,7 +367,7 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa annotations[util.LogicalSwitchAnnotation] = intSubnet.Name if len(gw.Spec.InternalIPs) != 0 { // set internal IPs - annotations[util.IPPoolAnnotation] = strings.Join(gw.Spec.InternalIPs, ";") + annotations[util.IPPoolAnnotation] = strings.Join(gw.Spec.InternalIPs, ";") // is it ; okay? } if len(gw.Spec.ExternalIPs) != 0 { // set external IPs @@ -489,7 +490,7 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa return "", nil, nil, nil, err } deploy.Spec.Template.Spec.Containers = append(deploy.Spec.Template.Spec.Containers, *bgpContainer) - } + } // generate hash for the workload to determine whether to update the existing workload or not hash, err := util.Sha256HashObject(deploy) @@ -1052,82 +1053,93 @@ func (c *Controller) handlePodEventForVpcEgressGateway(pod *corev1.Pod) error { return nil } - func vpcEgressGatewayContainerBGP(speakerImage, gatewayName string, speakerParams *kubeovnv1.VpcEgressGatewayBGPConfig) (*corev1.Container, error) { - if speakerImage == "" { - return nil, fmt.Errorf("BGP speaker image must be specified") - } - if speakerParams == nil { - return nil, fmt.Errorf("BGP config must not be nil") - } - if speakerParams.ASN == 0 { - return nil, fmt.Errorf("ASN not set, but must be non-zero value") - } - if speakerParams.RemoteASN == 0 { - return nil, fmt.Errorf("remote ASN not set, but must be non-zero value") - } - if len(speakerParams.Neighbors) == 0 { - return nil, fmt.Errorf("no BGP neighbors specified") - } - - args := []string{} - if speakerParams.RouterID != "" { - args = append(args, "--router-id="+speakerParams.RouterID) - } - if speakerParams.Password != "" { - args = append(args, "--auth-password="+speakerParams.Password) - } - if speakerParams.EnableGracefulRestart { - args = append(args, "--graceful-restart") - } - if speakerParams.HoldTime != (metav1.Duration{}) { - args = append(args, "--holdtime="+speakerParams.HoldTime.Duration.String()) - } - - args = append(args, fmt.Sprintf("--cluster-as=%d", speakerParams.ASN)) - args = append(args, fmt.Sprintf("--neighbor-as=%d", speakerParams.RemoteASN)) - - var neighIPv4, neighIPv6 []string - for _, neighbor := range speakerParams.Neighbors { - switch util.CheckProtocol(neighbor) { - case kubeovnv1.ProtocolIPv4: - neighIPv4 = append(neighIPv4, neighbor) - case kubeovnv1.ProtocolIPv6: - neighIPv6 = append(neighIPv6, neighbor) - default: - return nil, fmt.Errorf("unsupported protocol for peer %s", neighbor) - } - } - if len(neighIPv4) > 0 { - args = append(args, "--neighbor-address="+strings.Join(neighIPv4, ",")) - } - if len(neighIPv6) > 0 { - args = append(args, "--neighbor-ipv6-address="+strings.Join(neighIPv6, ",")) - } - - args = append(args, speakerParams.ExtraArgs...) - - container := &corev1.Container{ - Name: "vpc-egress-gw-speaker", - Image: speakerImage, - Command: []string{"/kube-ovn/kube-ovn-speaker"}, - ImagePullPolicy: corev1.PullIfNotPresent, - Env: []corev1.EnvVar{ - { - Name: "EGRESS_GATEWAY_NAME", - Value: gatewayName, - }, - { - Name: "POD_IP", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIP", - }, - }, - }, - }, - Args: args, - } - - return container, nil -} \ No newline at end of file + if speakerImage == "" { + return nil, errors.New("BGP speaker image must be specified") + } + if speakerParams == nil { + return nil, errors.New("BGP config must not be nil") + } + if speakerParams.ASN == 0 { + return nil, errors.New("ASN not set, but must be non-zero value") + } + if speakerParams.RemoteASN == 0 { + return nil, errors.New("remote ASN not set, but must be non-zero value") + } + if len(speakerParams.Neighbors) == 0 { + return nil, errors.New("no BGP neighbors specified") + } + + args := []string{} + if speakerParams.EdgeRouterMode { + args = append(args, "--edge-router-mode=true") + } + if speakerParams.RouterID != "" { + args = append(args, "--router-id="+speakerParams.RouterID) + } + if speakerParams.Password != "" { + args = append(args, "--auth-password="+speakerParams.Password) + } + if speakerParams.EnableGracefulRestart { + args = append(args, "--graceful-restart") + } + if speakerParams.HoldTime != (metav1.Duration{}) { + args = append(args, "--holdtime="+speakerParams.HoldTime.Duration.String()) + } + + args = append(args, fmt.Sprintf("--cluster-as=%d", speakerParams.ASN)) + args = append(args, fmt.Sprintf("--neighbor-as=%d", speakerParams.RemoteASN)) + + var neighIPv4, neighIPv6 []string + for _, neighbor := range speakerParams.Neighbors { + switch util.CheckProtocol(neighbor) { + case kubeovnv1.ProtocolIPv4: + neighIPv4 = append(neighIPv4, neighbor) + case kubeovnv1.ProtocolIPv6: + neighIPv6 = append(neighIPv6, neighbor) + default: + return nil, fmt.Errorf("unsupported protocol for peer %s", neighbor) + } + } + if len(neighIPv4) > 0 { + args = append(args, "--neighbor-address="+strings.Join(neighIPv4, ",")) + } + if len(neighIPv6) > 0 { + args = append(args, "--neighbor-ipv6-address="+strings.Join(neighIPv6, ",")) + } + + args = append(args, speakerParams.ExtraArgs...) + + container := &corev1.Container{ + Name: "vpc-egress-gw-speaker", + Image: speakerImage, + Command: []string{"/kube-ovn/kube-ovn-speaker"}, + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + { + Name: "EGRESS_GATEWAY_NAME", + Value: gatewayName, + }, + { + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + }, + Args: args, + // bgp need to add/remove fib, it needs root user + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](0), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_BIND_SERVICE", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, + } + + return container, nil +} diff --git a/pkg/speaker/config.go b/pkg/speaker/config.go index a07d21c97ee..292f63519c9 100644 --- a/pkg/speaker/config.go +++ b/pkg/speaker/config.go @@ -65,8 +65,9 @@ type Configuration struct { KubeClient kubernetes.Interface KubeOvnClient clientset.Interface - PprofPort int32 - LogPerm string + PprofPort int32 + LogPerm string + EdgeRouterMode bool } func ParseFlags() (*Configuration, error) { @@ -94,6 +95,7 @@ func ParseFlags() (*Configuration, error) { argNatGwMode = pflag.BoolP("nat-gw-mode", "", false, "Make the BGP speaker announce EIPs from inside a NAT gateway, Pod IP/Service/Subnet announcements will be disabled") argEnableMetrics = pflag.BoolP("enable-metrics", "", true, "Whether to support metrics query") argLogPerm = pflag.String("log-perm", "640", "The permission for the log file") + argEdgeRouterMode = pflag.BoolP("edge-router-mode", "", false, "Make the BGP speaker announce inside subnet and get routes from the outside, work as edge router") ) klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) klog.InitFlags(klogFlags) @@ -161,6 +163,7 @@ func ParseFlags() (*Configuration, error) { NatGwMode: *argNatGwMode, EnableMetrics: *argEnableMetrics, LogPerm: *argLogPerm, + EdgeRouterMode: *argEdgeRouterMode, } if *argNeighborAddress != "" { diff --git a/pkg/speaker/controller.go b/pkg/speaker/controller.go index 5561e8ef393..bc520fd177b 100644 --- a/pkg/speaker/controller.go +++ b/pkg/speaker/controller.go @@ -108,11 +108,13 @@ func (c *Controller) Run(stopCh <-chan struct{}) { } klog.Info("Started workers") - go wait.Until(c.Reconcile, 5*time.Second, stopCh) - - // Start route syncer work - // run every c.routerSyncer.injectedRoutesSyncPeriod - c.routerSyncer.Run(stopCh) + if c.config.EdgeRouterMode { + // Start route syncer work + // run every c.routerSyncer.injectedRoutesSyncPeriod + c.routerSyncer.Run(stopCh) + } else { + go wait.Until(c.Reconcile, 5*time.Second, stopCh) + } <-stopCh klog.Info("Shutting down workers") diff --git a/pkg/speaker/route_syncer.go b/pkg/speaker/route_syncer.go index 456ff24ac84..bf7759bf68f 100644 --- a/pkg/speaker/route_syncer.go +++ b/pkg/speaker/route_syncer.go @@ -61,7 +61,7 @@ func (rs *RouteSyncer) watchBgpUpdates() { if path.NeighborIp == "" { return } - klog.V(2).Infof("Processing bgp route advertisement from peer: %s", path.NeighborIp) + klog.Infof("Processing bgp route advertisement from peer: %s", path.NeighborIp) if err := rs.injectRoute(path); err != nil { klog.Errorf("failed to inject routes due to: %v", err) } @@ -87,7 +87,7 @@ func (rs *RouteSyncer) watchBgpUpdates() { func (rs *RouteSyncer) AddInjectedRoute(dst *net.IPNet, route *netlink.Route) { rs.mutex.Lock() defer rs.mutex.Unlock() - klog.V(3).Infof("Adding route for destination: %s", dst) + klog.Infof("Adding route for destination: %s", dst) rs.routeTableStateMap[dst.String()] = route } @@ -96,7 +96,7 @@ func (rs *RouteSyncer) DelInjectedRoute(dst *net.IPNet) { rs.mutex.Lock() defer rs.mutex.Unlock() if _, ok := rs.routeTableStateMap[dst.String()]; ok { - klog.V(3).Infof("Removing route for destination: %s", dst) + klog.Infof("Removing route for destination: %s", dst) delete(rs.routeTableStateMap, dst.String()) } } @@ -105,9 +105,9 @@ func (rs *RouteSyncer) DelInjectedRoute(dst *net.IPNet) { func (rs *RouteSyncer) SyncLocalRouteTable() (*netlink.Route, error) { rs.mutex.Lock() defer rs.mutex.Unlock() - klog.V(2).Infof("Running local route table synchronization") + klog.Infof("Running local route table synchronization") for _, route := range rs.routeTableStateMap { - klog.V(3).Infof("Syncing route: %s -> %s via %s", route.Src, route.Dst, route.Gw) + klog.Infof("Syncing route: %s -> %s via %s", route.Src, route.Dst, route.Gw) err := rs.routeReplacer(route) if err != nil { return route, err @@ -139,7 +139,7 @@ func (rs *RouteSyncer) Run(stopCh <-chan struct{}) { // Delete the route from the kernel's routing table, and remove it from the local state map func (rs *RouteSyncer) injectRoute(path *gobgpapi.Path) error { - klog.V(2).Infof("injectRoute Path Looks Like: %s", path.String()) + klog.Infof("injectRoute Path Looks Like: %s", path.String()) var route *netlink.Route var link netlink.Link @@ -158,18 +158,17 @@ func (rs *RouteSyncer) injectRoute(path *gobgpapi.Path) error { return pathErr } - // TO-DO // If path is same with external interface subnet do not add if err := rs.checkExistingKernelRoute(dst); err != nil { // If it's already a kernel route, just log and skip - klog.V(2).Infof("Skipping BGP route injection for %s: %v", dst.String(), err) + klog.Infof("Skipping BGP route injection for %s: %v", dst.String(), err) return nil } // If the path we've received from GoBGP is a withdrawal, we should clean up any lingering routes that may exist // on the host (rather than creating a new one or updating an existing one), and then return. if path.IsWithdraw { - klog.V(2).Infof("Removing route: '%s via %s' from peer in the routing table", dst, nextHop) + klog.Infof("Removing route: '%s via %s' from peer in the routing table", dst, nextHop) // Delete route from state map so that it doesn't get re-synced after deletion rs.DelInjectedRoute(dst) @@ -223,7 +222,7 @@ func (rs *RouteSyncer) injectRoute(path *gobgpapi.Path) error { } // We have our route configured, let's add it to the host's routing table - klog.V(2).Infof("Inject route: '%s via %s' from peer to routing table", dst, nextHop) + klog.Infof("Inject route: '%s via %s' from peer to routing table", dst, nextHop) rs.AddInjectedRoute(dst, route) // Immediately sync the local route table regardless of timer _, syncLocalRouteTableErr := rs.SyncLocalRouteTable() @@ -293,7 +292,7 @@ func (rs *RouteSyncer) DeleteByDestination(destinationSubnet *net.IPNet) error { return fmt.Errorf("failed to get routes from netlink: %w", err) } for i, r := range routes { - klog.V(2).Infof("Found route to remove: %s", r.String()) + klog.Infof("Found route to remove: %s", r.String()) if err = netlink.RouteDel(&routes[i]); err != nil { return fmt.Errorf("failed to remove route due to %w", err) } @@ -317,7 +316,7 @@ func (rs *RouteSyncer) checkExistingKernelRoute(dst *net.IPNet) error { // If we found any kernel routes with the same destination, skip injection if len(routes) > 0 { for _, existingRoute := range routes { - klog.V(2).Infof("Found existing kernel route: %s", existingRoute.String()) + klog.Infof("Found existing kernel route: %s", existingRoute.String()) } return fmt.Errorf("destination %s already exists as kernel protocol route", dst.String()) } From e8fe3482c1a34594774ab166d57a3ae17764b105 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Wed, 16 Jul 2025 21:22:07 -0700 Subject: [PATCH 04/66] select external ip for router id automatically if router-id is null --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 4 ++ pkg/speaker/config.go | 46 +++++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index c5a1c013572..86cab8d00ab 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -1223,9 +1223,13 @@ spec: pattern: ^[0-9]+[smhd]$ routerId: type: string + # The routerId can be an IPv4 or IPv6 address, but we are not enforcing it here + # routerId could be nullable, so that we can use environment value to set it + # refer pkg/speaker/config.go#L186 anyOf: - format: ipv4 - format: ipv6 + - pattern: '^$' password: type: string enableGracefulRestart: diff --git a/pkg/speaker/config.go b/pkg/speaker/config.go index 292f63519c9..ebf7c756636 100644 --- a/pkg/speaker/config.go +++ b/pkg/speaker/config.go @@ -2,6 +2,7 @@ package speaker import ( "context" + "encoding/json" "errors" "flag" "fmt" @@ -184,11 +185,19 @@ func ParseFlags() (*Configuration, error) { } if config.RouterID == "" { - if podIPv4 != "" { - config.RouterID = podIPv4 - } else { - config.RouterID = podIPv6 + externalIP, err := GetExternalIP() + if err != nil || externalIP == "" { + klog.Warningf("failed to get external IP: %v", err) + return nil, err } + config.RouterID = externalIP + klog.Infof("using external IP %s as router ID", config.RouterID) + + // if podIPv4 != "" { + // config.RouterID = podIPv4 + // } else { + // config.RouterID = podIPv6 + // } if config.RouterID == "" { return nil, errors.New("no router id or POD_IPS") } @@ -368,3 +377,32 @@ func (config *Configuration) initBgpServer() error { config.BgpServer = s return nil } + +func GetExternalIP() (string, error) { + raw := os.Getenv("MULTI_NET_STATUS") + if raw == "" { + return "", errors.New("MULTI_NET_STATUS annotation is empty") + } + + type networkStatusEntry struct { + Name string `json:"name"` + Interface string `json:"interface"` + IPs []string `json:"ips"` + Default bool `json:"default"` + DNS struct{} `json:"dns"` + } + + var entries []networkStatusEntry + if err := json.Unmarshal([]byte(raw), &entries); err != nil { + return "", err + } + + for _, e := range entries { + // search for CNI network name is not "kube-ovn" + if e.Name != "kube-ovn" && len(e.IPs) > 0 { + return e.IPs[0], nil + } + } + + return "", errors.New("non–kube-ovn interface not found") +} From 62b2ae43fb0b032085666819ad3c648470364760 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Sun, 20 Jul 2025 20:09:10 -0700 Subject: [PATCH 05/66] bgp-edge-router code modification --- pkg/apis/kubeovn/v1/bgp-edge-router.go | 170 +++ pkg/apis/kubeovn/v1/register.go | 2 + pkg/apis/kubeovn/v1/zz_generated.deepcopy.go | 269 ++++ .../typed/kubeovn/v1/bgpedgerouter.go | 114 ++ .../typed/kubeovn/v1/generated_expansion.go | 2 + .../kubeovn/v1/bgpedgerouter.go | 89 ++ .../externalversions/kubeovn/v1/interface.go | 7 + .../listers/kubeovn/v1/bgpedgerouter.go | 81 ++ .../listers/kubeovn/v1/expansion_generated.go | 8 + pkg/controller/bgp_edge_router.go | 1133 +++++++++++++++++ pkg/controller/controller.go | 16 +- pkg/ovs/ovn.go | 1 + pkg/util/const.go | 1 + 13 files changed, 1892 insertions(+), 1 deletion(-) create mode 100644 pkg/apis/kubeovn/v1/bgp-edge-router.go create mode 100644 pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go create mode 100644 pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouter.go create mode 100644 pkg/client/listers/kubeovn/v1/bgpedgerouter.go create mode 100644 pkg/controller/bgp_edge_router.go diff --git a/pkg/apis/kubeovn/v1/bgp-edge-router.go b/pkg/apis/kubeovn/v1/bgp-edge-router.go new file mode 100644 index 00000000000..f3f67341033 --- /dev/null +++ b/pkg/apis/kubeovn/v1/bgp-edge-router.go @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 +package v1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type BerPhase string + +const ( + // PhasePending means the resource is pending and not processed yet + BerPhasePending BerPhase = "Pending" + // PhaseProcessing means the resource is being processed + BerPhaseProcessing BerPhase = "Processing" + // PhaseCompleted means the resource has been processed successfully + BerPhaseCompleted BerPhase = "Completed" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type BgpEdgeRouterList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []BgpEdgeRouter `json:"items"` +} + +// +genclient +// +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale +// +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +resourceName=vpc-egress-gateways +// vpc egress gateway is used to forward the egress traffic from the VPC to the external network +type BgpEdgeRouter struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + + Spec BgpEdgeRouterSpec `json:"spec"` + Status BgpEdgeRouterStatus `json:"status"` +} + +// VPC returns the VPC name +// If the BgpEdgeRouter has no VPC specified in the spec, it will return the default VPC name +func (g *BgpEdgeRouter) VPC(defaultVPC string) string { + if g.Spec.VPC != "" { + return g.Spec.VPC + } + return defaultVPC +} + +// Ready returns true if the BgpEdgeRouter has been processed successfully and is ready to serve traffic +func (g *BgpEdgeRouter) Ready() bool { + return g.Status.Ready && g.Status.Conditions.IsReady(g.Generation) +} + +type BgpEdgeRouterSpec struct { + // optional VPC name + // if not specified, the default VPC will be used + VPC string `json:"vpc,omitempty"` + // workload replicas + Replicas int32 `json:"replicas,omitempty"` + // optional name prefix used to generate the workload + // the workload name will be generated as + Prefix string `json:"prefix,omitempty"` + // optional image used by the workload + // if not specified, the default image passed in by kube-ovn-controller will be used + Image string `json:"image,omitempty"` + // optional internal subnet used to create the workload + // if not specified, the workload will be created in the default subnet of the VPC + InternalSubnet string `json:"internalSubnet,omitempty"` + // external subnet used to create the workload + ExternalSubnet string `json:"externalSubnet"` + // optional internal/external IPs used to create the workload + // these IPs must be in the internal/external subnet + // the IPs count must NOT be less than the replicas count + InternalIPs []string `json:"internalIPs,omitempty"` + ExternalIPs []string `json:"externalIPs,omitempty"` + // namespace/pod selectors + Selectors []BgpEdgeRouterSelector `json:"selectors,omitempty"` + // optional traffic policy used to control the traffic routing + // if not specified, the default traffic policy "Cluster" will be used + // if set to "Local", traffic will be routed to the gateway pod/instance on the same node when available + // currently it works only for the default vpc + TrafficPolicy string `json:"trafficPolicy,omitempty"` + + // BFD configuration + BFD BgpEdgeRouterBFDConfig `json:"bfd"` + // egress policies + // at least one policy must be specified + Policies []BgpEdgeRouterPolicy `json:"policies,omitempty"` + // optional node selector used to select the nodes where the workload will be running + NodeSelector []BgpEdgeRouterNodeSelector `json:"nodeSelector,omitempty"` + + // Add BGP configuration + BGP BgpEdgeRouterBGPConfig `json:"bgp"` + + // TODO, subnet to access kube-apiserver. this will be used to reconcile routes to vpc + // KubeApiSubnet string `json:"kubeApiSubnet,omitempty"` +} + +type BgpEdgeRouterSelector struct { + NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"` + PodSelector *metav1.LabelSelector `json:"podSelector,omitempty"` +} + +type BgpEdgeRouterBFDConfig struct { + // whether to enable BFD + // if set to true, the egress gateway will establish BFD session(s) with the VPC BFD LRP + // the VPC's .spec.bfd.enabled must be set to true to enable BFD + Enabled bool `json:"enabled"` + // optional BFD minRX/minTX/multiplier + MinRX int32 `json:"minRX,omitempty"` + MinTX int32 `json:"minTX,omitempty"` + Multiplier int32 `json:"multiplier,omitempty"` +} + +type BgpEdgeRouterPolicy struct { + // whether to enable SNAT/MASQUERADE for the egress traffic + SNAT bool `json:"snat"` + // CIDRs/subnets targeted by the egress traffic policy + IPBlocks []string `json:"ipBlocks,omitempty"` + Subnets []string `json:"subnets,omitempty"` +} + +type BgpEdgeRouterNodeSelector struct { + MatchLabels map[string]string `json:"matchLabels,omitempty"` + MatchExpressions []corev1.NodeSelectorRequirement `json:"matchExpressions,omitempty"` + MatchFields []corev1.NodeSelectorRequirement `json:"matchFields,omitempty"` +} + +type BgpEdgeRouterStatus struct { + // used by the scale subresource + Replicas int32 `json:"replicas,omitempty"` + LabelSelector string `json:"labelSelector,omitempty"` + + // whether the egress gateway is ready + Ready bool `json:"ready"` + Phase Phase `json:"phase"` + // internal/external IPs used by the workload + InternalIPs []string `json:"internalIPs,omitempty"` + ExternalIPs []string `json:"externalIPs,omitempty"` + Conditions Conditions `json:"conditions,omitempty"` + + // workload information + Workload BgpEdgeRouterWorkload `json:"workload"` +} + +type BgpEdgeRouterWorkload struct { + APIVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + Name string `json:"name,omitempty"` + // nodes where the workload is running + Nodes []string `json:"nodes,omitempty"` +} + +type BgpEdgeRouterBGPConfig struct { + // whether to enable BGP for the egress gateway + Enabled bool `json:"enabled"` + // optional bgp image used by the workload + // if not specified, the default image passed in by kube-ovn-controller will be used + Image string `json:"image,omitempty"` + ASN uint32 `json:"asn"` + RemoteASN uint32 `json:"remoteAsn"` + Neighbors []string `json:"neighbors"` + HoldTime metav1.Duration `json:"holdTime"` + RouterID string `json:"routerId"` + Password string `json:"password"` + EnableGracefulRestart bool `json:"enableGracefulRestart"` + ExtraArgs []string `json:"extraArgs"` +} diff --git a/pkg/apis/kubeovn/v1/register.go b/pkg/apis/kubeovn/v1/register.go index 020f13f8dc2..ea2e2fd478a 100644 --- a/pkg/apis/kubeovn/v1/register.go +++ b/pkg/apis/kubeovn/v1/register.go @@ -73,6 +73,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &VpcEgressGatewayList{}, &VpcNatGateway{}, &VpcNatGatewayList{}, + &BgpEdgeRouter{}, + &BgpEdgeRouterList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go index ae3130280da..325fa473021 100644 --- a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go +++ b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go @@ -2840,3 +2840,272 @@ func (in *VpcStatus) DeepCopy() *VpcStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouter) DeepCopyInto(out *BgpEdgeRouter) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGateway. +func (in *BgpEdgeRouter) DeepCopy() *BgpEdgeRouter { + if in == nil { + return nil + } + out := new(BgpEdgeRouter) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BgpEdgeRouter) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterBFDConfig) DeepCopyInto(out *BgpEdgeRouterBFDConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayBFDConfig. +func (in *BgpEdgeRouterBFDConfig) DeepCopy() *BgpEdgeRouterBFDConfig { + if in == nil { + return nil + } + out := new(BgpEdgeRouterBFDConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterList) DeepCopyInto(out *BgpEdgeRouterList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]BgpEdgeRouter, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayList. +func (in *BgpEdgeRouterList) DeepCopy() *BgpEdgeRouterList { + if in == nil { + return nil + } + out := new(BgpEdgeRouterList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BgpEdgeRouterList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterNodeSelector) DeepCopyInto(out *BgpEdgeRouterNodeSelector) { + *out = *in + if in.MatchLabels != nil { + in, out := &in.MatchLabels, &out.MatchLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.MatchExpressions != nil { + in, out := &in.MatchExpressions, &out.MatchExpressions + *out = make([]corev1.NodeSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.MatchFields != nil { + in, out := &in.MatchFields, &out.MatchFields + *out = make([]corev1.NodeSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayNodeSelector. +func (in *BgpEdgeRouterNodeSelector) DeepCopy() *BgpEdgeRouterNodeSelector { + if in == nil { + return nil + } + out := new(BgpEdgeRouterNodeSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterPolicy) DeepCopyInto(out *BgpEdgeRouterPolicy) { + *out = *in + if in.IPBlocks != nil { + in, out := &in.IPBlocks, &out.IPBlocks + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayPolicy. +func (in *BgpEdgeRouterPolicy) DeepCopy() *BgpEdgeRouterPolicy { + if in == nil { + return nil + } + out := new(BgpEdgeRouterPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterSelector) DeepCopyInto(out *BgpEdgeRouterSelector) { + *out = *in + if in.NamespaceSelector != nil { + in, out := &in.NamespaceSelector, &out.NamespaceSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.PodSelector != nil { + in, out := &in.PodSelector, &out.PodSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewaySelector. +func (in *BgpEdgeRouterSelector) DeepCopy() *BgpEdgeRouterSelector { + if in == nil { + return nil + } + out := new(BgpEdgeRouterSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterSpec) DeepCopyInto(out *BgpEdgeRouterSpec) { + *out = *in + if in.InternalIPs != nil { + in, out := &in.InternalIPs, &out.InternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalIPs != nil { + in, out := &in.ExternalIPs, &out.ExternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Selectors != nil { + in, out := &in.Selectors, &out.Selectors + *out = make([]BgpEdgeRouterSelector, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.BFD = in.BFD + if in.Policies != nil { + in, out := &in.Policies, &out.Policies + *out = make([]BgpEdgeRouterPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make([]BgpEdgeRouterNodeSelector, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpEdgeRouterSpec. +func (in *BgpEdgeRouterSpec) DeepCopy() *BgpEdgeRouterSpec { + if in == nil { + return nil + } + out := new(BgpEdgeRouterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterStatus) DeepCopyInto(out *BgpEdgeRouterStatus) { + *out = *in + if in.InternalIPs != nil { + in, out := &in.InternalIPs, &out.InternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalIPs != nil { + in, out := &in.ExternalIPs, &out.ExternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Workload.DeepCopyInto(&out.Workload) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpEdgeRouterStatus. +func (in *BgpEdgeRouterStatus) DeepCopy() *BgpEdgeRouterStatus { + if in == nil { + return nil + } + out := new(BgpEdgeRouterStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterWorkload) DeepCopyInto(out *BgpEdgeRouterWorkload) { + *out = *in + if in.Nodes != nil { + in, out := &in.Nodes, &out.Nodes + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpEdgeRouterWorkload. +func (in *BgpEdgeRouterWorkload) DeepCopy() *BgpEdgeRouterWorkload { + if in == nil { + return nil + } + out := new(BgpEdgeRouterWorkload) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go new file mode 100644 index 00000000000..c168b9ab6d1 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go @@ -0,0 +1,114 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + + // apiskubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + // versioned "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + // internalinterfaces "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions/internalinterfaces" + // kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" + // autoscalingv1 "k8s.io/api/autoscaling/v1" + // metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + // runtime "k8s.io/apimachinery/pkg/runtime" + // types "k8s.io/apimachinery/pkg/types" + // watch "k8s.io/apimachinery/pkg/watch" + // cache "k8s.io/client-go/tools/cache" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + scheme "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" + autoscalingv1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// BgpEdgeRoutersGetter has a method to return a BgpEdgeRouterInterface. +// A group's client should implement this interface. +type BgpEdgeRoutersGetter interface { + BgpEdgeRouters(namespace string) BgpEdgeRouterInterface +} + +// BgpEdgeRouterInterface has methods to work with BgpEdgeRouter resources. +type BgpEdgeRouterInterface interface { + Create(ctx context.Context, bgpEdgeRouter *kubeovnv1.BgpEdgeRouter, opts metav1.CreateOptions) (*kubeovnv1.BgpEdgeRouter, error) + Update(ctx context.Context, bgpEdgeRouter *kubeovnv1.BgpEdgeRouter, opts metav1.UpdateOptions) (*kubeovnv1.BgpEdgeRouter, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, bgpEdgeRouter *kubeovnv1.BgpEdgeRouter, opts metav1.UpdateOptions) (*kubeovnv1.BgpEdgeRouter, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*kubeovnv1.BgpEdgeRouter, error) + List(ctx context.Context, opts metav1.ListOptions) (*kubeovnv1.BgpEdgeRouterList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *kubeovnv1.BgpEdgeRouter, err error) + GetScale(ctx context.Context, bgpEdgeRouterName string, options metav1.GetOptions) (*autoscalingv1.Scale, error) + UpdateScale(ctx context.Context, bgpEdgeRouterName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (*autoscalingv1.Scale, error) + + BgpEdgeRouterExpansion +} + +// bgpEdgeRouters implements BgpEdgeRouterInterface +type bgpEdgeRouters struct { + *gentype.ClientWithList[*kubeovnv1.BgpEdgeRouter, *kubeovnv1.BgpEdgeRouterList] +} + +// newBgpEdgeRouters returns a BgpEdgeRouters +func newBgpEdgeRouters(c *KubeovnV1Client, namespace string) *bgpEdgeRouters { + return &bgpEdgeRouters{ + gentype.NewClientWithList[*kubeovnv1.BgpEdgeRouter, *kubeovnv1.BgpEdgeRouterList]( + "bgp-edgerouters", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *kubeovnv1.BgpEdgeRouter { return &kubeovnv1.BgpEdgeRouter{} }, + func() *kubeovnv1.BgpEdgeRouterList { return &kubeovnv1.BgpEdgeRouterList{} }, + ), + } +} + +// GetScale takes name of the bgpEdgeRouter, and returns the corresponding autoscalingv1.Scale object, and an error if there is any. +func (c *bgpEdgeRouters) GetScale(ctx context.Context, bgpEdgeRouterName string, options metav1.GetOptions) (result *autoscalingv1.Scale, err error) { + result = &autoscalingv1.Scale{} + err = c.GetClient().Get(). + Namespace(c.GetNamespace()). + Resource("bgp-edgerouters"). + Name(bgpEdgeRouterName). + SubResource("scale"). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// UpdateScale takes the top resource name and the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *bgpEdgeRouters) UpdateScale(ctx context.Context, bgpEdgeRouterName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (result *autoscalingv1.Scale, err error) { + result = &autoscalingv1.Scale{} + err = c.GetClient().Put(). + Namespace(c.GetNamespace()). + Resource("bgp-edge-routers"). + Name(bgpEdgeRouterName). + SubResource("scale"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(scale). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go index a4a3dcf75f0..e1185cd5250 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go @@ -59,3 +59,5 @@ type VpcDnsExpansion interface{} type VpcEgressGatewayExpansion interface{} type VpcNatGatewayExpansion interface{} + +type BgpEdgeRouterExpansion interface{} diff --git a/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouter.go b/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouter.go new file mode 100644 index 00000000000..dfbd88dbf1c --- /dev/null +++ b/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouter.go @@ -0,0 +1,89 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + time "time" + + apiskubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + versioned "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions/internalinterfaces" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// BgpEdgeRouterInformer provides access to a shared informer and lister for +// BgpEdgeRouters. +type BgpEdgeRouterInformer interface { + Informer() cache.SharedIndexInformer + Lister() kubeovnv1.BgpEdgeRouterLister +} + +type bgpEdgeRouterInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewVlanInformer constructs a new informer for Vlan type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewBgpEdgeRouterInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredBgpEdgeRouterInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredVlanInformer constructs a new informer for Vlan type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredBgpEdgeRouterInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().BgpEdgeRouters().List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().BgpEdgeRouters().Watch(context.TODO(), options) + }, + }, + &apiskubeovnv1.BgpEdgeRouter{}, + resyncPeriod, + indexers, + ) +} + +func (f *bgpEdgeRouterInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredBgpEdgeRouterInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *bgpEdgeRouterInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apiskubeovnv1.BgpEdgeRouter{}, f.defaultInformer) +} + +func (f *bgpEdgeRouterInformer) Lister() kubeovnv1.BgpEdgeRouterLister { + return kubeovnv1.NewBgpEdgeRouterLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/kubeovn/v1/interface.go b/pkg/client/informers/externalversions/kubeovn/v1/interface.go index 69cad95925a..ed144452d54 100644 --- a/pkg/client/informers/externalversions/kubeovn/v1/interface.go +++ b/pkg/client/informers/externalversions/kubeovn/v1/interface.go @@ -66,6 +66,8 @@ type Interface interface { VpcEgressGateways() VpcEgressGatewayInformer // VpcNatGateways returns a VpcNatGatewayInformer. VpcNatGateways() VpcNatGatewayInformer + // BgpEdgeRouters returns a BgpEdgeRouterInformer. + BgpEdgeRouters() BgpEdgeRouterInformer } type version struct { @@ -183,3 +185,8 @@ func (v *version) VpcEgressGateways() VpcEgressGatewayInformer { func (v *version) VpcNatGateways() VpcNatGatewayInformer { return &vpcNatGatewayInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} } + +// BgpEdgeRouters returns a BgpEdgeRouterInformer. +func (v *version) BgpEdgeRouters() BgpEdgeRouterInformer { + return &bgpEdgeRouterInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/client/listers/kubeovn/v1/bgpedgerouter.go b/pkg/client/listers/kubeovn/v1/bgpedgerouter.go new file mode 100644 index 00000000000..c7c9bf2d6cd --- /dev/null +++ b/pkg/client/listers/kubeovn/v1/bgpedgerouter.go @@ -0,0 +1,81 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// type BgpEdgeRouterInformer interface { +// Informer() cache.SharedIndexInformer +// Lister() kubeovnv1.BgpEdgeRouterLister +// } + +// type bgpEdgeRouterInformer struct { +// factory internalinterfaces.SharedInformerFactory +// tweakListOptions internalinterfaces.TweakListOptionsFunc +// namespace string +// } + +// BgpEdgeRouterLister helps list BgpEdgeRouters. +// All objects returned here must be treated as read-only. +type BgpEdgeRouterLister interface { + // List lists all BgpEdgeRouters in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*kubeovnv1.BgpEdgeRouter, err error) + // BgpEdgeRouters returns an object that can list and get BgpEdgeRouters. + BgpEdgeRouters(namespace string) BgpEdgeRouterNamespaceLister + BgpEdgeRouterListerExpansion +} + +// bgpEdgeRouterLister implements the bgpEdgeRouterLister interface. +type bgpEdgeRouterLister struct { + listers.ResourceIndexer[*kubeovnv1.BgpEdgeRouter] +} + +// NewBgpEdgeRouterLister returns a new bgpEdgeRouterLister. +func NewBgpEdgeRouterLister(indexer cache.Indexer) BgpEdgeRouterLister { + return &bgpEdgeRouterLister{listers.New[*kubeovnv1.BgpEdgeRouter](indexer, kubeovnv1.Resource("bgpedgerouter"))} +} + +// bgpEdgeRouters returns an object that can list and get bgpEdgeRouters. +func (s *bgpEdgeRouterLister) BgpEdgeRouters(namespace string) BgpEdgeRouterNamespaceLister { + return bgpEdgeRouterNamespaceLister{listers.NewNamespaced[*kubeovnv1.BgpEdgeRouter](s.ResourceIndexer, namespace)} +} + +// bgpEdgeRouterNamespaceLister helps list and get bgpEdgeRouters. +// All objects returned here must be treated as read-only. +type BgpEdgeRouterNamespaceLister interface { + // List lists all bgpEdgeRouters in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*kubeovnv1.BgpEdgeRouter, err error) + // Get retrieves the bgpEdgeRouter from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*kubeovnv1.BgpEdgeRouter, error) + BgpEdgeRouterNamespaceListerExpansion +} + +// bgpEdgeRouterNamespaceLister implements the bgpEdgeRouterNamespaceLister +// interface. +type bgpEdgeRouterNamespaceLister struct { + listers.ResourceIndexer[*kubeovnv1.BgpEdgeRouter] +} diff --git a/pkg/client/listers/kubeovn/v1/expansion_generated.go b/pkg/client/listers/kubeovn/v1/expansion_generated.go index b474d11b77d..cc2027db016 100644 --- a/pkg/client/listers/kubeovn/v1/expansion_generated.go +++ b/pkg/client/listers/kubeovn/v1/expansion_generated.go @@ -105,3 +105,11 @@ type VpcEgressGatewayNamespaceListerExpansion interface{} // VpcNatGatewayListerExpansion allows custom methods to be added to // VpcNatGatewayLister. type VpcNatGatewayListerExpansion interface{} + +// BgpEdgeRouterListerExpansion allows custom methods to be added to +// BgpEdgeRouterLister. +type BgpEdgeRouterListerExpansion interface{} + +// BgpEdgeRouterNamespaceListerExpansion allows custom methods to be added to +// BgpEdgeRouterNamespaceLister. +type BgpEdgeRouterNamespaceListerExpansion interface{} diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go new file mode 100644 index 00000000000..1646544b135 --- /dev/null +++ b/pkg/controller/bgp_edge_router.go @@ -0,0 +1,1133 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "maps" + "reflect" + "slices" + "strconv" + "strings" + + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + "k8s.io/utils/set" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +func (c *Controller) enqueueAddBgpEdgeRouter(obj any) { + key := cache.MetaObjectToName(obj.(*kubeovnv1.BgpEdgeRouter)).String() + klog.V(3).Infof("enqueue add bgp-edge-router %s", key) + c.addOrUpdateBgpEdgeRouterQueue.Add(key) +} + +func (c *Controller) enqueueUpdateBgpEdgeRouter(_, newObj any) { + key := cache.MetaObjectToName(newObj.(*kubeovnv1.BgpEdgeRouter)).String() + klog.V(3).Infof("enqueue update bgp-edge-router %s", key) + c.addOrUpdateBgpEdgeRouterQueue.Add(key) +} + +func (c *Controller) enqueueDeleteBgpEdgeRouter(obj any) { + key := cache.MetaObjectToName(obj.(*kubeovnv1.BgpEdgeRouter)).String() + klog.V(3).Infof("enqueue delete bgp-edge-router %s", key) + c.delBgpEdgeRouterQueue.Add(key) +} + +func bgpEdgeRouterWorkloadLabels(bgpEdgeRouterName string) map[string]string { + return map[string]string{"app": "bgp-edge-router", util.BgpEdgeRouterLabel: bgpEdgeRouterName} +} + +func (c *Controller) handleAddOrUpdateBgpEdgeRouter(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.bgpEdgeRouterKeyMutex.LockKey(key) + defer func() { _ = c.bgpEdgeRouterKeyMutex.UnlockKey(key) }() + + cachedRouter, err := c.bgpEdgeRouterLister.BgpEdgeRouters(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + if !cachedRouter.DeletionTimestamp.IsZero() { + c.delBgpEdgeRouterQueue.Add(key) + return nil + } + + klog.Infof("reconciling bgp-edge-router %s", key) + router := cachedRouter.DeepCopy() + if router, err = c.initbgpEdgeRouterStatus(router); err != nil { + return err + } + + vpcName := router.Spec.VPC + if vpcName == "" { + vpcName = c.config.ClusterRouter + } + vpc, err := c.vpcsLister.Get(vpcName) + if err != nil { + klog.Error(err) + return err + } + if router.Spec.BFD.Enabled && vpc.Status.BFDPort.IP == "" { + err = fmt.Errorf("vpc %s bfd port is not enabled or not ready", vpc.Name) + klog.Error(err) + router.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "VpcBfdPortNotEnabled", err.Error(), router.Generation) + _, _ = c.updatebgpEdgeRouterStatus(router) + return err + } + + if controllerutil.AddFinalizer(router, util.KubeOVNControllerFinalizer) { + updatedGateway, err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouters(router.Namespace). + Update(context.Background(), router, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to add finalizer for bgp-edge-router %s/%s: %w", router.Namespace, router.Name, err) + klog.Error(err) + return err + } + router = updatedGateway + } + + var bfdIP, bfdIPv4, bfdIPv6 string + if router.Spec.BFD.Enabled { + bfdIP = vpc.Status.BFDPort.IP + bfdIPv4, bfdIPv6 = util.SplitStringIP(bfdIP) + } + + // reconcile the vpc egress gateway workload and get the route sources for later OVN resources reconciliation + attachmentNetworkName, ipv4Src, ipv6Src, deploy, err := c.reconcilebgpEdgeRouterWorkload(router, vpc, bfdIP, bfdIPv4, bfdIPv6) + router.Status.Replicas = router.Spec.Replicas + router.Status.LabelSelector = labels.FormatLabels(bgpEdgeRouterWorkloadLabels(router.Name)) + if err != nil { + klog.Error(err) + router.Status.Replicas = 0 + router.Status.Conditions.SetCondition(kubeovnv1.Ready, corev1.ConditionFalse, "ReconcileWorkloadFailed", err.Error(), router.Generation) + _, _ = c.updatebgpEdgeRouterStatus(router) + return err + } + + router.Status.InternalIPs = nil + router.Status.ExternalIPs = nil + router.Status.Workload.APIVersion = deploy.APIVersion + router.Status.Workload.Kind = deploy.Kind + router.Status.Workload.Name = deploy.Name + router.Status.Workload.Nodes = nil + nodeNexthopIPv4 := make(map[string]string, int(router.Spec.Replicas)) + nodeNexthopIPv6 := make(map[string]string, int(router.Spec.Replicas)) + ready := util.DeploymentIsReady(deploy) + if !ready { + router.Status.Ready = false + msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) + router.Status.Conditions.SetCondition(kubeovnv1.Ready, corev1.ConditionFalse, "Processing", msg, router.Generation) + } + // get the pods of the deployment to collect the pod IPs + podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) + if err != nil { + err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return err + } + + pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) + if err != nil { + err = fmt.Errorf("failed to list pods of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return err + } + + // update gateway status including the internal/external IPs and the nodes where the pods are running + router.Status.Workload.Nodes = make([]string, 0, len(pods)) + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + extIPs, err := util.PodAttachmentIPs(pod, attachmentNetworkName) + if err != nil { + klog.Error(err) + continue + } + + ips := util.PodIPs(*pod) + ipv4, ipv6 := util.SplitIpsByProtocol(ips) + if len(ipv4) != 0 { + nodeNexthopIPv4[pod.Spec.NodeName] = ipv4[0] + } + if len(ipv6) != 0 { + nodeNexthopIPv6[pod.Spec.NodeName] = ipv6[0] + } + router.Status.InternalIPs = append(router.Status.InternalIPs, strings.Join(ips, ",")) + router.Status.ExternalIPs = append(router.Status.ExternalIPs, strings.Join(extIPs, ",")) + router.Status.Workload.Nodes = append(router.Status.Workload.Nodes, pod.Spec.NodeName) + } + if router, err = c.updatebgpEdgeRouterStatus(router); err != nil { + klog.Error(err) + return err + } + + // reconcile OVN routes + if err = c.reconcilebgpEdgeRouterOVNRoutes(router, 4, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv4, nodeNexthopIPv4, ipv4Src); err != nil { + klog.Error(err) + return err + } + if err = c.reconcilebgpEdgeRouterOVNRoutes(router, 6, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv6, nodeNexthopIPv6, ipv6Src); err != nil { + klog.Error(err) + return err + } + + if ready { + router.Status.Ready = true + router.Status.Phase = kubeovnv1.PhaseCompleted + router.Status.Conditions.SetReady("ReconcileSuccess", router.Generation) + if _, err = c.updatebgpEdgeRouterStatus(router); err != nil { + return err + } + } + + klog.Infof("finished reconciling bgp-edge-router %s", key) + + return nil +} + +func (c *Controller) initbgpEdgeRouterStatus(router *kubeovnv1.BgpEdgeRouter) (*kubeovnv1.BgpEdgeRouter, error) { + var err error + if router.Status.Phase == "" || router.Status.Phase == kubeovnv1.PhasePending { + router.Status.Phase = kubeovnv1.PhaseProcessing + router, err = c.updatebgpEdgeRouterStatus(router) + } + return router, err +} + +func (c *Controller) updatebgpEdgeRouterStatus(router *kubeovnv1.BgpEdgeRouter) (*kubeovnv1.BgpEdgeRouter, error) { + if len(router.Status.Conditions) == 0 { + router.Status.Conditions.SetCondition(kubeovnv1.Init, corev1.ConditionUnknown, "Processing", "", router.Generation) + } + if !router.Status.Ready { + router.Status.Phase = kubeovnv1.PhaseProcessing + } + + updateRouter, err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouters(router.Namespace). + UpdateStatus(context.Background(), router, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to update status of bgp-edge-router %s/%s: %w", router.Namespace, router.Name, err) + klog.Error(err) + return nil, err + } + + return updateRouter, nil +} + +// create or update bgp edge router workload +func (c *Controller) reconcilebgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRouter, vpc *kubeovnv1.Vpc, bfdIP, bfdIPv4, bfdIPv6 string) (string, set.Set[string], set.Set[string], *appsv1.Deployment, error) { + image := c.config.Image + bgpImage := c.config.Image + if router.Spec.Image != "" { + image = router.Spec.Image + } + if router.Spec.BGP.Image != "" { + bgpImage = router.Spec.BGP.Image + } + if image == "" { + err := fmt.Errorf("no image specified for vpc egress gateway %s/%s", router.Namespace, router.Name) + klog.Error(err) + return "", nil, nil, nil, err + } + + if len(router.Spec.InternalIPs) != 0 && len(router.Spec.InternalIPs) < int(router.Spec.Replicas) { + err := fmt.Errorf("internal IPs count %d is less than replicas %d", len(router.Spec.InternalIPs), router.Spec.Replicas) + klog.Error(err) + return "", nil, nil, nil, err + } + if len(router.Spec.ExternalIPs) != 0 && len(router.Spec.ExternalIPs) < int(router.Spec.Replicas) { + err := fmt.Errorf("external IPs count %d is less than replicas %d", len(router.Spec.ExternalIPs), router.Spec.Replicas) + klog.Error(err) + return "", nil, nil, nil, err + } + + internalSubnet := router.Spec.InternalSubnet + if internalSubnet == "" { + internalSubnet = vpc.Status.DefaultLogicalSwitch + } + if internalSubnet == "" { + err := fmt.Errorf("default subnet of vpc %s not found, please set internal subnet of the egress gateway", vpc.Name) + klog.Error(err) + return "", nil, nil, nil, err + } + intSubnet, err := c.subnetsLister.Get(internalSubnet) + if err != nil { + klog.Error(err) + return "", nil, nil, nil, err + } + extSubnet, err := c.subnetsLister.Get(router.Spec.ExternalSubnet) + if err != nil { + klog.Error(err) + return "", nil, nil, nil, err + } + if !strings.ContainsRune(extSubnet.Spec.Provider, '.') { + err = fmt.Errorf("please set correct provider of subnet %s to get the network-attachment-definition", extSubnet.Name) + klog.Error(err) + return "", nil, nil, nil, err + } + subStrings := strings.Split(extSubnet.Spec.Provider, ".") + nadName, nadNamespace := subStrings[0], subStrings[1] + if _, err = c.netAttachLister.NetworkAttachmentDefinitions(nadNamespace).Get(nadName); err != nil { + klog.Errorf("failed to get net-attach-def %s/%s: %v", nadNamespace, nadName, err) + return "", nil, nil, nil, err + } + attachmentNetworkName := fmt.Sprintf("%s/%s", nadNamespace, nadName) + + // collect egress policies + ipv4ForwardSrc, ipv6ForwardSrc := set.New[string](), set.New[string]() + ipv4SNATSrc, ipv6SNATSrc := set.New[string](), set.New[string]() + for _, policy := range router.Spec.Policies { + ipv4, ipv6 := util.SplitIpsByProtocol(policy.IPBlocks) + if policy.SNAT { + ipv4SNATSrc.Insert(ipv4...) + ipv6SNATSrc.Insert(ipv6...) + } else { + ipv4ForwardSrc.Insert(ipv4...) + ipv6ForwardSrc.Insert(ipv6...) + } + for _, subnetName := range policy.Subnets { + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + if subnet.Status.IsNotValidated() { + err = fmt.Errorf("subnet %s is not validated", subnet.Name) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + // TODO: check subnet's vpc and vlan + ipv4, ipv6 := util.SplitStringIP(subnet.Spec.CIDRBlock) + if policy.SNAT { + ipv4SNATSrc.Insert(ipv4) + ipv6SNATSrc.Insert(ipv6) + } else { + ipv4ForwardSrc.Insert(ipv4) + ipv6ForwardSrc.Insert(ipv6) + } + } + } + + // calculate internal route destinations and forward source CIDR blocks + intRouteDstIPv4, intRouteDstIPv6 := ipv4ForwardSrc.Union(ipv4SNATSrc), ipv6ForwardSrc.Union(ipv6SNATSrc) + intRouteDstIPv4.Delete("") + intRouteDstIPv6.Delete("") + ipv4ForwardSrc.Delete("") + ipv6ForwardSrc.Delete("") + + // generate route annotations used to configure routes in the pod + routes := util.NewPodRoutes() + intGatewayIPv4, intGatewayIPv6 := util.SplitStringIP(intSubnet.Spec.Gateway) + extGatewayIPv4, extGatewayIPv6 := util.SplitStringIP(extSubnet.Spec.Gateway) + // add routes for the VPC BFD Port so that the egress gateway can establish BFD session(s) with it + routes.Add(util.OvnProvider, bfdIPv4, intGatewayIPv4) + routes.Add(util.OvnProvider, bfdIPv6, intGatewayIPv6) + // add routes for the internal networks + for _, dst := range intRouteDstIPv4.UnsortedList() { + routes.Add(util.OvnProvider, dst, intGatewayIPv4) + } + for _, dst := range intRouteDstIPv6.UnsortedList() { + routes.Add(util.OvnProvider, dst, intGatewayIPv6) + } + // add default routes to forward traffic to the external network + routes.Add(extSubnet.Spec.Provider, "0.0.0.0/0", extGatewayIPv4) + routes.Add(extSubnet.Spec.Provider, "::/0", extGatewayIPv6) + + // generate pod annotations + annotations, err := routes.ToAnnotations() + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + annotations[nadv1.NetworkAttachmentAnnot] = attachmentNetworkName + annotations[util.LogicalSwitchAnnotation] = intSubnet.Name + if len(router.Spec.InternalIPs) != 0 { + // set internal IPs + annotations[util.IPPoolAnnotation] = strings.Join(router.Spec.InternalIPs, ";") + } + if len(router.Spec.ExternalIPs) != 0 { + // set external IPs + annotations[fmt.Sprintf(util.IPPoolAnnotationTemplate, extSubnet.Spec.Provider)] = strings.Join(router.Spec.ExternalIPs, ";") + } + + // generate init container environment variables + // the init container is responsible for adding routes and SNAT rules to the pod network namespace + initEnv, err := bgpEdgeRouterInitContainerEnv(4, intGatewayIPv4, extGatewayIPv4, ipv4ForwardSrc) + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + ipv6Env, err := bgpEdgeRouterInitContainerEnv(6, intGatewayIPv6, extGatewayIPv6, ipv6ForwardSrc) + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + initEnv = append(initEnv, ipv6Env...) + + // generate workload + labels := vegWorkloadLabels(router.Name) + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: router.Spec.Prefix + router.Name, + Namespace: router.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: ptr.To(intstr.FromInt(1)), + MaxSurge: ptr.To(intstr.FromInt(0)), + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: berMergeNodeSelector(router.Spec.NodeSelector), + }, + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: corev1.LabelHostname, + }}, + }, + }, + InitContainers: []corev1.Container{{ + Name: "init", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"bash", "/kube-ovn/init-vpc-egress-gateway.sh"}, + Env: initEnv, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "usr-local-sbin", + MountPath: "/usr/local/sbin", + }}, + }}, + Containers: []corev1.Container{{ + Name: "gateway", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sleep", "infinity"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](65534), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "usr-local-sbin", + MountPath: "/usr/local/sbin", + }}, + }}, + Volumes: []corev1.Volume{{ + Name: "usr-local-sbin", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }}, + TerminationGracePeriodSeconds: ptr.To[int64](0), + }, + }, + }, + } + // set owner reference so that the workload will be deleted automatically when the vpc egress gateway is deleted + if err = util.SetOwnerReference(router, deploy); err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + + if bfdIP != "" { + // run BFD in the gateway container to establish BFD session(s) with the VPC BFD LRP + container := bgpEdgeRouterContainerBFDD(image, bfdIP, router.Spec.BFD.MinTX, router.Spec.BFD.MinRX, router.Spec.BFD.Multiplier) + deploy.Spec.Template.Spec.Containers[0] = container + } + + // bgp sidecar container logic + if router.Spec.BGP.Enabled { + // run BGP in the gateway container + bgpContainer, err := bgpEdgeRouterContainerBGP(bgpImage, router.Name, &router.Spec.BGP) + if err != nil { + klog.Errorf("failed to create a BGP speaker container for gateway %s: %v", router.Name, err) + return "", nil, nil, nil, err + } + deploy.Spec.Template.Spec.Containers = append(deploy.Spec.Template.Spec.Containers, *bgpContainer) + } + + // generate hash for the workload to determine whether to update the existing workload or not + hash, err := util.Sha256HashObject(deploy) + if err != nil { + err = fmt.Errorf("failed to hash generated deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + + hash = hash[:12] + // replicas and the hash annotation should be excluded from hash calculation + deploy.Spec.Replicas = ptr.To(router.Spec.Replicas) + deploy.Annotations = map[string]string{util.GenerateHashAnnotation: hash} + if currentDeploy, err := c.deploymentsLister.Deployments(router.Namespace).Get(deploy.Name); err != nil { + if !k8serrors.IsNotFound(err) { + err = fmt.Errorf("failed to get deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + if deploy, err = c.config.KubeClient.AppsV1().Deployments(router.Namespace). + Create(context.Background(), deploy, metav1.CreateOptions{}); err != nil { + err = fmt.Errorf("failed to create deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + } else if !reflect.DeepEqual(currentDeploy.Spec.Replicas, deploy.Spec.Replicas) || + currentDeploy.Annotations[util.GenerateHashAnnotation] != hash { + // update the deployment if replicas or hash annotation is changed + if deploy, err = c.config.KubeClient.AppsV1().Deployments(router.Namespace). + Update(context.Background(), deploy, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to update deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + } else { + // no need to create or update the deployment + deploy = currentDeploy + } + + // return the source CIDR blocks for later OVN resources reconciliation + deploy.APIVersion, deploy.Kind = deploymentGroupVersion, deploymentKind + return attachmentNetworkName, intRouteDstIPv4, intRouteDstIPv6, deploy, nil +} + +func (c *Controller) reconcilebgpEdgeRouterOVNRoutes(router *kubeovnv1.BgpEdgeRouter, af int, lrName, lrpName, bfdIP string, nextHops map[string]string, sources set.Set[string]) error { + if len(nextHops) == 0 { + return nil + } + + externalIDs := map[string]string{ + ovs.ExternalIDVendor: util.CniTypeName, + ovs.ExternalIDBgpEdgeRouter: fmt.Sprintf("%s/%s", router.Namespace, router.Name), + "af": strconv.Itoa(af), + } + bfdList, err := c.OVNNbClient.FindBFD(externalIDs) + if err != nil { + klog.Error(err) + return err + } + + // reconcile OVN port group + ports := set.New[string]() + for _, selector := range router.Spec.Selectors { + sel := labels.Everything() + if selector.NamespaceSelector != nil { + if sel, err = metav1.LabelSelectorAsSelector(selector.NamespaceSelector); err != nil { + err = fmt.Errorf("failed to create label selector for namespace selector %#v: %w", selector.NamespaceSelector, err) + klog.Error(err) + return err + } + } + namespaces, err := c.namespacesLister.List(sel) + if err != nil { + err = fmt.Errorf("failed to list namespaces with selector %s: %w", sel, err) + klog.Error(err) + return err + } + sel = labels.Everything() + if selector.PodSelector != nil { + if sel, err = metav1.LabelSelectorAsSelector(selector.PodSelector); err != nil { + err = fmt.Errorf("failed to create label selector for pod selector %#v: %w", selector.PodSelector, err) + klog.Error(err) + return err + } + } + for _, ns := range namespaces { + pods, err := c.podsLister.Pods(ns.Name).List(sel) + if err != nil { + err = fmt.Errorf("failed to list pods with selector %s in namespace %s: %w", sel, ns.Name, err) + klog.Error(err) + return err + } + for _, pod := range pods { + if pod.Spec.HostNetwork || + pod.Annotations[util.AllocatedAnnotation] != "true" || + pod.Annotations[util.LogicalRouterAnnotation] != router.VPC(c.config.ClusterRouter) || + !isPodAlive(pod) { + continue + } + podName := c.getNameByPod(pod) + ports.Insert(ovs.PodNameToPortName(podName, pod.Namespace, util.OvnProvider)) + } + } + } + key := cache.MetaObjectToName(router).String() + pgName := vegPortGroupName(key) + if err = c.OVNNbClient.CreatePortGroup(pgName, externalIDs); err != nil { + err = fmt.Errorf("failed to create port group %s: %w", pgName, err) + klog.Error(err) + return err + } + if err = c.OVNNbClient.PortGroupSetPorts(pgName, ports.UnsortedList()); err != nil { + err = fmt.Errorf("failed to set ports of port group %s: %w", pgName, err) + klog.Error(err) + return err + } + + // reconcile OVN address set + asName := vegAddressSetName(key, af) + if err = c.OVNNbClient.CreateAddressSet(asName, externalIDs); err != nil { + err = fmt.Errorf("failed to create address set %s: %w", asName, err) + klog.Error(err) + return err + } + if err = c.OVNNbClient.AddressSetUpdateAddress(asName, sources.SortedList()...); err != nil { + err = fmt.Errorf("failed to update address set %s: %w", asName, err) + klog.Error(err) + return err + } + + // reconcile OVN BFD entries + bfdIDs := set.New[string]() + staleBFDIDs := set.New[string]() + bfdDstIPs := set.New(slices.Collect(maps.Values(nextHops))...) + bfdMap := make(map[string]string, bfdDstIPs.Len()) + for _, bfd := range bfdList { + if bfdIP == "" || bfd.LogicalPort != lrpName || !bfdDstIPs.Has(bfd.DstIP) { + staleBFDIDs.Insert(bfd.UUID) + } + if bfdIP == "" || (bfd.LogicalPort == lrpName && bfdDstIPs.Has(bfd.DstIP)) { + // TODO: update min_rx, min_tx and multiplier + if bfdIP != "" { + bfdIDs.Insert(bfd.UUID) + bfdMap[bfd.DstIP] = bfd.UUID + } + bfdDstIPs.Delete(bfd.DstIP) + } + } + if bfdIP != "" { + for _, dstIP := range bfdDstIPs.UnsortedList() { + bfd, err := c.OVNNbClient.CreateBFD(lrpName, dstIP, int(router.Spec.BFD.MinRX), int(router.Spec.BFD.MinTX), int(router.Spec.BFD.Multiplier), externalIDs) + if err != nil { + klog.Error(err) + return err + } + bfdIDs.Insert(bfd.UUID) + bfdMap[bfd.DstIP] = bfd.UUID + } + } + + // reconcile LR policy + if router.Spec.TrafficPolicy == kubeovnv1.TrafficPolicyLocal { + rules := make(map[string]string, len(nextHops)) + for nodeName, nexthop := range nextHops { + node, err := c.nodesLister.Get(nodeName) + if err != nil { + if k8serrors.IsNotFound(err) { + continue + } + klog.Errorf("failed to get node %s: %v", nodeName, err) + return err + } + portName := node.Annotations[util.PortNameAnnotation] + if portName == "" { + err = fmt.Errorf("node %s does not have port name annotation", nodeName) + klog.Error(err) + return err + } + localPgName := strings.ReplaceAll(portName, "-", ".") + rules[fmt.Sprintf("ip%d.src == $%s_ip%d && ip%d.src == $%s_ip%d", af, localPgName, af, af, pgName, af)] = nexthop + rules[fmt.Sprintf("ip%d.src == $%s_ip%d && ip%d.src == $%s", af, localPgName, af, af, asName)] = nexthop + } + policies, err := c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayLocalPolicyPriority, externalIDs, false) + if err != nil { + klog.Error(err) + return err + } + // update/delete existing policies + for _, policy := range policies { + nexthop := rules[policy.Match] + bfdSessions := set.New(bfdMap[nexthop]).Delete("") + if nexthop == "" { + if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { + err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) + klog.Error(err) + return err + } + } else { + var changed bool + if len(policy.Nexthops) != 1 || policy.Nexthops[0] != nexthop { + policy.Nexthops = []string{nexthop} + changed = true + } + if !bfdSessions.Equal(set.New(policy.BFDSessions...)) { + policy.BFDSessions = bfdSessions.UnsortedList() + changed = true + } + if changed { + if err = c.OVNNbClient.UpdateLogicalRouterPolicy(policy, &policy.Nexthops, &policy.BFDSessions); err != nil { + err = fmt.Errorf("failed to update logical router policy %s: %w", policy.UUID, err) + klog.Error(err) + return err + } + } + } + delete(rules, policy.Match) + } + // create new policies + for match, nexthop := range rules { + if err = c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayLocalPolicyPriority, match, + ovnnb.LogicalRouterPolicyActionReroute, []string{nexthop}, []string{bfdMap[nexthop]}, externalIDs); err != nil { + klog.Error(err) + return err + } + } + } else { + if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, util.EgressGatewayLocalPolicyPriority, externalIDs); err != nil { + klog.Error(err) + return err + } + } + policies, err := c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayPolicyPriority, externalIDs, false) + if err != nil { + klog.Error(err) + return err + } + matches := set.New( + fmt.Sprintf("ip%d.src == $%s_ip%d", af, pgName, af), + fmt.Sprintf("ip%d.src == $%s", af, asName), + ) + bfdIPs := set.New(slices.Collect(maps.Values(nextHops))...) + bfdSessions := bfdIDs.UnsortedList() + for _, policy := range policies { + if matches.Has(policy.Match) { + if !bfdIPs.Equal(set.New(policy.Nexthops...)) || !bfdIDs.Equal(set.New(policy.BFDSessions...)) { + policy.Nexthops, policy.BFDSessions = bfdIPs.UnsortedList(), bfdSessions + if err = c.OVNNbClient.UpdateLogicalRouterPolicy(policy, &policy.Nexthops, &policy.BFDSessions); err != nil { + err = fmt.Errorf("failed to update bfd sessions of logical router policy %s: %w", policy.UUID, err) + klog.Error(err) + return err + } + } + matches.Delete(policy.Match) + continue + } + if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { + err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) + klog.Error(err) + return err + } + } + for _, match := range matches.UnsortedList() { + if err = c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayPolicyPriority, match, + ovnnb.LogicalRouterPolicyActionReroute, bfdIPs.UnsortedList(), bfdSessions, externalIDs); err != nil { + klog.Error(err) + return err + } + } + + if router.Spec.BFD.Enabled { + // drop traffic if no nexthop is available + if policies, err = c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayDropPolicyPriority, externalIDs, false); err != nil { + klog.Error(err) + return err + } + matches = set.New( + fmt.Sprintf("ip%d.src == $%s_ip%d", af, pgName, af), + fmt.Sprintf("ip%d.src == $%s", af, asName), + ) + for _, policy := range policies { + if matches.Has(policy.Match) { + matches.Delete(policy.Match) + continue + } + if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { + err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) + klog.Error(err) + return err + } + } + for _, match := range matches.UnsortedList() { + if err = c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayDropPolicyPriority, match, + ovnnb.LogicalRouterPolicyActionDrop, nil, nil, externalIDs); err != nil { + klog.Error(err) + return err + } + } + } else if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, util.EgressGatewayDropPolicyPriority, externalIDs); err != nil { + klog.Error(err) + return err + } + + for _, bfdID := range staleBFDIDs.UnsortedList() { + if err = c.OVNNbClient.DeleteBFD(bfdID); err != nil { + err = fmt.Errorf("failed to delete bfd %s: %w", bfdID, err) + klog.Error(err) + return err + } + } + + return nil +} + +func berMergeNodeSelector(nodeSelector []kubeovnv1.BgpEdgeRouterNodeSelector) *corev1.NodeSelector { + if len(nodeSelector) == 0 { + return nil + } + + result := &corev1.NodeSelector{ + NodeSelectorTerms: make([]corev1.NodeSelectorTerm, len(nodeSelector)), + } + for i, selector := range nodeSelector { + result.NodeSelectorTerms[i] = corev1.NodeSelectorTerm{ + MatchExpressions: make([]corev1.NodeSelectorRequirement, len(selector.MatchExpressions), len(selector.MatchLabels)+len(selector.MatchExpressions)), + MatchFields: make([]corev1.NodeSelectorRequirement, len(selector.MatchFields)), + } + for j := range selector.MatchExpressions { + selector.MatchExpressions[j].DeepCopyInto(&result.NodeSelectorTerms[i].MatchExpressions[j]) + } + for _, key := range slices.Sorted(maps.Keys(selector.MatchLabels)) { + result.NodeSelectorTerms[i].MatchExpressions = append(result.NodeSelectorTerms[i].MatchExpressions, corev1.NodeSelectorRequirement{ + Key: key, + Operator: corev1.NodeSelectorOpIn, + Values: []string{selector.MatchLabels[key]}, + }) + } + for j := range selector.MatchFields { + selector.MatchFields[j].DeepCopyInto(&result.NodeSelectorTerms[i].MatchFields[j]) + } + } + + return result +} + +func bgpEdgeRouterInitContainerEnv(af int, internalGateway, externalGateway string, forwardSrc set.Set[string]) ([]corev1.EnvVar, error) { + if internalGateway == "" { + return nil, nil + } + + return []corev1.EnvVar{{ + Name: fmt.Sprintf("INTERNAL_GATEWAY_IPV%d", af), + Value: internalGateway, + }, { + Name: fmt.Sprintf("EXTERNAL_GATEWAY_IPV%d", af), + Value: externalGateway, + }, { + Name: fmt.Sprintf("NO_SNAT_SOURCES_IPV%d", af), + Value: strings.Join(forwardSrc.SortedList(), ","), + }}, nil +} + +func bgpEdgeRouterContainerBFDD(image, bfdIP string, minTX, minRX, multiplier int32) corev1.Container { + return corev1.Container{ + Name: "bfdd", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"bash", "/kube-ovn/start-bfdd.sh"}, + Env: []corev1.EnvVar{{ + Name: "POD_IPS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIPs", + }, + }, + }, { + Name: "BFD_PEER_IPS", + Value: bfdIP, + }, { + Name: "BFD_MIN_TX", + Value: strconv.Itoa(int(minTX)), + }, { + Name: "BFD_MIN_RX", + Value: strconv.Itoa(int(minRX)), + }, { + Name: "BFD_MULTI", + Value: strconv.Itoa(int(multiplier)), + }}, + // wait for the BFD process to be running and initialize the BFD configuration + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bash", "/kube-ovn/bfdd-prestart.sh"}, + }, + }, + InitialDelaySeconds: 1, + FailureThreshold: 1, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bfdd-control", "status"}, + }, + }, + InitialDelaySeconds: 1, + PeriodSeconds: 5, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bfdd-control", "status"}, + }, + }, + InitialDelaySeconds: 3, + PeriodSeconds: 3, + FailureThreshold: 1, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](65534), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_BIND_SERVICE", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "usr-local-sbin", + MountPath: "/usr/local/sbin", + }}, + } +} + +func (c *Controller) handleDelbgpEdgeRouter(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.bgpEdgeRouterKeyMutex.LockKey(key) + defer func() { _ = c.bgpEdgeRouterKeyMutex.UnlockKey(key) }() + + cachedGateway, err := c.bgpEdgeRouterLister.BgpEdgeRouters(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + err = fmt.Errorf("failed to get bgp-edge-router %s: %w", key, err) + klog.Error(err) + return err + } + return nil + } + + klog.Infof("handle deleting bgp-edge-router %s", key) + if err = c.cleanOVNForbgpEdgeRouter(key, cachedGateway.Spec.VPC); err != nil { + klog.Error(err) + return err + } + + router := cachedGateway.DeepCopy() + if controllerutil.RemoveFinalizer(router, util.KubeOVNControllerFinalizer) { + if _, err = c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouters(router.Namespace). + Update(context.Background(), router, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to remove finalizer from bgp-edge-router %s: %w", key, err) + klog.Error(err) + } + } + + return nil +} + +func (c *Controller) cleanOVNForbgpEdgeRouter(key, lrName string) error { + externalIDs := map[string]string{ + ovs.ExternalIDVendor: util.CniTypeName, + ovs.ExternalIDBgpEdgeRouter: key, + } + + bfdList, err := c.OVNNbClient.FindBFD(externalIDs) + if err != nil { + klog.Error(err) + return err + } + for _, bfd := range bfdList { + if err = c.OVNNbClient.DeleteBFD(bfd.UUID); err != nil { + klog.Error(err) + return err + } + } + + if lrName == "" { + lrName = c.config.ClusterRouter + } + if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, -1, externalIDs); err != nil { + klog.Error(err) + return err + } + if err = c.OVNNbClient.DeletePortGroup(vegPortGroupName(key)); err != nil { + klog.Error(err) + return err + } + for _, af := range [...]int{4, 6} { + if err = c.OVNNbClient.DeleteAddressSet(vegAddressSetName(key, af)); err != nil { + klog.Error(err) + return err + } + } + + return nil +} + +func berPortGroupName(key string) string { + hash := util.Sha256Hash([]byte(key)) + return "VEG." + hash[:12] +} + +func berAddressSetName(key string, af int) string { + hash := util.Sha256Hash([]byte(key)) + return fmt.Sprintf("VEG.%s.ipv%d", hash[:12], af) +} + +func (c *Controller) handlePodEventForbgpEdgeRouter(pod *corev1.Pod) error { + if !pod.DeletionTimestamp.IsZero() || pod.Annotations[util.AllocatedAnnotation] != "true" { + return nil + } + vpc := pod.Annotations[util.LogicalRouterAnnotation] + if vpc == "" { + return nil + } + + ns, err := c.namespacesLister.Get(pod.Namespace) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + klog.Errorf("failed to get namespace %s: %v", pod.Namespace, err) + utilruntime.HandleError(err) + return err + } + + gateways, err := c.bgpEdgeRouterLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list vpc egress gateways: %v", err) + utilruntime.HandleError(err) + return err + } + + for _, veg := range gateways { + if veg.VPC(c.config.ClusterRouter) != vpc { + continue + } + + for _, selector := range veg.Spec.Selectors { + if selector.NamespaceSelector != nil && !util.ObjectMatchesLabelSelector(ns, selector.NamespaceSelector) { + continue + } + if selector.PodSelector != nil && !util.ObjectMatchesLabelSelector(pod, selector.PodSelector) { + continue + } + c.addOrUpdateBgpEdgeRouterQueue.Add(cache.MetaObjectToName(veg).String()) + } + } + return nil +} + +func bgpEdgeRouterContainerBGP(speakerImage, gatewayName string, speakerParams *kubeovnv1.BgpEdgeRouterBGPConfig) (*corev1.Container, error) { + if speakerImage == "" { + return nil, errors.New("BGP speaker image must be specified") + } + if speakerParams == nil { + return nil, errors.New("BGP config must not be nil") + } + if speakerParams.ASN == 0 { + return nil, errors.New("ASN not set, but must be non-zero value") + } + if speakerParams.RemoteASN == 0 { + return nil, errors.New("remote ASN not set, but must be non-zero value") + } + if len(speakerParams.Neighbors) == 0 { + return nil, errors.New("no BGP neighbors specified") + } + + args := []string{} + if speakerParams.RouterID != "" { + args = append(args, "--router-id="+speakerParams.RouterID) + } + if speakerParams.Password != "" { + args = append(args, "--auth-password="+speakerParams.Password) + } + if speakerParams.EnableGracefulRestart { + args = append(args, "--graceful-restart") + } + if speakerParams.HoldTime != (metav1.Duration{}) { + args = append(args, "--holdtime="+speakerParams.HoldTime.Duration.String()) + } + + args = append(args, fmt.Sprintf("--cluster-as=%d", speakerParams.ASN)) + args = append(args, fmt.Sprintf("--neighbor-as=%d", speakerParams.RemoteASN)) + + var neighIPv4, neighIPv6 []string + for _, neighbor := range speakerParams.Neighbors { + switch util.CheckProtocol(neighbor) { + case kubeovnv1.ProtocolIPv4: + neighIPv4 = append(neighIPv4, neighbor) + case kubeovnv1.ProtocolIPv6: + neighIPv6 = append(neighIPv6, neighbor) + default: + return nil, fmt.Errorf("unsupported protocol for peer %s", neighbor) + } + } + if len(neighIPv4) > 0 { + args = append(args, "--neighbor-address="+strings.Join(neighIPv4, ",")) + } + if len(neighIPv6) > 0 { + args = append(args, "--neighbor-ipv6-address="+strings.Join(neighIPv6, ",")) + } + + args = append(args, speakerParams.ExtraArgs...) + + container := &corev1.Container{ + Name: "vpc-egress-router-speaker", + Image: speakerImage, + Command: []string{"/kube-ovn/kube-ovn-speaker"}, + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + { + Name: "EGRESS_GATEWAY_NAME", + Value: gatewayName, + }, + { + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + }, + Args: args, + } + + return container, nil +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 73fe6a70c46..70f6edc6457 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -289,7 +289,12 @@ type Controller struct { anpInformerFactory anpinformer.SharedInformerFactory // Database health check - dbFailureCount int + dbFailureCount int + bgpEdgeRouterLister kubeovnlister.BgpEdgeRouterLister + bgpEdgeRouterSynced cache.InformerSynced + addOrUpdateBgpEdgeRouterQueue workqueue.TypedRateLimitingInterface[string] + delBgpEdgeRouterQueue workqueue.TypedRateLimitingInterface[string] + bgpEdgeRouterKeyMutex keymutex.KeyMutex } func newTypedRateLimitingQueue[T comparable](name string, rateLimiter workqueue.TypedRateLimiter[T]) workqueue.TypedRateLimitingInterface[T] { @@ -350,6 +355,7 @@ func Run(ctx context.Context, config *Configuration) { vpcInformer := kubeovnInformerFactory.Kubeovn().V1().Vpcs() vpcNatGatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcNatGateways() vpcEgressGatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcEgressGateways() + bgpEdgeRouterInformer := kubeovnInformerFactory.Kubeovn().V1().BgpEdgeRouters() subnetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets() ippoolInformer := kubeovnInformerFactory.Kubeovn().V1().IPPools() ipInformer := kubeovnInformerFactory.Kubeovn().V1().IPs() @@ -744,6 +750,14 @@ func Run(ctx context.Context, config *Configuration) { util.LogFatalAndExit(err, "failed to add vpc nat gateway event handler") } + if _, err = bgpEdgeRouterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddBgpEdgeRouter, + UpdateFunc: controller.enqueueUpdateBgpEdgeRouter, + DeleteFunc: controller.enqueueDeleteBgpEdgeRouter, + }); err != nil { + util.LogFatalAndExit(err, "failed to add bgp edge router event handler") + } + if _, err = vpcEgressGatewayInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueAddVpcEgressGateway, UpdateFunc: controller.enqueueUpdateVpcEgressGateway, diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go index 1b6fe901470..9e340d64d9c 100644 --- a/pkg/ovs/ovn.go +++ b/pkg/ovs/ovn.go @@ -48,6 +48,7 @@ const ( ExternalIDVendor = "vendor" ExternalIDVpcEgressGateway = "vpc-egress-gateway" + ExternalIDBgpEdgeRouter = "bgp-edge-router" ) // NewLegacyClient init a legacy ovn client diff --git a/pkg/util/const.go b/pkg/util/const.go index 4cf39564fea..8566235d53b 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -114,6 +114,7 @@ const ( ACLActionsLogAnnotation = "ovn.kubernetes.io/log_acl_actions" VpcEgressGatewayLabel = "ovn.kubernetes.io/vpc-egress-gateway" + BgpEdgeRouterLabel = "ovn.kubernetes.io/bgp-edge-router" GenerateHashAnnotation = "ovn.kubernetes.io/generate-hash" VpcLastName = "ovn.kubernetes.io/last_vpc_name" From b18a909a629c2608b05d7dfeb31b14770d69006f Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Sun, 20 Jul 2025 21:08:24 -0700 Subject: [PATCH 06/66] name modifcation --- .../clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go index c168b9ab6d1..c61a2c4b090 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go @@ -74,7 +74,7 @@ type bgpEdgeRouters struct { func newBgpEdgeRouters(c *KubeovnV1Client, namespace string) *bgpEdgeRouters { return &bgpEdgeRouters{ gentype.NewClientWithList[*kubeovnv1.BgpEdgeRouter, *kubeovnv1.BgpEdgeRouterList]( - "bgp-edgerouters", + "bgp-edge-routers", c.RESTClient(), scheme.ParameterCodec, namespace, @@ -89,7 +89,7 @@ func (c *bgpEdgeRouters) GetScale(ctx context.Context, bgpEdgeRouterName string, result = &autoscalingv1.Scale{} err = c.GetClient().Get(). Namespace(c.GetNamespace()). - Resource("bgp-edgerouters"). + Resource("bgp-edge-routers"). Name(bgpEdgeRouterName). SubResource("scale"). VersionedParams(&options, scheme.ParameterCodec). From e429571ca7a4c2b4655ff91fcbbf51c4db9ab335 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Sun, 20 Jul 2025 23:15:13 -0700 Subject: [PATCH 07/66] bgp edge router mod and update-codegen.sh mod --- hack/update-codegen.sh | 8 +- .../kubeovn/v1/fake/fake_bgpedgerouter.go | 79 +++++++++++++++++++ .../kubeovn/v1/fake/fake_kubeovn_client.go | 4 + .../typed/kubeovn/v1/kubeovn_client.go | 5 ++ .../kubeovn/v1/bgpedgerouter.go | 17 ++-- 5 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouter.go diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index d048f706e4b..b31d3690272 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -4,12 +4,16 @@ set -o pipefail set -x SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. -CODEGEN_PKG=$GOPATH/src/k8s.io/code-generator +CODEGEN_PKG="/usr/local/pkg/mod/k8s.io/code-generator@v0.32.6" +MODULE=github.com/kubeovn/kube-ovn # generate the code with: # --output-base because this script should also be able to run inside the vendor dir of # k8s.io/kubernetes. The output-base is needed for the generators to output into the vendor dir # instead of the $GOPATH directly. For normal projects this can be dropped. -${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \ +# ${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \ +# github.com/kubeovn/kube-ovn/pkg/client github.com/kubeovn/kube-ovn/pkg/apis \ +# kubeovn:v1 +${CODEGEN_PKG}/kube_codegen.sh "deepcopy,client,informer,lister" \ github.com/kubeovn/kube-ovn/pkg/client github.com/kubeovn/kube-ovn/pkg/apis \ kubeovn:v1 diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouter.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouter.go new file mode 100644 index 00000000000..15bfe5383b1 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouter.go @@ -0,0 +1,79 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + context "context" + + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gentype "k8s.io/client-go/gentype" + testing "k8s.io/client-go/testing" +) + +// fakeBgpEdgeRouters implements BgpEdgeRouterInterface +type fakeBgpEdgeRouters struct { + *gentype.FakeClientWithList[*v1.BgpEdgeRouter, *v1.BgpEdgeRouterList] + Fake *FakeKubeovnV1 +} + +func newFakeBgpEdgeRouters(fake *FakeKubeovnV1, namespace string) kubeovnv1.BgpEdgeRouterInterface { + return &fakeBgpEdgeRouters{ + gentype.NewFakeClientWithList[*v1.BgpEdgeRouter, *v1.BgpEdgeRouterList]( + fake.Fake, + namespace, + v1.SchemeGroupVersion.WithResource("bgp-edge-routers"), + v1.SchemeGroupVersion.WithKind("BgpEdgeRouter"), + func() *v1.BgpEdgeRouter { return &v1.BgpEdgeRouter{} }, + func() *v1.BgpEdgeRouterList { return &v1.BgpEdgeRouterList{} }, + func(dst, src *v1.BgpEdgeRouterList) { dst.ListMeta = src.ListMeta }, + func(list *v1.BgpEdgeRouterList) []*v1.BgpEdgeRouter { return gentype.ToPointerSlice(list.Items) }, + func(list *v1.BgpEdgeRouterList, items []*v1.BgpEdgeRouter) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} + +// GetScale takes name of the bgpEdgeRouter, and returns the corresponding scale object, and an error if there is any. +func (c *fakeBgpEdgeRouters) GetScale(ctx context.Context, bgpEdgeRouterName string, options metav1.GetOptions) (result *autoscalingv1.Scale, err error) { + emptyResult := &autoscalingv1.Scale{} + obj, err := c.Fake. + Invokes(testing.NewGetSubresourceActionWithOptions(c.Resource(), c.Namespace(), "scale", bgpEdgeRouterName, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*autoscalingv1.Scale), err +} + +// UpdateScale takes the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *fakeBgpEdgeRouters) UpdateScale(ctx context.Context, bgpEdgeRouterName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (result *autoscalingv1.Scale, err error) { + emptyResult := &autoscalingv1.Scale{} + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceActionWithOptions(c.Resource(), "scale", c.Namespace(), scale, opts), &autoscalingv1.Scale{}) + + if obj == nil { + return emptyResult, err + } + return obj.(*autoscalingv1.Scale), err +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go index 8be54ee7e18..3745fc15417 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go @@ -112,6 +112,10 @@ func (c *FakeKubeovnV1) VpcNatGateways() v1.VpcNatGatewayInterface { return newFakeVpcNatGateways(c) } +func (c *FakeKubeovnV1) BgpEdgeRouters(namespace string) v1.BgpEdgeRouterInterface { + return newFakeBgpEdgeRouters(c, namespace) +} + // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. func (c *FakeKubeovnV1) RESTClient() rest.Interface { diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go index 24dc5ea5a1f..0af283168c1 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go @@ -49,6 +49,7 @@ type KubeovnV1Interface interface { VpcDnsesGetter VpcEgressGatewaysGetter VpcNatGatewaysGetter + BgpEdgeRoutersGetter } // KubeovnV1Client is used to interact with features provided by the kubeovn.io group. @@ -140,6 +141,10 @@ func (c *KubeovnV1Client) VpcNatGateways() VpcNatGatewayInterface { return newVpcNatGateways(c) } +func (c *KubeovnV1Client) BgpEdgeRouters(namespace string) BgpEdgeRouterInterface { + return newBgpEdgeRouters(c, namespace) +} + // NewForConfig creates a new KubeovnV1Client for the given config. // NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), // where httpClient was generated with rest.HTTPClientFor(c). diff --git a/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouter.go b/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouter.go index dfbd88dbf1c..3453626947a 100644 --- a/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouter.go +++ b/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouter.go @@ -42,32 +42,33 @@ type BgpEdgeRouterInformer interface { type bgpEdgeRouterInformer struct { factory internalinterfaces.SharedInformerFactory tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string } -// NewVlanInformer constructs a new informer for Vlan type. +// NewBgpEdgeRouterInformer constructs a new informer for BgpEdgeRouter type. // Always prefer using an informer factory to get a shared informer instead of getting an independent // one. This reduces memory footprint and number of connections to the server. -func NewBgpEdgeRouterInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { - return NewFilteredBgpEdgeRouterInformer(client, resyncPeriod, indexers, nil) +func NewBgpEdgeRouterInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredBgpEdgeRouterInformer(client, namespace, resyncPeriod, indexers, nil) } -// NewFilteredVlanInformer constructs a new informer for Vlan type. +// NewFilteredBgpEdgeRouterInformer constructs a new informer for BgpEdgeRouter type. // Always prefer using an informer factory to get a shared informer instead of getting an independent // one. This reduces memory footprint and number of connections to the server. -func NewFilteredBgpEdgeRouterInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { +func NewFilteredBgpEdgeRouterInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { return cache.NewSharedIndexInformer( &cache.ListWatch{ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.KubeovnV1().BgpEdgeRouters().List(context.TODO(), options) + return client.KubeovnV1().BgpEdgeRouters(namespace).List(context.TODO(), options) }, WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.KubeovnV1().BgpEdgeRouters().Watch(context.TODO(), options) + return client.KubeovnV1().BgpEdgeRouters(namespace).Watch(context.TODO(), options) }, }, &apiskubeovnv1.BgpEdgeRouter{}, @@ -77,7 +78,7 @@ func NewFilteredBgpEdgeRouterInformer(client versioned.Interface, resyncPeriod t } func (f *bgpEdgeRouterInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - return NewFilteredBgpEdgeRouterInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) + return NewFilteredBgpEdgeRouterInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) } func (f *bgpEdgeRouterInformer) Informer() cache.SharedIndexInformer { From 63eeefde04947c4e8d5582e54f746dd97a8a6ca8 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 21 Jul 2025 01:44:55 -0700 Subject: [PATCH 08/66] bgp edge router mod --- pkg/controller/bgp_edge_router.go | 24 ++++++++++++------------ pkg/controller/controller.go | 10 +++++++++- pkg/controller/pod.go | 7 +++++++ 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go index 1646544b135..4e045df8508 100644 --- a/pkg/controller/bgp_edge_router.go +++ b/pkg/controller/bgp_edge_router.go @@ -596,7 +596,7 @@ func (c *Controller) reconcilebgpEdgeRouterOVNRoutes(router *kubeovnv1.BgpEdgeRo } } key := cache.MetaObjectToName(router).String() - pgName := vegPortGroupName(key) + pgName := berPortGroupName(key) if err = c.OVNNbClient.CreatePortGroup(pgName, externalIDs); err != nil { err = fmt.Errorf("failed to create port group %s: %w", pgName, err) klog.Error(err) @@ -609,7 +609,7 @@ func (c *Controller) reconcilebgpEdgeRouterOVNRoutes(router *kubeovnv1.BgpEdgeRo } // reconcile OVN address set - asName := vegAddressSetName(key, af) + asName := berAddressSetName(key, af) if err = c.OVNNbClient.CreateAddressSet(asName, externalIDs); err != nil { err = fmt.Errorf("failed to create address set %s: %w", asName, err) klog.Error(err) @@ -922,7 +922,7 @@ func bgpEdgeRouterContainerBFDD(image, bfdIP string, minTX, minRX, multiplier in } } -func (c *Controller) handleDelbgpEdgeRouter(key string) error { +func (c *Controller) handleDelBgpEdgeRouter(key string) error { ns, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) @@ -985,12 +985,12 @@ func (c *Controller) cleanOVNForbgpEdgeRouter(key, lrName string) error { klog.Error(err) return err } - if err = c.OVNNbClient.DeletePortGroup(vegPortGroupName(key)); err != nil { + if err = c.OVNNbClient.DeletePortGroup(berPortGroupName(key)); err != nil { klog.Error(err) return err } for _, af := range [...]int{4, 6} { - if err = c.OVNNbClient.DeleteAddressSet(vegAddressSetName(key, af)); err != nil { + if err = c.OVNNbClient.DeleteAddressSet(berAddressSetName(key, af)); err != nil { klog.Error(err) return err } @@ -1001,15 +1001,15 @@ func (c *Controller) cleanOVNForbgpEdgeRouter(key, lrName string) error { func berPortGroupName(key string) string { hash := util.Sha256Hash([]byte(key)) - return "VEG." + hash[:12] + return "BER." + hash[:12] } func berAddressSetName(key string, af int) string { hash := util.Sha256Hash([]byte(key)) - return fmt.Sprintf("VEG.%s.ipv%d", hash[:12], af) + return fmt.Sprintf("BER.%s.ipv%d", hash[:12], af) } -func (c *Controller) handlePodEventForbgpEdgeRouter(pod *corev1.Pod) error { +func (c *Controller) handlePodEventForBgpEdgeRouter(pod *corev1.Pod) error { if !pod.DeletionTimestamp.IsZero() || pod.Annotations[util.AllocatedAnnotation] != "true" { return nil } @@ -1035,19 +1035,19 @@ func (c *Controller) handlePodEventForbgpEdgeRouter(pod *corev1.Pod) error { return err } - for _, veg := range gateways { - if veg.VPC(c.config.ClusterRouter) != vpc { + for _, ber := range gateways { + if ber.VPC(c.config.ClusterRouter) != vpc { continue } - for _, selector := range veg.Spec.Selectors { + for _, selector := range ber.Spec.Selectors { if selector.NamespaceSelector != nil && !util.ObjectMatchesLabelSelector(ns, selector.NamespaceSelector) { continue } if selector.PodSelector != nil && !util.ObjectMatchesLabelSelector(pod, selector.PodSelector) { continue } - c.addOrUpdateBgpEdgeRouterQueue.Add(cache.MetaObjectToName(veg).String()) + c.addOrUpdateBgpEdgeRouterQueue.Add(cache.MetaObjectToName(ber).String()) } } return nil diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 70f6edc6457..cf37d40e9af 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -420,6 +420,12 @@ func Run(ctx context.Context, config *Configuration) { delVpcEgressGatewayQueue: newTypedRateLimitingQueue("DeleteVpcEgressGateway", custCrdRateLimiter), vpcEgressGatewayKeyMutex: keymutex.NewHashed(numKeyLocks), + bgpEdgeRouterLister: bgpEdgeRouterInformer.Lister(), + bgpEdgeRouterSynced: bgpEdgeRouterInformer.Informer().HasSynced, + addOrUpdateBgpEdgeRouterQueue: newTypedRateLimitingQueue("AddOrUpdateBgpEdgeRouter", custCrdRateLimiter), + delBgpEdgeRouterQueue: newTypedRateLimitingQueue("DeleteBgpEdgeRouter", custCrdRateLimiter), + bgpEdgeRouterKeyMutex: keymutex.NewHashed(numKeyLocks), + subnetsLister: subnetInformer.Lister(), subnetSynced: subnetInformer.Informer().HasSynced, addOrUpdateSubnetQueue: newTypedRateLimitingQueue[string]("AddSubnet", nil), @@ -672,7 +678,7 @@ func Run(ctx context.Context, config *Configuration) { controller.vlanSynced, controller.podsSynced, controller.namespacesSynced, controller.nodesSynced, controller.serviceSynced, controller.endpointSlicesSynced, controller.deploymentsSynced, controller.configMapsSynced, controller.ovnEipSynced, controller.ovnFipSynced, controller.ovnSnatRuleSynced, - controller.ovnDnatRuleSynced, + controller.ovnDnatRuleSynced, controller.bgpEdgeRouterSynced, } if controller.config.EnableLb { cacheSyncs = append(cacheSyncs, controller.switchLBRuleSynced, controller.vpcDNSSynced) @@ -1200,6 +1206,8 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(runWorker("delete vpc nat gateway", c.delVpcNatGatewayQueue, c.handleDelVpcNatGw), time.Second, ctx.Done()) go wait.Until(runWorker("add/update vpc egress gateway", c.addOrUpdateVpcEgressGatewayQueue, c.handleAddOrUpdateVpcEgressGateway), time.Second, ctx.Done()) go wait.Until(runWorker("delete vpc egress gateway", c.delVpcEgressGatewayQueue, c.handleDelVpcEgressGateway), time.Second, ctx.Done()) + go wait.Until(runWorker("add/update bgp edge router", c.addOrUpdateBgpEdgeRouterQueue, c.handleAddOrUpdateBgpEdgeRouter), time.Second, ctx.Done()) + go wait.Until(runWorker("delete bgp edge router", c.delBgpEdgeRouterQueue, c.handleDelBgpEdgeRouter), time.Second, ctx.Done()) go wait.Until(runWorker("update fip for vpc nat gateway", c.updateVpcFloatingIPQueue, c.handleUpdateVpcFloatingIP), time.Second, ctx.Done()) go wait.Until(runWorker("update eip for vpc nat gateway", c.updateVpcEipQueue, c.handleUpdateVpcEip), time.Second, ctx.Done()) go wait.Until(runWorker("update dnat for vpc nat gateway", c.updateVpcDnatQueue, c.handleUpdateVpcDnat), time.Second, ctx.Done()) diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 8197c315eaa..2c1bb62c59c 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -235,6 +235,10 @@ func (c *Controller) enqueueAddPod(obj any) { if err = c.handlePodEventForVpcEgressGateway(p); err != nil { klog.Errorf("failed to handle pod event for vpc egress gateway: %v", err) } + + if err = c.handlePodEventForBgpEdgeRouter(p); err != nil { + klog.Errorf("failed to handle pod event for vpc egress gateway: %v", err) + } } func (c *Controller) enqueueDeletePod(obj any) { @@ -376,6 +380,9 @@ func (c *Controller) enqueueUpdatePod(oldObj, newObj any) { klog.Errorf("failed to handle pod event for vpc egress gateway: %v", err) } + if err = c.handlePodEventForBgpEdgeRouter(newPod); err != nil { + klog.Errorf("failed to handle pod event for bgp edge router: %v", err) + } // do not delete statefulset pod unless ownerReferences is deleted if isStateful && isStatefulSetPodToDel(c.config.KubeClient, newPod, statefulSetName, statefulSetUID) { go func() { From d01b7f38b9dc1ece5614ed4483f5a010cbf8c2d7 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Wed, 23 Jul 2025 03:43:42 -0700 Subject: [PATCH 09/66] Fixing Phase and Ready status, bgp edge router modification --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 419 ++++++++++++++++++ charts/kube-ovn/templates/ovn-CR.yaml | 2 + pkg/apis/kubeovn/v1/bgp-edge-router.go | 11 - .../informers/externalversions/generic.go | 2 + .../externalversions/kubeovn/v1/interface.go | 2 +- pkg/controller/bgp_edge_router.go | 46 +- pkg/controller/controller.go | 70 ++- pkg/controller/deployment.go | 12 + 8 files changed, 513 insertions(+), 51 deletions(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index 86cab8d00ab..a121d797f56 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3164,3 +3164,422 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgp-edge-routers.kubeovn.io +spec: + group: kubeovn.io + names: + plural: bgp-edge-routers + singular: bgp-edge-router + shortNames: + - bgp-er + - ber + kind: BgpEdgeRouter + listKind: BgpEdgeRouterList + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vpc + name: VPC + type: string + - jsonPath: .spec.replicas + name: REPLICAS + type: integer + - jsonPath: .spec.bfd.enabled + name: BFD ENABLED + type: boolean + - jsonPath: .spec.externalSubnet + name: EXTERNAL SUBNET + type: string + - jsonPath: .status.phase + name: PHASE + type: string + - jsonPath: .status.ready + name: READY + type: boolean + - jsonPath: .status.internalIPs + name: INTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.externalIPs + name: EXTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.workload.nodes + name: WORKING NODES + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + scale: + # specReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Spec.Replicas. + specReplicasPath: .spec.replicas + # statusReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Replicas. + statusReplicasPath: .status.replicas + # labelSelectorPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Selector. + labelSelectorPath: .status.labelSelector + schema: + openAPIV3Schema: + type: object + properties: + status: + properties: + replicas: + type: integer + format: int32 + labelSelector: + type: string + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + internalIPs: + items: + type: string + type: array + externalIPs: + items: + type: string + type: array + phase: + type: string + default: Pending + enum: + - Pending + - Processing + - Completed + ready: + type: boolean + default: false + workload: + type: object + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + nodes: + type: array + items: + type: string + required: + - conditions + - phase + type: object + spec: + type: object + required: + - externalSubnet + x-kubernetes-validations: + - rule: "!has(self.internalIPs) || size(self.internalIPs) == 0 || size(self.internalIPs) >= self.replicas" + message: 'Size of Internal IPs MUST be equal to or greater than Replicas' + fieldPath: ".internalIPs" + - rule: "!has(self.externalIPs) || size(self.externalIPs) == 0 || size(self.externalIPs) >= self.replicas" + message: 'Size of External IPs MUST be equal to or greater than Replicas' + fieldPath: ".externalIPs" + - rule: "size(self.policies) != 0 || size(self.selectors) != 0" + message: 'Each BGP Edeg Router MUST have at least one policy or selector' + properties: + replicas: + type: integer + format: int32 + default: 1 + minimum: 0 + maximum: 10 + prefix: + type: string + anyOf: + - pattern: ^$ + - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*[-\.]?$ + x-kubernetes-validations: + - rule: "self == oldSelf" + message: "This field is immutable." + vpc: + type: string + internalSubnet: + type: string + externalSubnet: + type: string + internalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + externalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + image: + type: string + bfd: + type: object + properties: + enabled: + type: boolean + default: false + minRX: + type: integer + format: int32 + default: 1000 + minTX: + type: integer + format: int32 + default: 1000 + multiplier: + type: integer + format: int32 + default: 3 + selectors: + type: array + items: + type: object + properties: + namespaceSelector: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + x-kubernetes-validations: + - rule: "size(self.matchLabels) != 0 || size(self.matchExpressions) != 0" + message: 'Each namespace selector MUST have at least one matchLabels or matchExpressions' + podSelector: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + x-kubernetes-validations: + - rule: "size(self.matchLabels) != 0 || size(self.matchExpressions) != 0" + message: 'Each pod selector MUST have at least one matchLabels or matchExpressions' + policies: + type: array + items: + type: object + properties: + snat: + type: boolean + default: false + ipBlocks: + type: array + x-kubernetes-list-type: set + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + - format: cidr + subnets: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + x-kubernetes-validations: + - rule: "size(self.ipBlocks) != 0 || size(self.subnets) != 0" + message: 'Each policy MUST have at least one ipBlock or subnet' + trafficPolicy: + type: string + enum: + - Local + - Cluster + default: Cluster + nodeSelector: + type: array + items: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator + bgp: + type: object + properties: + edgeRouterMode: + type: boolean + default: false + enabled: + type: boolean + default: false + image: + type: string + asn: + type: integer + format: int32 + minimum: 0 + remoteAsn: + type: integer + format: int32 + minimum: 0 + neighbors: + type: array + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + holdTime: + type: string + pattern: ^[0-9]+[smhd]$ + routerId: + type: string + # The routerId can be an IPv4 or IPv6 address, but we are not enforcing it here + # routerId could be nullable, so that we can use environment value to set it + # refer pkg/speaker/config.go#L186 + anyOf: + - format: ipv4 + - format: ipv6 + - pattern: '^$' + password: + type: string + enableGracefulRestart: + type: boolean + default: false + extraArgs: + type: array + items: + type: string \ No newline at end of file diff --git a/charts/kube-ovn/templates/ovn-CR.yaml b/charts/kube-ovn/templates/ovn-CR.yaml index d02a15a9d6e..c5f54a4ecee 100644 --- a/charts/kube-ovn/templates/ovn-CR.yaml +++ b/charts/kube-ovn/templates/ovn-CR.yaml @@ -50,6 +50,8 @@ rules: - vpc-dnses/status - qos-policies - qos-policies/status + - bgp-edge-routers + - bgp-edge-routers/status verbs: - "*" - apiGroups: diff --git a/pkg/apis/kubeovn/v1/bgp-edge-router.go b/pkg/apis/kubeovn/v1/bgp-edge-router.go index f3f67341033..404954bffa1 100644 --- a/pkg/apis/kubeovn/v1/bgp-edge-router.go +++ b/pkg/apis/kubeovn/v1/bgp-edge-router.go @@ -6,17 +6,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -type BerPhase string - -const ( - // PhasePending means the resource is pending and not processed yet - BerPhasePending BerPhase = "Pending" - // PhaseProcessing means the resource is being processed - BerPhaseProcessing BerPhase = "Processing" - // PhaseCompleted means the resource has been processed successfully - BerPhaseCompleted BerPhase = "Completed" -) - // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type BgpEdgeRouterList struct { metav1.TypeMeta `json:",inline"` diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index 01c67b1c70c..a6482a5d8c6 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -93,6 +93,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcDnses().Informer()}, nil case v1.SchemeGroupVersion.WithResource("vpc-egress-gateways"): return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcEgressGateways().Informer()}, nil + case v1.SchemeGroupVersion.WithResource("bgp-edge-routers"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().BgpEdgeRouters().Informer()}, nil case v1.SchemeGroupVersion.WithResource("vpc-nat-gateways"): return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcNatGateways().Informer()}, nil diff --git a/pkg/client/informers/externalversions/kubeovn/v1/interface.go b/pkg/client/informers/externalversions/kubeovn/v1/interface.go index ed144452d54..791dcaa93da 100644 --- a/pkg/client/informers/externalversions/kubeovn/v1/interface.go +++ b/pkg/client/informers/externalversions/kubeovn/v1/interface.go @@ -188,5 +188,5 @@ func (v *version) VpcNatGateways() VpcNatGatewayInformer { // BgpEdgeRouters returns a BgpEdgeRouterInformer. func (v *version) BgpEdgeRouters() BgpEdgeRouterInformer { - return &bgpEdgeRouterInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} + return &bgpEdgeRouterInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go index 4e045df8508..d1e29635e9f 100644 --- a/pkg/controller/bgp_edge_router.go +++ b/pkg/controller/bgp_edge_router.go @@ -116,7 +116,7 @@ func (c *Controller) handleAddOrUpdateBgpEdgeRouter(key string) error { bfdIPv4, bfdIPv6 = util.SplitStringIP(bfdIP) } - // reconcile the vpc egress gateway workload and get the route sources for later OVN resources reconciliation + // reconcile the bgp edge router workload and get the route sources for later OVN resources reconciliation attachmentNetworkName, ipv4Src, ipv6Src, deploy, err := c.reconcilebgpEdgeRouterWorkload(router, vpc, bfdIP, bfdIPv4, bfdIPv6) router.Status.Replicas = router.Spec.Replicas router.Status.LabelSelector = labels.FormatLabels(bgpEdgeRouterWorkloadLabels(router.Name)) @@ -157,7 +157,7 @@ func (c *Controller) handleAddOrUpdateBgpEdgeRouter(key string) error { return err } - // update gateway status including the internal/external IPs and the nodes where the pods are running + // update router status including the internal/external IPs and the nodes where the pods are running router.Status.Workload.Nodes = make([]string, 0, len(pods)) for _, pod := range pods { if len(pod.Status.PodIPs) == 0 { @@ -187,11 +187,11 @@ func (c *Controller) handleAddOrUpdateBgpEdgeRouter(key string) error { } // reconcile OVN routes - if err = c.reconcilebgpEdgeRouterOVNRoutes(router, 4, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv4, nodeNexthopIPv4, ipv4Src); err != nil { + if err = c.reconcileBgpEdgeRouterOVNRoutes(router, 4, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv4, nodeNexthopIPv4, ipv4Src); err != nil { klog.Error(err) return err } - if err = c.reconcilebgpEdgeRouterOVNRoutes(router, 6, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv6, nodeNexthopIPv6, ipv6Src); err != nil { + if err = c.reconcileBgpEdgeRouterOVNRoutes(router, 6, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv6, nodeNexthopIPv6, ipv6Src); err != nil { klog.Error(err) return err } @@ -249,7 +249,7 @@ func (c *Controller) reconcilebgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRou bgpImage = router.Spec.BGP.Image } if image == "" { - err := fmt.Errorf("no image specified for vpc egress gateway %s/%s", router.Namespace, router.Name) + err := fmt.Errorf("no image specified for bgp edge router %s/%s", router.Namespace, router.Name) klog.Error(err) return "", nil, nil, nil, err } @@ -270,7 +270,7 @@ func (c *Controller) reconcilebgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRou internalSubnet = vpc.Status.DefaultLogicalSwitch } if internalSubnet == "" { - err := fmt.Errorf("default subnet of vpc %s not found, please set internal subnet of the egress gateway", vpc.Name) + err := fmt.Errorf("default subnet of vpc %s not found, please set internal subnet of the bgp edge router", vpc.Name) klog.Error(err) return "", nil, nil, nil, err } @@ -343,7 +343,7 @@ func (c *Controller) reconcilebgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRou routes := util.NewPodRoutes() intGatewayIPv4, intGatewayIPv6 := util.SplitStringIP(intSubnet.Spec.Gateway) extGatewayIPv4, extGatewayIPv6 := util.SplitStringIP(extSubnet.Spec.Gateway) - // add routes for the VPC BFD Port so that the egress gateway can establish BFD session(s) with it + // add routes for the VPC BFD Port so that the bgp edge router can establish BFD session(s) with it routes.Add(util.OvnProvider, bfdIPv4, intGatewayIPv4) routes.Add(util.OvnProvider, bfdIPv6, intGatewayIPv6) // add routes for the internal networks @@ -389,7 +389,7 @@ func (c *Controller) reconcilebgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRou initEnv = append(initEnv, ipv6Env...) // generate workload - labels := vegWorkloadLabels(router.Name) + labels := bgpEdgeRouterWorkloadLabels(router.Name) deploy := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: router.Spec.Prefix + router.Name, @@ -469,24 +469,24 @@ func (c *Controller) reconcilebgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRou }, }, } - // set owner reference so that the workload will be deleted automatically when the vpc egress gateway is deleted + // set owner reference so that the workload will be deleted automatically when the bgp edge router is deleted if err = util.SetOwnerReference(router, deploy); err != nil { klog.Error(err) return attachmentNetworkName, nil, nil, nil, err } if bfdIP != "" { - // run BFD in the gateway container to establish BFD session(s) with the VPC BFD LRP + // run BFD in the router container to establish BFD session(s) with the VPC BFD LRP container := bgpEdgeRouterContainerBFDD(image, bfdIP, router.Spec.BFD.MinTX, router.Spec.BFD.MinRX, router.Spec.BFD.Multiplier) deploy.Spec.Template.Spec.Containers[0] = container } // bgp sidecar container logic if router.Spec.BGP.Enabled { - // run BGP in the gateway container + // run BGP in the router container bgpContainer, err := bgpEdgeRouterContainerBGP(bgpImage, router.Name, &router.Spec.BGP) if err != nil { - klog.Errorf("failed to create a BGP speaker container for gateway %s: %v", router.Name, err) + klog.Errorf("failed to create a BGP speaker container for router %s: %v", router.Name, err) return "", nil, nil, nil, err } deploy.Spec.Template.Spec.Containers = append(deploy.Spec.Template.Spec.Containers, *bgpContainer) @@ -504,7 +504,13 @@ func (c *Controller) reconcilebgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRou // replicas and the hash annotation should be excluded from hash calculation deploy.Spec.Replicas = ptr.To(router.Spec.Replicas) deploy.Annotations = map[string]string{util.GenerateHashAnnotation: hash} - if currentDeploy, err := c.deploymentsLister.Deployments(router.Namespace).Get(deploy.Name); err != nil { + + realDeploy, err := c.config.KubeClient.AppsV1().Deployments(router.Namespace).Get(context.Background(), deploy.Name, metav1.GetOptions{}) + if err == nil { + // 이미 존재함 + klog.Infof("Deployment %s/%s already exists in API server realDeploy name: %s", router.Namespace, deploy.Name, realDeploy.Name) + } + if currentDeploy, err := c.berDeploymentsLister.Deployments(router.Namespace).Get(deploy.Name); err != nil { if !k8serrors.IsNotFound(err) { err = fmt.Errorf("failed to get deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) klog.Error(err) @@ -535,7 +541,7 @@ func (c *Controller) reconcilebgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRou return attachmentNetworkName, intRouteDstIPv4, intRouteDstIPv6, deploy, nil } -func (c *Controller) reconcilebgpEdgeRouterOVNRoutes(router *kubeovnv1.BgpEdgeRouter, af int, lrName, lrpName, bfdIP string, nextHops map[string]string, sources set.Set[string]) error { +func (c *Controller) reconcileBgpEdgeRouterOVNRoutes(router *kubeovnv1.BgpEdgeRouter, af int, lrName, lrpName, bfdIP string, nextHops map[string]string, sources set.Set[string]) error { if len(nextHops) == 0 { return nil } @@ -1028,14 +1034,14 @@ func (c *Controller) handlePodEventForBgpEdgeRouter(pod *corev1.Pod) error { return err } - gateways, err := c.bgpEdgeRouterLister.List(labels.Everything()) + router, err := c.bgpEdgeRouterLister.List(labels.Everything()) if err != nil { - klog.Errorf("failed to list vpc egress gateways: %v", err) + klog.Errorf("failed to list bgp edge router: %v", err) utilruntime.HandleError(err) return err } - for _, ber := range gateways { + for _, ber := range router { if ber.VPC(c.config.ClusterRouter) != vpc { continue } @@ -1053,7 +1059,7 @@ func (c *Controller) handlePodEventForBgpEdgeRouter(pod *corev1.Pod) error { return nil } -func bgpEdgeRouterContainerBGP(speakerImage, gatewayName string, speakerParams *kubeovnv1.BgpEdgeRouterBGPConfig) (*corev1.Container, error) { +func bgpEdgeRouterContainerBGP(speakerImage, routerName string, speakerParams *kubeovnv1.BgpEdgeRouterBGPConfig) (*corev1.Container, error) { if speakerImage == "" { return nil, errors.New("BGP speaker image must be specified") } @@ -1108,14 +1114,14 @@ func bgpEdgeRouterContainerBGP(speakerImage, gatewayName string, speakerParams * args = append(args, speakerParams.ExtraArgs...) container := &corev1.Container{ - Name: "vpc-egress-router-speaker", + Name: "bgp-router-speaker", Image: speakerImage, Command: []string{"/kube-ovn/kube-ovn-speaker"}, ImagePullPolicy: corev1.PullIfNotPresent, Env: []corev1.EnvVar{ { Name: "EGRESS_GATEWAY_NAME", - Value: gatewayName, + Value: routerName, }, { Name: "POD_IP", diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index cf37d40e9af..e648e274166 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -107,6 +107,12 @@ type Controller struct { delVpcEgressGatewayQueue workqueue.TypedRateLimitingInterface[string] vpcEgressGatewayKeyMutex keymutex.KeyMutex + bgpEdgeRouterLister kubeovnlister.BgpEdgeRouterLister + bgpEdgeRouterSynced cache.InformerSynced + addOrUpdateBgpEdgeRouterQueue workqueue.TypedRateLimitingInterface[string] + delBgpEdgeRouterQueue workqueue.TypedRateLimitingInterface[string] + bgpEdgeRouterKeyMutex keymutex.KeyMutex + switchLBRuleLister kubeovnlister.SwitchLBRuleLister switchLBRuleSynced cache.InformerSynced addSwitchLBRuleQueue workqueue.TypedRateLimitingInterface[string] @@ -233,6 +239,9 @@ type Controller struct { deploymentsLister appsv1.DeploymentLister deploymentsSynced cache.InformerSynced + berDeploymentsLister appsv1.DeploymentLister + berDeploymentsSynced cache.InformerSynced + npsLister netv1.NetworkPolicyLister npsSynced cache.InformerSynced updateNpQueue workqueue.TypedRateLimitingInterface[string] @@ -281,20 +290,16 @@ type Controller struct { netAttachSynced cache.InformerSynced netAttachInformerFactory netAttach.SharedInformerFactory - recorder record.EventRecorder - informerFactory kubeinformers.SharedInformerFactory - cmInformerFactory kubeinformers.SharedInformerFactory - deployInformerFactory kubeinformers.SharedInformerFactory - kubeovnInformerFactory kubeovninformer.SharedInformerFactory - anpInformerFactory anpinformer.SharedInformerFactory + recorder record.EventRecorder + informerFactory kubeinformers.SharedInformerFactory + cmInformerFactory kubeinformers.SharedInformerFactory + deployInformerFactory kubeinformers.SharedInformerFactory + berDeployInformerFactory kubeinformers.SharedInformerFactory + kubeovnInformerFactory kubeovninformer.SharedInformerFactory + anpInformerFactory anpinformer.SharedInformerFactory // Database health check - dbFailureCount int - bgpEdgeRouterLister kubeovnlister.BgpEdgeRouterLister - bgpEdgeRouterSynced cache.InformerSynced - addOrUpdateBgpEdgeRouterQueue workqueue.TypedRateLimitingInterface[string] - delBgpEdgeRouterQueue workqueue.TypedRateLimitingInterface[string] - bgpEdgeRouterKeyMutex keymutex.KeyMutex + dbFailureCount int } func newTypedRateLimitingQueue[T comparable](name string, rateLimiter workqueue.TypedRateLimiter[T]) workqueue.TypedRateLimitingInterface[T] { @@ -321,6 +326,11 @@ func Run(ctx context.Context, config *Configuration) { util.LogFatalAndExit(err, "failed to create label selector for vpc egress gateway workload") } + berSelector, berErr := labels.Parse(util.BgpEdgeRouterLabel) + if berErr != nil { + util.LogFatalAndExit(berErr, "failed to create label selector for bgp edge router workload") + } + informerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeFactoryClient, 0, kubeinformers.WithTweakListOptions(func(listOption *metav1.ListOptions) { listOption.AllowWatchBookmarks = true @@ -335,6 +345,12 @@ func Run(ctx context.Context, config *Configuration) { listOption.AllowWatchBookmarks = true listOption.LabelSelector = selector.String() })) + // deployment informer used to list/watch bgp edge router workloads + berDeployInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeFactoryClient, 0, + kubeinformers.WithTweakListOptions(func(listOption *metav1.ListOptions) { + listOption.AllowWatchBookmarks = true + listOption.LabelSelector = berSelector.String() + })) kubeovnInformerFactory := kubeovninformer.NewSharedInformerFactoryWithOptions(config.KubeOvnFactoryClient, 0, kubeovninformer.WithTweakListOptions(func(listOption *metav1.ListOptions) { listOption.AllowWatchBookmarks = true @@ -373,6 +389,7 @@ func Run(ctx context.Context, config *Configuration) { serviceInformer := informerFactory.Core().V1().Services() endpointSliceInformer := informerFactory.Discovery().V1().EndpointSlices() deploymentInformer := deployInformerFactory.Apps().V1().Deployments() + berDeploymentInformer := berDeployInformerFactory.Apps().V1().Deployments() qosPolicyInformer := kubeovnInformerFactory.Kubeovn().V1().QoSPolicies() configMapInformer := cmInformerFactory.Core().V1().ConfigMaps() npInformer := informerFactory.Networking().V1().NetworkPolicies() @@ -529,6 +546,9 @@ func Run(ctx context.Context, config *Configuration) { deploymentsLister: deploymentInformer.Lister(), deploymentsSynced: deploymentInformer.Informer().HasSynced, + berDeploymentsLister: berDeploymentInformer.Lister(), + berDeploymentsSynced: berDeploymentInformer.Informer().HasSynced, + qosPoliciesLister: qosPolicyInformer.Lister(), qosPolicySynced: qosPolicyInformer.Informer().HasSynced, addQoSPolicyQueue: newTypedRateLimitingQueue("AddQoSPolicy", custCrdRateLimiter), @@ -582,12 +602,13 @@ func Run(ctx context.Context, config *Configuration) { netAttachSynced: netAttachInformer.Informer().HasSynced, netAttachInformerFactory: attachNetInformerFactory, - recorder: recorder, - informerFactory: informerFactory, - cmInformerFactory: cmInformerFactory, - deployInformerFactory: deployInformerFactory, - kubeovnInformerFactory: kubeovnInformerFactory, - anpInformerFactory: anpInformerFactory, + recorder: recorder, + informerFactory: informerFactory, + cmInformerFactory: cmInformerFactory, + deployInformerFactory: deployInformerFactory, + berDeployInformerFactory: berDeployInformerFactory, + kubeovnInformerFactory: kubeovnInformerFactory, + anpInformerFactory: anpInformerFactory, } if controller.OVNNbClient, err = ovs.NewOvnNbClient( @@ -664,6 +685,7 @@ func Run(ctx context.Context, config *Configuration) { controller.informerFactory.Start(ctx.Done()) controller.cmInformerFactory.Start(ctx.Done()) controller.deployInformerFactory.Start(ctx.Done()) + controller.berDeployInformerFactory.Start(ctx.Done()) controller.kubeovnInformerFactory.Start(ctx.Done()) controller.anpInformerFactory.Start(ctx.Done()) controller.StartKubevirtInformerFactory(ctx, kubevirtInformerFactory) @@ -676,7 +698,7 @@ func Run(ctx context.Context, config *Configuration) { controller.ipSynced, controller.virtualIpsSynced, controller.iptablesEipSynced, controller.iptablesFipSynced, controller.iptablesDnatRuleSynced, controller.iptablesSnatRuleSynced, controller.vlanSynced, controller.podsSynced, controller.namespacesSynced, controller.nodesSynced, - controller.serviceSynced, controller.endpointSlicesSynced, controller.deploymentsSynced, controller.configMapsSynced, + controller.serviceSynced, controller.endpointSlicesSynced, controller.deploymentsSynced, controller.berDeploymentsSynced, controller.configMapsSynced, controller.ovnEipSynced, controller.ovnFipSynced, controller.ovnSnatRuleSynced, controller.ovnDnatRuleSynced, controller.bgpEdgeRouterSynced, } @@ -740,6 +762,13 @@ func Run(ctx context.Context, config *Configuration) { util.LogFatalAndExit(err, "failed to add deployment event handler") } + if _, err = berDeploymentInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddDeployment, + UpdateFunc: controller.enqueueUpdateDeployment, + }); err != nil { + util.LogFatalAndExit(err, "failed to add deployment event handler") + } + if _, err = vpcInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueAddVpc, UpdateFunc: controller.enqueueUpdateVpc, @@ -1113,6 +1142,9 @@ func (c *Controller) shutdown() { c.addOrUpdateVpcEgressGatewayQueue.ShutDown() c.delVpcEgressGatewayQueue.ShutDown() + c.addOrUpdateBgpEdgeRouterQueue.ShutDown() + c.delBgpEdgeRouterQueue.ShutDown() + if c.config.EnableLb { c.addSwitchLBRuleQueue.ShutDown() c.delSwitchLBRuleQueue.ShutDown() diff --git a/pkg/controller/deployment.go b/pkg/controller/deployment.go index c0828e47fe2..edc5b031544 100644 --- a/pkg/controller/deployment.go +++ b/pkg/controller/deployment.go @@ -15,6 +15,8 @@ var ( deploymentKind string vpcEgressGatewayGroupVersion string vpcEgressGatewayKind string + bgpEdgeRouterGroupVersion string + bgpEdgeRouterKind string ) func init() { @@ -27,6 +29,11 @@ func init() { gvk = kubeovnv1.SchemeGroupVersion.WithKind(name) vpcEgressGatewayGroupVersion = gvk.GroupVersion().String() vpcEgressGatewayKind = gvk.Kind + + name = reflect.TypeOf(&kubeovnv1.BgpEdgeRouter{}).Elem().Name() + gvk = kubeovnv1.SchemeGroupVersion.WithKind(name) + bgpEdgeRouterGroupVersion = gvk.GroupVersion().String() + bgpEdgeRouterKind = gvk.Kind } func (c *Controller) enqueueAddDeployment(obj any) { @@ -37,6 +44,11 @@ func (c *Controller) enqueueAddDeployment(obj any) { klog.V(3).Infof("enqueue update vpc-egress-gateway %s", key) c.addOrUpdateVpcEgressGatewayQueue.Add(key) return + } else if ref.APIVersion == bgpEdgeRouterGroupVersion && ref.Kind == bgpEdgeRouterKind { + key := types.NamespacedName{Namespace: deploy.Namespace, Name: ref.Name}.String() + klog.V(3).Infof("enqueue update bgp-edge-router %s", key) + c.addOrUpdateBgpEdgeRouterQueue.Add(key) + return } } } From 1e1a4a3d77d78f1f369ff5f8d06dcaa45fb8b2b0 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Wed, 23 Jul 2025 23:38:02 -0700 Subject: [PATCH 10/66] ber container /var/log/kube-ovn volumeMount issue is resolved --- pkg/controller/bgp_edge_router.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go index d1e29635e9f..c826aeff131 100644 --- a/pkg/controller/bgp_edge_router.go +++ b/pkg/controller/bgp_edge_router.go @@ -458,12 +458,20 @@ func (c *Controller) reconcilebgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRou MountPath: "/usr/local/sbin", }}, }}, - Volumes: []corev1.Volume{{ - Name: "usr-local-sbin", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, + Volumes: []corev1.Volume{ + { + Name: "usr-local-sbin", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, }, - }}, + { + Name: "kube-ovn-logs", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, TerminationGracePeriodSeconds: ptr.To[int64](0), }, }, @@ -1133,6 +1141,12 @@ func bgpEdgeRouterContainerBGP(speakerImage, routerName string, speakerParams *k }, }, Args: args, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "kube-ovn-logs", + MountPath: "/var/log/kube-ovn", + }, + }, } return container, nil From 9aac688815187d7a5364237506b25a44df46a873 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 28 Jul 2025 05:11:59 -0700 Subject: [PATCH 11/66] unecessary code deleted --- pkg/controller/bgp_edge_router.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go index c826aeff131..c9faa0d1734 100644 --- a/pkg/controller/bgp_edge_router.go +++ b/pkg/controller/bgp_edge_router.go @@ -117,7 +117,7 @@ func (c *Controller) handleAddOrUpdateBgpEdgeRouter(key string) error { } // reconcile the bgp edge router workload and get the route sources for later OVN resources reconciliation - attachmentNetworkName, ipv4Src, ipv6Src, deploy, err := c.reconcilebgpEdgeRouterWorkload(router, vpc, bfdIP, bfdIPv4, bfdIPv6) + attachmentNetworkName, ipv4Src, ipv6Src, deploy, err := c.reconcileBgpEdgeRouterWorkload(router, vpc, bfdIP, bfdIPv4, bfdIPv6) router.Status.Replicas = router.Spec.Replicas router.Status.LabelSelector = labels.FormatLabels(bgpEdgeRouterWorkloadLabels(router.Name)) if err != nil { @@ -239,7 +239,7 @@ func (c *Controller) updatebgpEdgeRouterStatus(router *kubeovnv1.BgpEdgeRouter) } // create or update bgp edge router workload -func (c *Controller) reconcilebgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRouter, vpc *kubeovnv1.Vpc, bfdIP, bfdIPv4, bfdIPv6 string) (string, set.Set[string], set.Set[string], *appsv1.Deployment, error) { +func (c *Controller) reconcileBgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRouter, vpc *kubeovnv1.Vpc, bfdIP, bfdIPv4, bfdIPv6 string) (string, set.Set[string], set.Set[string], *appsv1.Deployment, error) { image := c.config.Image bgpImage := c.config.Image if router.Spec.Image != "" { @@ -513,11 +513,6 @@ func (c *Controller) reconcilebgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRou deploy.Spec.Replicas = ptr.To(router.Spec.Replicas) deploy.Annotations = map[string]string{util.GenerateHashAnnotation: hash} - realDeploy, err := c.config.KubeClient.AppsV1().Deployments(router.Namespace).Get(context.Background(), deploy.Name, metav1.GetOptions{}) - if err == nil { - // 이미 존재함 - klog.Infof("Deployment %s/%s already exists in API server realDeploy name: %s", router.Namespace, deploy.Name, realDeploy.Name) - } if currentDeploy, err := c.berDeploymentsLister.Deployments(router.Namespace).Get(deploy.Name); err != nil { if !k8serrors.IsNotFound(err) { err = fmt.Errorf("failed to get deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) @@ -957,7 +952,7 @@ func (c *Controller) handleDelBgpEdgeRouter(key string) error { } klog.Infof("handle deleting bgp-edge-router %s", key) - if err = c.cleanOVNForbgpEdgeRouter(key, cachedGateway.Spec.VPC); err != nil { + if err = c.cleanOVNForBgpEdgeRouter(key, cachedGateway.Spec.VPC); err != nil { klog.Error(err) return err } @@ -974,7 +969,7 @@ func (c *Controller) handleDelBgpEdgeRouter(key string) error { return nil } -func (c *Controller) cleanOVNForbgpEdgeRouter(key, lrName string) error { +func (c *Controller) cleanOVNForBgpEdgeRouter(key, lrName string) error { externalIDs := map[string]string{ ovs.ExternalIDVendor: util.CniTypeName, ovs.ExternalIDBgpEdgeRouter: key, From 40f4826fbea273fceadc8d7668563db21a8a09c7 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 28 Jul 2025 06:41:13 -0700 Subject: [PATCH 12/66] bgp edge router advertisement mod --- .../bgp_edge_router_advertisement.go | 941 ++++++++++++++++++ 1 file changed, 941 insertions(+) create mode 100644 pkg/controller/bgp_edge_router_advertisement.go diff --git a/pkg/controller/bgp_edge_router_advertisement.go b/pkg/controller/bgp_edge_router_advertisement.go new file mode 100644 index 00000000000..33aa06e7357 --- /dev/null +++ b/pkg/controller/bgp_edge_router_advertisement.go @@ -0,0 +1,941 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "maps" + "reflect" + "slices" + "strconv" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + "k8s.io/utils/set" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +func (c *Controller) enqueueAddBgpEdgeRouterAdvertisement(obj any) { + key := cache.MetaObjectToName(obj.(*kubeovnv1.BgpEdgeRouterAdvertisement)).String() + klog.V(3).Infof("enqueue add bgp-edge-router-advertisement %s", key) + c.addBgpEdgeRouterAdvertisementQueue.Add(key) +} + +func (c *Controller) enqueueUpdateBgpEdgeRouterAdvertisement(_, newObj any) { + key := cache.MetaObjectToName(newObj.(*kubeovnv1.BgpEdgeRouterAdvertisement)).String() + klog.V(3).Infof("enqueue update bgp-edge-router-advertisement %s", key) + c.updateBgpEdgeRouterAdvertisementQueue.Add(key) +} + +func (c *Controller) enqueueDeleteBgpEdgeRouterAdvertisement(obj any) { + key := cache.MetaObjectToName(obj.(*kubeovnv1.BgpEdgeRouterAdvertisement)).String() + klog.V(3).Infof("enqueue delete bgp-edge-router-advertisement %s", key) + c.delBgpEdgeRouterAdvertisementQueue.Add(key) +} + +func bgpEdgeRouterAdvertisementWorkloadLabels(bgpEdgeRouterAdvertisementName string) map[string]string { + return map[string]string{"app": "bgp-edge-router-advertisement", util.BgpEdgeRouterLabel: bgpEdgeRouterAdvertisementName} +} + +func (c *Controller) handleAddOrUpdateBgpEdgeRouterAdvertisement(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.bgpEdgeRouterAdvertisementKeyMutex.LockKey(key) + defer func() { _ = c.bgpEdgeRouterAdvertisementKeyMutex.UnlockKey(key) }() + + cachedAdvertisement, err := c.bgpEdgeRouterAdvertisementLister.BgpEdgeRouterAdvertisements(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + if !cachedAdvertisement.DeletionTimestamp.IsZero() { + c.delBgpEdgeRouterAdvertisementQueue.Add(key) + return nil + } + + klog.Infof("reconciling bgp-edge-router-advertisement %s", key) + advertisement := cachedAdvertisement.DeepCopy() + + subnetName := advertisement.Spec.Subnets + if subnetName == "" { + subnetName = c.config.ClusterRouter + } + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + klog.Error(err) + return err + } + + if controllerutil.AddFinalizer(advertisement, util.KubeOVNControllerFinalizer) { + updatedAdvertisement, err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouterAdvertisements(advertisement.Namespace). + Update(context.Background(), advertisement, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to add finalizer for bgp-edge-router %s/%s: %w", advertisement.Namespace, advertisement.Name, err) + klog.Error(err) + return err + } + advertisement = updatedAdvertisement + } + + // reconcile the bgp edge router workload and get the route sources for later OVN resources reconciliation + deploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(advertisement.Spec.BerName) + if err != nil { + klog.Error(err) + return err + } + + advertisement.Status.Workload.APIVersion = deploy.APIVersion + advertisement.Status.Workload.Kind = deploy.Kind + advertisement.Status.Workload.Name = deploy.Name + advertisement.Status.Workload.Nodes = nil + nodeNexthopIPv4 := make(map[string]string, int(advertisement.Spec.Replicas)) + nodeNexthopIPv6 := make(map[string]string, int(advertisement.Spec.Replicas)) + ready := util.DeploymentIsReady(deploy) + if !ready { + readyErr = fmt.Sprintf("Deployment %s is not ready", deploy.Kind, deploy.Name) + klog.Error(readyErr) + return fmt.Errorf(readyErr) + } + // get the pods of the deployment to collect the pod IPs + podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) + if err != nil { + err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return err + } + + pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) + if err != nil { + err = fmt.Errorf("failed to list pods of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return err + } + + // update router status including the internal/external IPs and the nodes where the pods are running + router.Status.Workload.Nodes = make([]string, 0, len(pods)) + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + extIPs, err := util.PodAttachmentIPs(pod, attachmentNetworkName) + if err != nil { + klog.Error(err) + continue + } + + ips := util.PodIPs(*pod) + ipv4, ipv6 := util.SplitIpsByProtocol(ips) + if len(ipv4) != 0 { + nodeNexthopIPv4[pod.Spec.NodeName] = ipv4[0] + } + if len(ipv6) != 0 { + nodeNexthopIPv6[pod.Spec.NodeName] = ipv6[0] + } + router.Status.InternalIPs = append(router.Status.InternalIPs, strings.Join(ips, ",")) + router.Status.ExternalIPs = append(router.Status.ExternalIPs, strings.Join(extIPs, ",")) + router.Status.Workload.Nodes = append(router.Status.Workload.Nodes, pod.Spec.NodeName) + } + + // reconcile OVN routes + if err = c.reconcileBgpEdgeRouterOVNRoutes(router, 4, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv4, nodeNexthopIPv4, ipv4Src); err != nil { + klog.Error(err) + return err + } + if err = c.reconcileBgpEdgeRouterOVNRoutes(router, 6, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv6, nodeNexthopIPv6, ipv6Src); err != nil { + klog.Error(err) + return err + } + + klog.Infof("finished reconciling bgp-edge-router-advertisement %s", key) + + return nil +} + +// create or update bgp edge router workload +func (c *Controller) reconcileBgpEdgeRouterWorkload(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement, vpc *kubeovnv1.Vpc) (string, set.Set[string], set.Set[string], *appsv1.Deployment, error) { + // generate init container environment variables + // the init container is responsible for adding routes and SNAT rules to the pod network namespace + initEnv, err := bgpEdgeRouterInitContainerEnv(4, intGatewayIPv4, extGatewayIPv4, ipv4ForwardSrc) + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + ipv6Env, err := bgpEdgeRouterInitContainerEnv(6, intGatewayIPv6, extGatewayIPv6, ipv6ForwardSrc) + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + initEnv = append(initEnv, ipv6Env...) + + // generate workload + labels := bgpEdgeRouterWorkloadLabels(router.Name) + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: router.Spec.Prefix + router.Name, + Namespace: router.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: ptr.To(intstr.FromInt(1)), + MaxSurge: ptr.To(intstr.FromInt(0)), + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: berMergeNodeSelector(router.Spec.NodeSelector), + }, + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: corev1.LabelHostname, + }}, + }, + }, + InitContainers: []corev1.Container{{ + Name: "init", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"bash", "/kube-ovn/init-vpc-egress-gateway.sh"}, + Env: initEnv, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "usr-local-sbin", + MountPath: "/usr/local/sbin", + }}, + }}, + Containers: []corev1.Container{{ + Name: "gateway", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sleep", "infinity"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](65534), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "usr-local-sbin", + MountPath: "/usr/local/sbin", + }}, + }}, + Volumes: []corev1.Volume{ + { + Name: "usr-local-sbin", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "kube-ovn-logs", + VolumeSource: corev1.VolumeSource{}, + }, + }, + TerminationGracePeriodSeconds: ptr.To[int64](0), + }, + }, + }, + } + // set owner reference so that the workload will be deleted automatically when the bgp edge router is deleted + if err = util.SetOwnerReference(router, deploy); err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + + // bgp sidecar container logic + if router.Spec.BGP.Enabled { + // run BGP in the router container + bgpContainer, err := bgpEdgeRouterContainerBGP(bgpImage, router.Name, &router.Spec.BGP) + if err != nil { + klog.Errorf("failed to create a BGP speaker container for router %s: %v", router.Name, err) + return "", nil, nil, nil, err + } + deploy.Spec.Template.Spec.Containers = append(deploy.Spec.Template.Spec.Containers, *bgpContainer) + } + + // generate hash for the workload to determine whether to update the existing workload or not + hash, err := util.Sha256HashObject(deploy) + if err != nil { + err = fmt.Errorf("failed to hash generated deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + + hash = hash[:12] + // replicas and the hash annotation should be excluded from hash calculation + deploy.Spec.Replicas = ptr.To(router.Spec.Replicas) + deploy.Annotations = map[string]string{util.GenerateHashAnnotation: hash} + + if currentDeploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(deploy.Name); err != nil { + if !k8serrors.IsNotFound(err) { + err = fmt.Errorf("failed to get deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + if deploy, err = c.config.KubeClient.AppsV1().Deployments(router.Namespace). + Create(context.Background(), deploy, metav1.CreateOptions{}); err != nil { + err = fmt.Errorf("failed to create deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + } else if !reflect.DeepEqual(currentDeploy.Spec.Replicas, deploy.Spec.Replicas) || + currentDeploy.Annotations[util.GenerateHashAnnotation] != hash { + // update the deployment if replicas or hash annotation is changed + if deploy, err = c.config.KubeClient.AppsV1().Deployments(router.Namespace). + Update(context.Background(), deploy, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to update deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + } else { + // no need to create or update the deployment + deploy = currentDeploy + } + + // return the source CIDR blocks for later OVN resources reconciliation + deploy.APIVersion, deploy.Kind = deploymentGroupVersion, deploymentKind + return attachmentNetworkName, intRouteDstIPv4, intRouteDstIPv6, deploy, nil +} + +func (c *Controller) reconcileBgpEdgeRouterOVNRoutes(router *kubeovnv1.BgpEdgeRouter, af int, lrName, lrpName, bfdIP string, nextHops map[string]string, sources set.Set[string]) error { + if len(nextHops) == 0 { + return nil + } + + externalIDs := map[string]string{ + ovs.ExternalIDVendor: util.CniTypeName, + ovs.ExternalIDBgpEdgeRouter: fmt.Sprintf("%s/%s", router.Namespace, router.Name), + "af": strconv.Itoa(af), + } + bfdList, err := c.OVNNbClient.FindBFD(externalIDs) + if err != nil { + klog.Error(err) + return err + } + + // reconcile OVN port group + ports := set.New[string]() + for _, selector := range router.Spec.Selectors { + sel := labels.Everything() + if selector.NamespaceSelector != nil { + if sel, err = metav1.LabelSelectorAsSelector(selector.NamespaceSelector); err != nil { + err = fmt.Errorf("failed to create label selector for namespace selector %#v: %w", selector.NamespaceSelector, err) + klog.Error(err) + return err + } + } + namespaces, err := c.namespacesLister.List(sel) + if err != nil { + err = fmt.Errorf("failed to list namespaces with selector %s: %w", sel, err) + klog.Error(err) + return err + } + sel = labels.Everything() + if selector.PodSelector != nil { + if sel, err = metav1.LabelSelectorAsSelector(selector.PodSelector); err != nil { + err = fmt.Errorf("failed to create label selector for pod selector %#v: %w", selector.PodSelector, err) + klog.Error(err) + return err + } + } + for _, ns := range namespaces { + pods, err := c.podsLister.Pods(ns.Name).List(sel) + if err != nil { + err = fmt.Errorf("failed to list pods with selector %s in namespace %s: %w", sel, ns.Name, err) + klog.Error(err) + return err + } + for _, pod := range pods { + if pod.Spec.HostNetwork || + pod.Annotations[util.AllocatedAnnotation] != "true" || + pod.Annotations[util.LogicalRouterAnnotation] != router.VPC(c.config.ClusterRouter) || + !isPodAlive(pod) { + continue + } + podName := c.getNameByPod(pod) + ports.Insert(ovs.PodNameToPortName(podName, pod.Namespace, util.OvnProvider)) + } + } + } + key := cache.MetaObjectToName(router).String() + pgName := berPortGroupName(key) + if err = c.OVNNbClient.CreatePortGroup(pgName, externalIDs); err != nil { + err = fmt.Errorf("failed to create port group %s: %w", pgName, err) + klog.Error(err) + return err + } + if err = c.OVNNbClient.PortGroupSetPorts(pgName, ports.UnsortedList()); err != nil { + err = fmt.Errorf("failed to set ports of port group %s: %w", pgName, err) + klog.Error(err) + return err + } + + // reconcile OVN address set + asName := berAddressSetName(key, af) + if err = c.OVNNbClient.CreateAddressSet(asName, externalIDs); err != nil { + err = fmt.Errorf("failed to create address set %s: %w", asName, err) + klog.Error(err) + return err + } + if err = c.OVNNbClient.AddressSetUpdateAddress(asName, sources.SortedList()...); err != nil { + err = fmt.Errorf("failed to update address set %s: %w", asName, err) + klog.Error(err) + return err + } + + // reconcile OVN BFD entries + bfdIDs := set.New[string]() + staleBFDIDs := set.New[string]() + bfdDstIPs := set.New(slices.Collect(maps.Values(nextHops))...) + bfdMap := make(map[string]string, bfdDstIPs.Len()) + for _, bfd := range bfdList { + if bfdIP == "" || bfd.LogicalPort != lrpName || !bfdDstIPs.Has(bfd.DstIP) { + staleBFDIDs.Insert(bfd.UUID) + } + if bfdIP == "" || (bfd.LogicalPort == lrpName && bfdDstIPs.Has(bfd.DstIP)) { + // TODO: update min_rx, min_tx and multiplier + if bfdIP != "" { + bfdIDs.Insert(bfd.UUID) + bfdMap[bfd.DstIP] = bfd.UUID + } + bfdDstIPs.Delete(bfd.DstIP) + } + } + if bfdIP != "" { + for _, dstIP := range bfdDstIPs.UnsortedList() { + bfd, err := c.OVNNbClient.CreateBFD(lrpName, dstIP, int(router.Spec.BFD.MinRX), int(router.Spec.BFD.MinTX), int(router.Spec.BFD.Multiplier), externalIDs) + if err != nil { + klog.Error(err) + return err + } + bfdIDs.Insert(bfd.UUID) + bfdMap[bfd.DstIP] = bfd.UUID + } + } + + // reconcile LR policy + if router.Spec.TrafficPolicy == kubeovnv1.TrafficPolicyLocal { + rules := make(map[string]string, len(nextHops)) + for nodeName, nexthop := range nextHops { + node, err := c.nodesLister.Get(nodeName) + if err != nil { + if k8serrors.IsNotFound(err) { + continue + } + klog.Errorf("failed to get node %s: %v", nodeName, err) + return err + } + portName := node.Annotations[util.PortNameAnnotation] + if portName == "" { + err = fmt.Errorf("node %s does not have port name annotation", nodeName) + klog.Error(err) + return err + } + localPgName := strings.ReplaceAll(portName, "-", ".") + rules[fmt.Sprintf("ip%d.src == $%s_ip%d && ip%d.src == $%s_ip%d", af, localPgName, af, af, pgName, af)] = nexthop + rules[fmt.Sprintf("ip%d.src == $%s_ip%d && ip%d.src == $%s", af, localPgName, af, af, asName)] = nexthop + } + policies, err := c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayLocalPolicyPriority, externalIDs, false) + if err != nil { + klog.Error(err) + return err + } + // update/delete existing policies + for _, policy := range policies { + nexthop := rules[policy.Match] + bfdSessions := set.New(bfdMap[nexthop]).Delete("") + if nexthop == "" { + if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { + err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) + klog.Error(err) + return err + } + } else { + var changed bool + if len(policy.Nexthops) != 1 || policy.Nexthops[0] != nexthop { + policy.Nexthops = []string{nexthop} + changed = true + } + if !bfdSessions.Equal(set.New(policy.BFDSessions...)) { + policy.BFDSessions = bfdSessions.UnsortedList() + changed = true + } + if changed { + if err = c.OVNNbClient.UpdateLogicalRouterPolicy(policy, &policy.Nexthops, &policy.BFDSessions); err != nil { + err = fmt.Errorf("failed to update logical router policy %s: %w", policy.UUID, err) + klog.Error(err) + return err + } + } + } + delete(rules, policy.Match) + } + // create new policies + for match, nexthop := range rules { + if err = c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayLocalPolicyPriority, match, + ovnnb.LogicalRouterPolicyActionReroute, []string{nexthop}, []string{bfdMap[nexthop]}, externalIDs); err != nil { + klog.Error(err) + return err + } + } + } else { + if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, util.EgressGatewayLocalPolicyPriority, externalIDs); err != nil { + klog.Error(err) + return err + } + } + policies, err := c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayPolicyPriority, externalIDs, false) + if err != nil { + klog.Error(err) + return err + } + matches := set.New( + fmt.Sprintf("ip%d.src == $%s_ip%d", af, pgName, af), + fmt.Sprintf("ip%d.src == $%s", af, asName), + ) + bfdIPs := set.New(slices.Collect(maps.Values(nextHops))...) + bfdSessions := bfdIDs.UnsortedList() + for _, policy := range policies { + if matches.Has(policy.Match) { + if !bfdIPs.Equal(set.New(policy.Nexthops...)) || !bfdIDs.Equal(set.New(policy.BFDSessions...)) { + policy.Nexthops, policy.BFDSessions = bfdIPs.UnsortedList(), bfdSessions + if err = c.OVNNbClient.UpdateLogicalRouterPolicy(policy, &policy.Nexthops, &policy.BFDSessions); err != nil { + err = fmt.Errorf("failed to update bfd sessions of logical router policy %s: %w", policy.UUID, err) + klog.Error(err) + return err + } + } + matches.Delete(policy.Match) + continue + } + if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { + err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) + klog.Error(err) + return err + } + } + for _, match := range matches.UnsortedList() { + if err = c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayPolicyPriority, match, + ovnnb.LogicalRouterPolicyActionReroute, bfdIPs.UnsortedList(), bfdSessions, externalIDs); err != nil { + klog.Error(err) + return err + } + } + + if router.Spec.BFD.Enabled { + // drop traffic if no nexthop is available + if policies, err = c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayDropPolicyPriority, externalIDs, false); err != nil { + klog.Error(err) + return err + } + matches = set.New( + fmt.Sprintf("ip%d.src == $%s_ip%d", af, pgName, af), + fmt.Sprintf("ip%d.src == $%s", af, asName), + ) + for _, policy := range policies { + if matches.Has(policy.Match) { + matches.Delete(policy.Match) + continue + } + if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { + err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) + klog.Error(err) + return err + } + } + for _, match := range matches.UnsortedList() { + if err = c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayDropPolicyPriority, match, + ovnnb.LogicalRouterPolicyActionDrop, nil, nil, externalIDs); err != nil { + klog.Error(err) + return err + } + } + } else if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, util.EgressGatewayDropPolicyPriority, externalIDs); err != nil { + klog.Error(err) + return err + } + + for _, bfdID := range staleBFDIDs.UnsortedList() { + if err = c.OVNNbClient.DeleteBFD(bfdID); err != nil { + err = fmt.Errorf("failed to delete bfd %s: %w", bfdID, err) + klog.Error(err) + return err + } + } + + return nil +} + +func berMergeNodeSelector(nodeSelector []kubeovnv1.BgpEdgeRouterNodeSelector) *corev1.NodeSelector { + if len(nodeSelector) == 0 { + return nil + } + + result := &corev1.NodeSelector{ + NodeSelectorTerms: make([]corev1.NodeSelectorTerm, len(nodeSelector)), + } + for i, selector := range nodeSelector { + result.NodeSelectorTerms[i] = corev1.NodeSelectorTerm{ + MatchExpressions: make([]corev1.NodeSelectorRequirement, len(selector.MatchExpressions), len(selector.MatchLabels)+len(selector.MatchExpressions)), + MatchFields: make([]corev1.NodeSelectorRequirement, len(selector.MatchFields)), + } + for j := range selector.MatchExpressions { + selector.MatchExpressions[j].DeepCopyInto(&result.NodeSelectorTerms[i].MatchExpressions[j]) + } + for _, key := range slices.Sorted(maps.Keys(selector.MatchLabels)) { + result.NodeSelectorTerms[i].MatchExpressions = append(result.NodeSelectorTerms[i].MatchExpressions, corev1.NodeSelectorRequirement{ + Key: key, + Operator: corev1.NodeSelectorOpIn, + Values: []string{selector.MatchLabels[key]}, + }) + } + for j := range selector.MatchFields { + selector.MatchFields[j].DeepCopyInto(&result.NodeSelectorTerms[i].MatchFields[j]) + } + } + + return result +} + +func bgpEdgeRouterInitContainerEnv(af int, internalGateway, externalGateway string, forwardSrc set.Set[string]) ([]corev1.EnvVar, error) { + if internalGateway == "" { + return nil, nil + } + + return []corev1.EnvVar{{ + Name: fmt.Sprintf("INTERNAL_GATEWAY_IPV%d", af), + Value: internalGateway, + }, { + Name: fmt.Sprintf("EXTERNAL_GATEWAY_IPV%d", af), + Value: externalGateway, + }, { + Name: fmt.Sprintf("NO_SNAT_SOURCES_IPV%d", af), + Value: strings.Join(forwardSrc.SortedList(), ","), + }}, nil +} + +func bgpEdgeRouterContainerBFDD(image, bfdIP string, minTX, minRX, multiplier int32) corev1.Container { + return corev1.Container{ + Name: "bfdd", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"bash", "/kube-ovn/start-bfdd.sh"}, + Env: []corev1.EnvVar{{ + Name: "POD_IPS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIPs", + }, + }, + }, { + Name: "BFD_PEER_IPS", + Value: bfdIP, + }, { + Name: "BFD_MIN_TX", + Value: strconv.Itoa(int(minTX)), + }, { + Name: "BFD_MIN_RX", + Value: strconv.Itoa(int(minRX)), + }, { + Name: "BFD_MULTI", + Value: strconv.Itoa(int(multiplier)), + }}, + // wait for the BFD process to be running and initialize the BFD configuration + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bash", "/kube-ovn/bfdd-prestart.sh"}, + }, + }, + InitialDelaySeconds: 1, + FailureThreshold: 1, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bfdd-control", "status"}, + }, + }, + InitialDelaySeconds: 1, + PeriodSeconds: 5, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bfdd-control", "status"}, + }, + }, + InitialDelaySeconds: 3, + PeriodSeconds: 3, + FailureThreshold: 1, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](65534), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_BIND_SERVICE", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "usr-local-sbin", + MountPath: "/usr/local/sbin", + }}, + } +} + +func (c *Controller) handleDelBgpEdgeRouter(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.bgpEdgeRouterKeyMutex.LockKey(key) + defer func() { _ = c.bgpEdgeRouterKeyMutex.UnlockKey(key) }() + + cachedGateway, err := c.bgpEdgeRouterLister.BgpEdgeRouters(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + err = fmt.Errorf("failed to get bgp-edge-router %s: %w", key, err) + klog.Error(err) + return err + } + return nil + } + + klog.Infof("handle deleting bgp-edge-router %s", key) + if err = c.cleanOVNForBgpEdgeRouter(key, cachedGateway.Spec.VPC); err != nil { + klog.Error(err) + return err + } + + router := cachedGateway.DeepCopy() + if controllerutil.RemoveFinalizer(router, util.KubeOVNControllerFinalizer) { + if _, err = c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouters(router.Namespace). + Update(context.Background(), router, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to remove finalizer from bgp-edge-router %s: %w", key, err) + klog.Error(err) + } + } + + return nil +} + +func (c *Controller) cleanOVNForBgpEdgeRouter(key, lrName string) error { + externalIDs := map[string]string{ + ovs.ExternalIDVendor: util.CniTypeName, + ovs.ExternalIDBgpEdgeRouter: key, + } + + bfdList, err := c.OVNNbClient.FindBFD(externalIDs) + if err != nil { + klog.Error(err) + return err + } + for _, bfd := range bfdList { + if err = c.OVNNbClient.DeleteBFD(bfd.UUID); err != nil { + klog.Error(err) + return err + } + } + + if lrName == "" { + lrName = c.config.ClusterRouter + } + if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, -1, externalIDs); err != nil { + klog.Error(err) + return err + } + if err = c.OVNNbClient.DeletePortGroup(berPortGroupName(key)); err != nil { + klog.Error(err) + return err + } + for _, af := range [...]int{4, 6} { + if err = c.OVNNbClient.DeleteAddressSet(berAddressSetName(key, af)); err != nil { + klog.Error(err) + return err + } + } + + return nil +} + +func berPortGroupName(key string) string { + hash := util.Sha256Hash([]byte(key)) + return "BER." + hash[:12] +} + +func berAddressSetName(key string, af int) string { + hash := util.Sha256Hash([]byte(key)) + return fmt.Sprintf("BER.%s.ipv%d", hash[:12], af) +} + +func (c *Controller) handlePodEventForBgpEdgeRouter(pod *corev1.Pod) error { + if !pod.DeletionTimestamp.IsZero() || pod.Annotations[util.AllocatedAnnotation] != "true" { + return nil + } + vpc := pod.Annotations[util.LogicalRouterAnnotation] + if vpc == "" { + return nil + } + + ns, err := c.namespacesLister.Get(pod.Namespace) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + klog.Errorf("failed to get namespace %s: %v", pod.Namespace, err) + utilruntime.HandleError(err) + return err + } + + router, err := c.bgpEdgeRouterLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list bgp edge router: %v", err) + utilruntime.HandleError(err) + return err + } + + for _, ber := range router { + if ber.VPC(c.config.ClusterRouter) != vpc { + continue + } + + for _, selector := range ber.Spec.Selectors { + if selector.NamespaceSelector != nil && !util.ObjectMatchesLabelSelector(ns, selector.NamespaceSelector) { + continue + } + if selector.PodSelector != nil && !util.ObjectMatchesLabelSelector(pod, selector.PodSelector) { + continue + } + c.addOrUpdateBgpEdgeRouterQueue.Add(cache.MetaObjectToName(ber).String()) + } + } + return nil +} + +func bgpEdgeRouterContainerBGP(speakerImage, routerName string, speakerParams *kubeovnv1.BgpEdgeRouterBGPConfig) (*corev1.Container, error) { + if speakerImage == "" { + return nil, errors.New("BGP speaker image must be specified") + } + if speakerParams == nil { + return nil, errors.New("BGP config must not be nil") + } + if speakerParams.ASN == 0 { + return nil, errors.New("ASN not set, but must be non-zero value") + } + if speakerParams.RemoteASN == 0 { + return nil, errors.New("remote ASN not set, but must be non-zero value") + } + if len(speakerParams.Neighbors) == 0 { + return nil, errors.New("no BGP neighbors specified") + } + + args := []string{} + if speakerParams.RouterID != "" { + args = append(args, "--router-id="+speakerParams.RouterID) + } + if speakerParams.Password != "" { + args = append(args, "--auth-password="+speakerParams.Password) + } + if speakerParams.EnableGracefulRestart { + args = append(args, "--graceful-restart") + } + if speakerParams.HoldTime != (metav1.Duration{}) { + args = append(args, "--holdtime="+speakerParams.HoldTime.Duration.String()) + } + + args = append(args, fmt.Sprintf("--cluster-as=%d", speakerParams.ASN)) + args = append(args, fmt.Sprintf("--neighbor-as=%d", speakerParams.RemoteASN)) + + var neighIPv4, neighIPv6 []string + for _, neighbor := range speakerParams.Neighbors { + switch util.CheckProtocol(neighbor) { + case kubeovnv1.ProtocolIPv4: + neighIPv4 = append(neighIPv4, neighbor) + case kubeovnv1.ProtocolIPv6: + neighIPv6 = append(neighIPv6, neighbor) + default: + return nil, fmt.Errorf("unsupported protocol for peer %s", neighbor) + } + } + if len(neighIPv4) > 0 { + args = append(args, "--neighbor-address="+strings.Join(neighIPv4, ",")) + } + if len(neighIPv6) > 0 { + args = append(args, "--neighbor-ipv6-address="+strings.Join(neighIPv6, ",")) + } + + args = append(args, speakerParams.ExtraArgs...) + + container := &corev1.Container{ + Name: "bgp-router-speaker", + Image: speakerImage, + Command: []string{"/kube-ovn/kube-ovn-speaker"}, + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + { + Name: "EGRESS_GATEWAY_NAME", + Value: routerName, + }, + { + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + }, + Args: args, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "kube-ovn-logs", + MountPath: "/var/log/kube-ovn", + }, + }, + } + + return container, nil +} From adacc4d1b2b0e4be28800b5068a81f69c6a7f850 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 28 Jul 2025 06:41:31 -0700 Subject: [PATCH 13/66] bgp edger router mod --- pkg/controller/bgp_edge_router.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go index c9faa0d1734..60d8d93fbf7 100644 --- a/pkg/controller/bgp_edge_router.go +++ b/pkg/controller/bgp_edge_router.go @@ -78,7 +78,7 @@ func (c *Controller) handleAddOrUpdateBgpEdgeRouter(key string) error { klog.Infof("reconciling bgp-edge-router %s", key) router := cachedRouter.DeepCopy() - if router, err = c.initbgpEdgeRouterStatus(router); err != nil { + if router, err = c.initBgpEdgeRouterStatus(router); err != nil { return err } @@ -210,7 +210,7 @@ func (c *Controller) handleAddOrUpdateBgpEdgeRouter(key string) error { return nil } -func (c *Controller) initbgpEdgeRouterStatus(router *kubeovnv1.BgpEdgeRouter) (*kubeovnv1.BgpEdgeRouter, error) { +func (c *Controller) initBgpEdgeRouterStatus(router *kubeovnv1.BgpEdgeRouter) (*kubeovnv1.BgpEdgeRouter, error) { var err error if router.Status.Phase == "" || router.Status.Phase == kubeovnv1.PhasePending { router.Status.Phase = kubeovnv1.PhaseProcessing From a314262e8c1d4d670810b4b69a2b40f7d4541032 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 28 Jul 2025 23:53:34 -0700 Subject: [PATCH 14/66] bgp edge router mod --- pkg/apis/kubeovn/v1/bgp-edge-router.go | 2 +- .../versioned/typed/kubeovn/v1/bgpedgerouter.go | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/pkg/apis/kubeovn/v1/bgp-edge-router.go b/pkg/apis/kubeovn/v1/bgp-edge-router.go index 404954bffa1..0bcadf52b4a 100644 --- a/pkg/apis/kubeovn/v1/bgp-edge-router.go +++ b/pkg/apis/kubeovn/v1/bgp-edge-router.go @@ -18,7 +18,7 @@ type BgpEdgeRouterList struct { // +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale // +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// +resourceName=vpc-egress-gateways +// +resourceName=bgp-edge-routers // vpc egress gateway is used to forward the egress traffic from the VPC to the external network type BgpEdgeRouter struct { metav1.TypeMeta `json:",inline"` diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go index c61a2c4b090..5c186b15062 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go @@ -21,17 +21,6 @@ package v1 import ( context "context" - // apiskubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" - // versioned "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" - // internalinterfaces "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions/internalinterfaces" - // kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" - // autoscalingv1 "k8s.io/api/autoscaling/v1" - // metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - // runtime "k8s.io/apimachinery/pkg/runtime" - // types "k8s.io/apimachinery/pkg/types" - // watch "k8s.io/apimachinery/pkg/watch" - // cache "k8s.io/client-go/tools/cache" - kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" scheme "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" autoscalingv1 "k8s.io/api/autoscaling/v1" From 23c29cd1d272cd6871f4ee2bb1b332d1e18416f2 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Tue, 29 Jul 2025 01:07:37 -0700 Subject: [PATCH 15/66] bgp edge router comment mod --- pkg/apis/kubeovn/v1/bgp-edge-router.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/kubeovn/v1/bgp-edge-router.go b/pkg/apis/kubeovn/v1/bgp-edge-router.go index 0bcadf52b4a..34cfb99af3e 100644 --- a/pkg/apis/kubeovn/v1/bgp-edge-router.go +++ b/pkg/apis/kubeovn/v1/bgp-edge-router.go @@ -19,7 +19,7 @@ type BgpEdgeRouterList struct { // +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +resourceName=bgp-edge-routers -// vpc egress gateway is used to forward the egress traffic from the VPC to the external network +// bgp edge router is used to forward the egress traffic from the VPC to the external network type BgpEdgeRouter struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata"` From af0b931c1d98b0c3df100b50f2c473c137415e88 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Tue, 29 Jul 2025 18:50:46 -0700 Subject: [PATCH 16/66] bgp edger router advertisement modification --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 289 +++++- .../v1/bgp-edge-router-advertisement.go | 45 + pkg/apis/kubeovn/v1/register.go | 2 + pkg/apis/kubeovn/v1/zz_generated.deepcopy.go | 85 ++ .../kubeovn/v1/bgpedgerouteradvertisement.go | 103 +++ .../fake/fake_bgpedgerouteradvertisement.go | 81 ++ .../kubeovn/v1/fake/fake_kubeovn_client.go | 4 + .../typed/kubeovn/v1/generated_expansion.go | 2 + .../typed/kubeovn/v1/kubeovn_client.go | 5 + .../kubeovn/v1/bgpedgerouteradvertisement.go | 89 ++ .../externalversions/kubeovn/v1/interface.go | 7 + .../kubeovn/v1/bgpedgerouteradvertisement.go | 70 ++ .../listers/kubeovn/v1/expansion_generated.go | 8 + .../bgp_edge_router_advertisement.go | 850 +++--------------- pkg/controller/controller.go | 18 + pkg/ovs/ovn.go | 7 +- pkg/util/const.go | 7 +- 17 files changed, 925 insertions(+), 747 deletions(-) create mode 100644 pkg/apis/kubeovn/v1/bgp-edge-router-advertisement.go create mode 100644 pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouteradvertisement.go create mode 100644 pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouteradvertisement.go create mode 100644 pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouteradvertisement.go create mode 100644 pkg/client/listers/kubeovn/v1/bgpedgerouteradvertisement.go diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index a121d797f56..0b9857e6770 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3582,4 +3582,291 @@ spec: extraArgs: type: array items: - type: string \ No newline at end of file + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgp-edge-router-advertisements.kubeovn.io +spec: + group: kubeovn.io + names: + plural: bgp-edge-router-advertisements + singular: bgp-edge-router-advertisement + shortNames: + - ber-ad + kind: BgpEdgeRouterAdvertisement + listKind: BgpEdgeRouterAdvertisementList + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.subnet + name: SUBNET + type: string + - jsonPath: .spec.bgpedgerouter + name: BGP-EDGE-ROUTER + type: string + - jsonPath: .status.ready + name: READY + type: boolean + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + scale: + # specReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Spec.Replicas. + specReplicasPath: .spec.replicas + # statusReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Replicas. + statusReplicasPath: .status.replicas + # labelSelectorPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Selector. + labelSelectorPath: .status.labelSelector + schema: + openAPIV3Schema: + type: object + properties: + status: + properties: + replicas: + type: integer + format: int32 + labelSelector: + type: string + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + internalIPs: + items: + type: string + type: array + externalIPs: + items: + type: string + type: array + phase: + type: string + default: Pending + enum: + - Pending + - Processing + - Completed + ready: + type: boolean + default: false + workload: + type: object + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + nodes: + type: array + items: + type: string + required: + - conditions + - phase + type: object + spec: + type: object + required: + - externalSubnet + x-kubernetes-validations: + - rule: "!has(self.internalIPs) || size(self.internalIPs) == 0 || size(self.internalIPs) >= self.replicas" + message: 'Size of Internal IPs MUST be equal to or greater than Replicas' + fieldPath: ".internalIPs" + - rule: "!has(self.externalIPs) || size(self.externalIPs) == 0 || size(self.externalIPs) >= self.replicas" + message: 'Size of External IPs MUST be equal to or greater than Replicas' + fieldPath: ".externalIPs" + - rule: "size(self.policies) != 0 || size(self.selectors) != 0" + message: 'Each BGP Edeg Router MUST have at least one policy or selector' + properties: + + + + selectors: + type: array + items: + type: object + properties: + namespaceSelector: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + x-kubernetes-validations: + - rule: "size(self.matchLabels) != 0 || size(self.matchExpressions) != 0" + message: 'Each namespace selector MUST have at least one matchLabels or matchExpressions' + podSelector: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + x-kubernetes-validations: + - rule: "size(self.matchLabels) != 0 || size(self.matchExpressions) != 0" + message: 'Each pod selector MUST have at least one matchLabels or matchExpressions' + policies: + type: array + items: + type: object + properties: + snat: + type: boolean + default: false + ipBlocks: + type: array + x-kubernetes-list-type: set + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + - format: cidr + subnets: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + x-kubernetes-validations: + - rule: "size(self.ipBlocks) != 0 || size(self.subnets) != 0" + message: 'Each policy MUST have at least one ipBlock or subnet' + nodeSelector: + type: array + items: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator diff --git a/pkg/apis/kubeovn/v1/bgp-edge-router-advertisement.go b/pkg/apis/kubeovn/v1/bgp-edge-router-advertisement.go new file mode 100644 index 00000000000..61ba91b1462 --- /dev/null +++ b/pkg/apis/kubeovn/v1/bgp-edge-router-advertisement.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type BgpEdgeRouterAdvertisementList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []BgpEdgeRouterAdvertisement `json:"items"` +} + +// +genclient +// +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale +// +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +resourceName=bgp-edge-router-advertisements +// bgp edge router advertisement is used to forward the egress traffic from the VPC to the external network +type BgpEdgeRouterAdvertisement struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + + Spec BgpEdgeRouterAdvertisementSpec `json:"spec"` +} + +// If the BgpEdgeRouter has no VPC specified in the spec, it will return the default VPC name +func (g *BgpEdgeRouterAdvertisement) Subnet(subnets []string) []string { + if len(subnets) != 0 { + return subnets + } + return nil +} + +// Ready returns true if the BgpEdgeRouter has been processed successfully and is ready to serve traffic +// func (g *BgpEdgeRouterAdvertisement) Ready() bool { +// return g.Status.Ready && g.Status.Conditions.IsReady(g.Generation) +// } + +type BgpEdgeRouterAdvertisementSpec struct { + Subnet []string `json:"subnet,omitempty"` + BgpEdgeRouter string `json:"bgpEdgeRouter,omitempty"` +} diff --git a/pkg/apis/kubeovn/v1/register.go b/pkg/apis/kubeovn/v1/register.go index ea2e2fd478a..53758d2be91 100644 --- a/pkg/apis/kubeovn/v1/register.go +++ b/pkg/apis/kubeovn/v1/register.go @@ -75,6 +75,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &VpcNatGatewayList{}, &BgpEdgeRouter{}, &BgpEdgeRouterList{}, + &BgpEdgeRouterAdvertisement{}, + &BgpEdgeRouterAdvertisementList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go index 325fa473021..51b49ad5962 100644 --- a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go +++ b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go @@ -3109,3 +3109,88 @@ func (in *BgpEdgeRouterWorkload) DeepCopy() *BgpEdgeRouterWorkload { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterAdvertisement) DeepCopyInto(out *BgpEdgeRouterAdvertisement) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGateway. +func (in *BgpEdgeRouterAdvertisement) DeepCopy() *BgpEdgeRouterAdvertisement { + if in == nil { + return nil + } + out := new(BgpEdgeRouterAdvertisement) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BgpEdgeRouterAdvertisement) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterAdvertisementList) DeepCopyInto(out *BgpEdgeRouterAdvertisementList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]BgpEdgeRouterAdvertisement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayList. +func (in *BgpEdgeRouterAdvertisementList) DeepCopy() *BgpEdgeRouterAdvertisementList { + if in == nil { + return nil + } + out := new(BgpEdgeRouterAdvertisementList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BgpEdgeRouterAdvertisementList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterAdvertisementSpec) DeepCopyInto(out *BgpEdgeRouterAdvertisementSpec) { + *out = *in + if in.Subnet != nil { + in, out := &in.Subnet, &out.Subnet + *out = *in + } + if in.BgpEdgeRouter != "" { + in, out := &in.BgpEdgeRouter, &out.BgpEdgeRouter + *out = *in + } + + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpEdgeRouterAdvertisementSpec. +func (in *BgpEdgeRouterAdvertisementSpec) DeepCopy() *BgpEdgeRouterAdvertisementSpec { + if in == nil { + return nil + } + out := new(BgpEdgeRouterAdvertisementSpec) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouteradvertisement.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouteradvertisement.go new file mode 100644 index 00000000000..40d7d2f16ef --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouteradvertisement.go @@ -0,0 +1,103 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + scheme "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" + autoscalingv1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// BgpEdgeRouterAdvertisementsGetter has a method to return a BgpEdgeRouterAdvertisementInterface. +// A group's client should implement this interface. +type BgpEdgeRouterAdvertisementsGetter interface { + BgpEdgeRouterAdvertisements(namespace string) BgpEdgeRouterAdvertisementInterface +} + +// BgpEdgeRouterAdvertisementInterface has methods to work with BgpEdgeRouterAdvertisement resources. +type BgpEdgeRouterAdvertisementInterface interface { + Create(ctx context.Context, bgpEdgeRouterAdvertisement *kubeovnv1.BgpEdgeRouterAdvertisement, opts metav1.CreateOptions) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) + Update(ctx context.Context, bgpEdgeRouterAdvertisement *kubeovnv1.BgpEdgeRouterAdvertisement, opts metav1.UpdateOptions) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, bgpEdgeRouterAdvertisement *kubeovnv1.BgpEdgeRouterAdvertisement, opts metav1.UpdateOptions) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) + List(ctx context.Context, opts metav1.ListOptions) (*kubeovnv1.BgpEdgeRouterAdvertisementList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *kubeovnv1.BgpEdgeRouterAdvertisement, err error) + GetScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, options metav1.GetOptions) (*autoscalingv1.Scale, error) + UpdateScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (*autoscalingv1.Scale, error) + + BgpEdgeRouterAdrvertisementExpansion +} + +// bgpEdgeRouterAdvertisements implements BgpEdgeRouterAdvertisementInterface +type bgpEdgeRouterAdvertisements struct { + *gentype.ClientWithList[*kubeovnv1.BgpEdgeRouterAdvertisement, *kubeovnv1.BgpEdgeRouterAdvertisementList] +} + +// newBgpEdgeRouters returns a BgpEdgeRouters +func newBgpEdgeRouterAdvertisements(c *KubeovnV1Client, namespace string) *bgpEdgeRouterAdvertisements { + return &bgpEdgeRouterAdvertisements{ + gentype.NewClientWithList[*kubeovnv1.BgpEdgeRouterAdvertisement, *kubeovnv1.BgpEdgeRouterAdvertisementList]( + "bgp-edge-router-advertisements", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *kubeovnv1.BgpEdgeRouterAdvertisement { return &kubeovnv1.BgpEdgeRouterAdvertisement{} }, + func() *kubeovnv1.BgpEdgeRouterAdvertisementList { return &kubeovnv1.BgpEdgeRouterAdvertisementList{} }, + ), + } +} + +// GetScale takes name of the bgpEdgeRouter, and returns the corresponding autoscalingv1.Scale object, and an error if there is any. +func (c *bgpEdgeRouterAdvertisements) GetScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, options metav1.GetOptions) (result *autoscalingv1.Scale, err error) { + result = &autoscalingv1.Scale{} + err = c.GetClient().Get(). + Namespace(c.GetNamespace()). + Resource("bgp-edge-router-advertisements"). + Name(bgpEdgeRouterAdvertisementName). + SubResource("scale"). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// UpdateScale takes the top resource name and the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *bgpEdgeRouterAdvertisements) UpdateScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (result *autoscalingv1.Scale, err error) { + result = &autoscalingv1.Scale{} + err = c.GetClient().Put(). + Namespace(c.GetNamespace()). + Resource("bgp-edge-router-advertisements"). + Name(bgpEdgeRouterAdvertisementName). + SubResource("scale"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(scale). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouteradvertisement.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouteradvertisement.go new file mode 100644 index 00000000000..a73909af703 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouteradvertisement.go @@ -0,0 +1,81 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + context "context" + + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gentype "k8s.io/client-go/gentype" + testing "k8s.io/client-go/testing" +) + +// fakeBgpEdgeRouters implements BgpEdgeRouterInterface +type fakeBgpEdgeRouterAdvertisements struct { + *gentype.FakeClientWithList[*v1.BgpEdgeRouterAdvertisement, *v1.BgpEdgeRouterAdvertisementList] + Fake *FakeKubeovnV1 +} + +func newFakeBgpEdgeRouterAdvertisements(fake *FakeKubeovnV1, namespace string) kubeovnv1.BgpEdgeRouterAdvertisementInterface { + return &fakeBgpEdgeRouterAdvertisements{ + gentype.NewFakeClientWithList[*v1.BgpEdgeRouterAdvertisement, *v1.BgpEdgeRouterAdvertisementList]( + fake.Fake, + namespace, + v1.SchemeGroupVersion.WithResource("bgp-edge-router-advertisements"), + v1.SchemeGroupVersion.WithKind("BgpEdgeRouterAdvertisement"), + func() *v1.BgpEdgeRouterAdvertisement { return &v1.BgpEdgeRouterAdvertisement{} }, + func() *v1.BgpEdgeRouterAdvertisementList { return &v1.BgpEdgeRouterAdvertisementList{} }, + func(dst, src *v1.BgpEdgeRouterAdvertisementList) { dst.ListMeta = src.ListMeta }, + func(list *v1.BgpEdgeRouterAdvertisementList) []*v1.BgpEdgeRouterAdvertisement { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1.BgpEdgeRouterAdvertisementList, items []*v1.BgpEdgeRouterAdvertisement) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} + +// GetScale takes name of the bgpEdgeRouter, and returns the corresponding scale object, and an error if there is any. +func (c *fakeBgpEdgeRouterAdvertisements) GetScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, options metav1.GetOptions) (result *autoscalingv1.Scale, err error) { + emptyResult := &autoscalingv1.Scale{} + obj, err := c.Fake. + Invokes(testing.NewGetSubresourceActionWithOptions(c.Resource(), c.Namespace(), "scale", bgpEdgeRouterAdvertisementName, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*autoscalingv1.Scale), err +} + +// UpdateScale takes the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *fakeBgpEdgeRouterAdvertisements) UpdateScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (result *autoscalingv1.Scale, err error) { + emptyResult := &autoscalingv1.Scale{} + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceActionWithOptions(c.Resource(), "scale", c.Namespace(), scale, opts), &autoscalingv1.Scale{}) + + if obj == nil { + return emptyResult, err + } + return obj.(*autoscalingv1.Scale), err +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go index 3745fc15417..e2e632cf062 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go @@ -116,6 +116,10 @@ func (c *FakeKubeovnV1) BgpEdgeRouters(namespace string) v1.BgpEdgeRouterInterfa return newFakeBgpEdgeRouters(c, namespace) } +func (c *FakeKubeovnV1) BgpEdgeRouterAdvertisements(namespace string) v1.BgpEdgeRouterAdvertisementInterface { + return newFakeBgpEdgeRouterAdvertisements(c, namespace) +} + // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. func (c *FakeKubeovnV1) RESTClient() rest.Interface { diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go index e1185cd5250..9299dea7c90 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go @@ -61,3 +61,5 @@ type VpcEgressGatewayExpansion interface{} type VpcNatGatewayExpansion interface{} type BgpEdgeRouterExpansion interface{} + +type BgpEdgeRouterAdrvertisementExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go index 0af283168c1..540e754cfd7 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go @@ -50,6 +50,7 @@ type KubeovnV1Interface interface { VpcEgressGatewaysGetter VpcNatGatewaysGetter BgpEdgeRoutersGetter + BgpEdgeRouterAdvertisementsGetter } // KubeovnV1Client is used to interact with features provided by the kubeovn.io group. @@ -145,6 +146,10 @@ func (c *KubeovnV1Client) BgpEdgeRouters(namespace string) BgpEdgeRouterInterfac return newBgpEdgeRouters(c, namespace) } +func (c *KubeovnV1Client) BgpEdgeRouterAdvertisements(namespace string) BgpEdgeRouterAdvertisementInterface { + return newBgpEdgeRouterAdvertisements(c, namespace) +} + // NewForConfig creates a new KubeovnV1Client for the given config. // NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), // where httpClient was generated with rest.HTTPClientFor(c). diff --git a/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouteradvertisement.go b/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouteradvertisement.go new file mode 100644 index 00000000000..bdfccf1813a --- /dev/null +++ b/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouteradvertisement.go @@ -0,0 +1,89 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + time "time" + + apiskubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + versioned "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions/internalinterfaces" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// BgpEdgeRouterAdvertisementInformer provides access to a shared informer and lister for +type BgpEdgeRouterAdvertisementInformer interface { + Informer() cache.SharedIndexInformer + Lister() kubeovnv1.BgpEdgeRouterAdvertisementLister +} + +type bgpEdgeRouterAdvertisementInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewBgpEdgeRouterAdvertisementInformer constructs a new informer for BgpEdgeRouter type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewBgpEdgeRouterAdvertisementInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredBgpEdgeRouterAdvertisementInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredBgpEdgeRouterInformer constructs a new informer for BgpEdgeRouter type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredBgpEdgeRouterAdvertisementInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().BgpEdgeRouterAdvertisements(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().BgpEdgeRouterAdvertisements(namespace).Watch(context.TODO(), options) + }, + }, + &apiskubeovnv1.BgpEdgeRouterAdvertisement{}, + resyncPeriod, + indexers, + ) +} + +func (f *bgpEdgeRouterAdvertisementInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredBgpEdgeRouterAdvertisementInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *bgpEdgeRouterAdvertisementInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apiskubeovnv1.BgpEdgeRouterAdvertisement{}, f.defaultInformer) +} + +func (f *bgpEdgeRouterAdvertisementInformer) Lister() kubeovnv1.BgpEdgeRouterAdvertisementLister { + return kubeovnv1.NewBgpEdgeRouterAdvertisementLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/kubeovn/v1/interface.go b/pkg/client/informers/externalversions/kubeovn/v1/interface.go index 791dcaa93da..57dcc7394c9 100644 --- a/pkg/client/informers/externalversions/kubeovn/v1/interface.go +++ b/pkg/client/informers/externalversions/kubeovn/v1/interface.go @@ -68,6 +68,8 @@ type Interface interface { VpcNatGateways() VpcNatGatewayInformer // BgpEdgeRouters returns a BgpEdgeRouterInformer. BgpEdgeRouters() BgpEdgeRouterInformer + // BgpEdgeRouters returns a BgpEdgeRouterInformer. + BgpEdgeRouterAdvertisements() BgpEdgeRouterAdvertisementInformer } type version struct { @@ -190,3 +192,8 @@ func (v *version) VpcNatGateways() VpcNatGatewayInformer { func (v *version) BgpEdgeRouters() BgpEdgeRouterInformer { return &bgpEdgeRouterInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } + +// BgpEdgeRouters returns a BgpEdgeRouterInformer. +func (v *version) BgpEdgeRouterAdvertisements() BgpEdgeRouterAdvertisementInformer { + return &bgpEdgeRouterAdvertisementInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/client/listers/kubeovn/v1/bgpedgerouteradvertisement.go b/pkg/client/listers/kubeovn/v1/bgpedgerouteradvertisement.go new file mode 100644 index 00000000000..11e95058e84 --- /dev/null +++ b/pkg/client/listers/kubeovn/v1/bgpedgerouteradvertisement.go @@ -0,0 +1,70 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// BgpEdgeRouterAdvertisementLister helps list BgpEdgeRouterAdvertisements. +// All objects returned here must be treated as read-only. +type BgpEdgeRouterAdvertisementLister interface { + // List lists all BgpEdgeRouterAdvertisements in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*kubeovnv1.BgpEdgeRouterAdvertisement, err error) + // BgpEdgeRouterAdvertisements returns an object that can list and get BgpEdgeRouterAdvertisements. + BgpEdgeRouterAdvertisements(namespace string) BgpEdgeRouterAdvertisementNamespaceLister + BgpEdgeRouterAdvertisementListerExpansion +} + +// bgpEdgeRouterLister implements the bgpEdgeRouterLister interface. +type bgpEdgeRouterAdvertisementLister struct { + listers.ResourceIndexer[*kubeovnv1.BgpEdgeRouterAdvertisement] +} + +// NewBgpEdgeRouterAdvertisementLister returns a new bgpEdgeRouterAdvertisementLister. +func NewBgpEdgeRouterAdvertisementLister(indexer cache.Indexer) BgpEdgeRouterAdvertisementLister { + return &bgpEdgeRouterAdvertisementLister{listers.New[*kubeovnv1.BgpEdgeRouterAdvertisement](indexer, kubeovnv1.Resource("bgpedgerouteradvertisement"))} +} + +// bgpEdgeRouterAdvertisements returns an object that can list and get bgpEdgeRouterAdvertisements. +func (s *bgpEdgeRouterAdvertisementLister) BgpEdgeRouterAdvertisements(namespace string) BgpEdgeRouterAdvertisementNamespaceLister { + return bgpEdgeRouterAdvertisementNamespaceLister{listers.NewNamespaced[*kubeovnv1.BgpEdgeRouterAdvertisement](s.ResourceIndexer, namespace)} +} + +// bgpEdgeRouterAdvertisementNamespaceLister helps list and get bgpEdgeRouterAdvertisements. +// All objects returned here must be treated as read-only. +type BgpEdgeRouterAdvertisementNamespaceLister interface { + // List lists all bgpEdgeRouterAdvertisements in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*kubeovnv1.BgpEdgeRouterAdvertisement, err error) + // Get retrieves the bgpEdgeRouter from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) + BgpEdgeRouterAdvertisementNamespaceListerExpansion +} + +// bgpEdgeRouterAdvertisementNamespaceLister implements the bgpEdgeRouterAdvertisementNamespaceLister +// interface. +type bgpEdgeRouterAdvertisementNamespaceLister struct { + listers.ResourceIndexer[*kubeovnv1.BgpEdgeRouterAdvertisement] +} diff --git a/pkg/client/listers/kubeovn/v1/expansion_generated.go b/pkg/client/listers/kubeovn/v1/expansion_generated.go index cc2027db016..c3f95d5071d 100644 --- a/pkg/client/listers/kubeovn/v1/expansion_generated.go +++ b/pkg/client/listers/kubeovn/v1/expansion_generated.go @@ -113,3 +113,11 @@ type BgpEdgeRouterListerExpansion interface{} // BgpEdgeRouterNamespaceListerExpansion allows custom methods to be added to // BgpEdgeRouterNamespaceLister. type BgpEdgeRouterNamespaceListerExpansion interface{} + +// BgpEdgeRouterListerExpansion allows custom methods to be added to +// BgpEdgeRouterLister. +type BgpEdgeRouterAdvertisementListerExpansion interface{} + +// BgpEdgeRouterNamespaceListerExpansion allows custom methods to be added to +// BgpEdgeRouterNamespaceLister. +type BgpEdgeRouterAdvertisementNamespaceListerExpansion interface{} diff --git a/pkg/controller/bgp_edge_router_advertisement.go b/pkg/controller/bgp_edge_router_advertisement.go index 33aa06e7357..7d4d9db8c1d 100644 --- a/pkg/controller/bgp_edge_router_advertisement.go +++ b/pkg/controller/bgp_edge_router_advertisement.go @@ -2,30 +2,16 @@ package controller import ( "context" - "errors" "fmt" - "maps" - "reflect" - "slices" - "strconv" - "strings" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/intstr" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" - "k8s.io/utils/ptr" - "k8s.io/utils/set" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" - "github.com/kubeovn/kube-ovn/pkg/ovs" - "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -51,7 +37,7 @@ func bgpEdgeRouterAdvertisementWorkloadLabels(bgpEdgeRouterAdvertisementName str return map[string]string{"app": "bgp-edge-router-advertisement", util.BgpEdgeRouterLabel: bgpEdgeRouterAdvertisementName} } -func (c *Controller) handleAddOrUpdateBgpEdgeRouterAdvertisement(key string) error { +func (c *Controller) handleAddBgpEdgeRouterAdvertisement(key string) error { ns, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) @@ -78,16 +64,6 @@ func (c *Controller) handleAddOrUpdateBgpEdgeRouterAdvertisement(key string) err klog.Infof("reconciling bgp-edge-router-advertisement %s", key) advertisement := cachedAdvertisement.DeepCopy() - subnetName := advertisement.Spec.Subnets - if subnetName == "" { - subnetName = c.config.ClusterRouter - } - subnet, err := c.subnetsLister.Get(subnetName) - if err != nil { - klog.Error(err) - return err - } - if controllerutil.AddFinalizer(advertisement, util.KubeOVNControllerFinalizer) { updatedAdvertisement, err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouterAdvertisements(advertisement.Namespace). Update(context.Background(), advertisement, metav1.UpdateOptions{}) @@ -100,21 +76,15 @@ func (c *Controller) handleAddOrUpdateBgpEdgeRouterAdvertisement(key string) err } // reconcile the bgp edge router workload and get the route sources for later OVN resources reconciliation - deploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(advertisement.Spec.BerName) + deploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(advertisement.Spec.BgpEdgeRouter) if err != nil { klog.Error(err) return err } - advertisement.Status.Workload.APIVersion = deploy.APIVersion - advertisement.Status.Workload.Kind = deploy.Kind - advertisement.Status.Workload.Name = deploy.Name - advertisement.Status.Workload.Nodes = nil - nodeNexthopIPv4 := make(map[string]string, int(advertisement.Spec.Replicas)) - nodeNexthopIPv6 := make(map[string]string, int(advertisement.Spec.Replicas)) ready := util.DeploymentIsReady(deploy) if !ready { - readyErr = fmt.Sprintf("Deployment %s is not ready", deploy.Kind, deploy.Name) + readyErr := fmt.Sprintf("Deployment %s is not ready", deploy.Kind, deploy.Name) klog.Error(readyErr) return fmt.Errorf(readyErr) } @@ -133,39 +103,10 @@ func (c *Controller) handleAddOrUpdateBgpEdgeRouterAdvertisement(key string) err return err } - // update router status including the internal/external IPs and the nodes where the pods are running - router.Status.Workload.Nodes = make([]string, 0, len(pods)) for _, pod := range pods { if len(pod.Status.PodIPs) == 0 { continue } - extIPs, err := util.PodAttachmentIPs(pod, attachmentNetworkName) - if err != nil { - klog.Error(err) - continue - } - - ips := util.PodIPs(*pod) - ipv4, ipv6 := util.SplitIpsByProtocol(ips) - if len(ipv4) != 0 { - nodeNexthopIPv4[pod.Spec.NodeName] = ipv4[0] - } - if len(ipv6) != 0 { - nodeNexthopIPv6[pod.Spec.NodeName] = ipv6[0] - } - router.Status.InternalIPs = append(router.Status.InternalIPs, strings.Join(ips, ",")) - router.Status.ExternalIPs = append(router.Status.ExternalIPs, strings.Join(extIPs, ",")) - router.Status.Workload.Nodes = append(router.Status.Workload.Nodes, pod.Spec.NodeName) - } - - // reconcile OVN routes - if err = c.reconcileBgpEdgeRouterOVNRoutes(router, 4, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv4, nodeNexthopIPv4, ipv4Src); err != nil { - klog.Error(err) - return err - } - if err = c.reconcileBgpEdgeRouterOVNRoutes(router, 6, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv6, nodeNexthopIPv6, ipv6Src); err != nil { - klog.Error(err) - return err } klog.Infof("finished reconciling bgp-edge-router-advertisement %s", key) @@ -173,558 +114,84 @@ func (c *Controller) handleAddOrUpdateBgpEdgeRouterAdvertisement(key string) err return nil } -// create or update bgp edge router workload -func (c *Controller) reconcileBgpEdgeRouterWorkload(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement, vpc *kubeovnv1.Vpc) (string, set.Set[string], set.Set[string], *appsv1.Deployment, error) { - // generate init container environment variables - // the init container is responsible for adding routes and SNAT rules to the pod network namespace - initEnv, err := bgpEdgeRouterInitContainerEnv(4, intGatewayIPv4, extGatewayIPv4, ipv4ForwardSrc) - if err != nil { - klog.Error(err) - return attachmentNetworkName, nil, nil, nil, err - } - ipv6Env, err := bgpEdgeRouterInitContainerEnv(6, intGatewayIPv6, extGatewayIPv6, ipv6ForwardSrc) +func (c *Controller) handleUpdateBgpEdgeRouterAdvertisement(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { - klog.Error(err) - return attachmentNetworkName, nil, nil, nil, err - } - initEnv = append(initEnv, ipv6Env...) - - // generate workload - labels := bgpEdgeRouterWorkloadLabels(router.Name) - deploy := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: router.Spec.Prefix + router.Name, - Namespace: router.Namespace, - Labels: labels, - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.RollingUpdateDeploymentStrategyType, - RollingUpdate: &appsv1.RollingUpdateDeployment{ - MaxUnavailable: ptr.To(intstr.FromInt(1)), - MaxSurge: ptr.To(intstr.FromInt(0)), - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - Annotations: annotations, - }, - Spec: corev1.PodSpec{ - Affinity: &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: berMergeNodeSelector(router.Spec.NodeSelector), - }, - PodAntiAffinity: &corev1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - TopologyKey: corev1.LabelHostname, - }}, - }, - }, - InitContainers: []corev1.Container{{ - Name: "init", - Image: image, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"bash", "/kube-ovn/init-vpc-egress-gateway.sh"}, - Env: initEnv, - SecurityContext: &corev1.SecurityContext{ - Privileged: ptr.To(true), - }, - VolumeMounts: []corev1.VolumeMount{{ - Name: "usr-local-sbin", - MountPath: "/usr/local/sbin", - }}, - }}, - Containers: []corev1.Container{{ - Name: "gateway", - Image: image, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"sleep", "infinity"}, - SecurityContext: &corev1.SecurityContext{ - Privileged: ptr.To(false), - RunAsUser: ptr.To[int64](65534), - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"NET_ADMIN", "NET_RAW"}, - Drop: []corev1.Capability{"ALL"}, - }, - }, - VolumeMounts: []corev1.VolumeMount{{ - Name: "usr-local-sbin", - MountPath: "/usr/local/sbin", - }}, - }}, - Volumes: []corev1.Volume{ - { - Name: "usr-local-sbin", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "kube-ovn-logs", - VolumeSource: corev1.VolumeSource{}, - }, - }, - TerminationGracePeriodSeconds: ptr.To[int64](0), - }, - }, - }, - } - // set owner reference so that the workload will be deleted automatically when the bgp edge router is deleted - if err = util.SetOwnerReference(router, deploy); err != nil { - klog.Error(err) - return attachmentNetworkName, nil, nil, nil, err + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil } - // bgp sidecar container logic - if router.Spec.BGP.Enabled { - // run BGP in the router container - bgpContainer, err := bgpEdgeRouterContainerBGP(bgpImage, router.Name, &router.Spec.BGP) - if err != nil { - klog.Errorf("failed to create a BGP speaker container for router %s: %v", router.Name, err) - return "", nil, nil, nil, err - } - deploy.Spec.Template.Spec.Containers = append(deploy.Spec.Template.Spec.Containers, *bgpContainer) - } + c.bgpEdgeRouterAdvertisementKeyMutex.LockKey(key) + defer func() { _ = c.bgpEdgeRouterAdvertisementKeyMutex.UnlockKey(key) }() - // generate hash for the workload to determine whether to update the existing workload or not - hash, err := util.Sha256HashObject(deploy) + cachedAdvertisement, err := c.bgpEdgeRouterAdvertisementLister.BgpEdgeRouterAdvertisements(ns).Get(name) if err != nil { - err = fmt.Errorf("failed to hash generated deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) - klog.Error(err) - return attachmentNetworkName, nil, nil, nil, err - } - - hash = hash[:12] - // replicas and the hash annotation should be excluded from hash calculation - deploy.Spec.Replicas = ptr.To(router.Spec.Replicas) - deploy.Annotations = map[string]string{util.GenerateHashAnnotation: hash} - - if currentDeploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(deploy.Name); err != nil { if !k8serrors.IsNotFound(err) { - err = fmt.Errorf("failed to get deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) - klog.Error(err) - return attachmentNetworkName, nil, nil, nil, err - } - if deploy, err = c.config.KubeClient.AppsV1().Deployments(router.Namespace). - Create(context.Background(), deploy, metav1.CreateOptions{}); err != nil { - err = fmt.Errorf("failed to create deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) klog.Error(err) - return attachmentNetworkName, nil, nil, nil, err - } - } else if !reflect.DeepEqual(currentDeploy.Spec.Replicas, deploy.Spec.Replicas) || - currentDeploy.Annotations[util.GenerateHashAnnotation] != hash { - // update the deployment if replicas or hash annotation is changed - if deploy, err = c.config.KubeClient.AppsV1().Deployments(router.Namespace). - Update(context.Background(), deploy, metav1.UpdateOptions{}); err != nil { - err = fmt.Errorf("failed to update deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) - klog.Error(err) - return attachmentNetworkName, nil, nil, nil, err + return err } - } else { - // no need to create or update the deployment - deploy = currentDeploy + return nil } - // return the source CIDR blocks for later OVN resources reconciliation - deploy.APIVersion, deploy.Kind = deploymentGroupVersion, deploymentKind - return attachmentNetworkName, intRouteDstIPv4, intRouteDstIPv6, deploy, nil -} - -func (c *Controller) reconcileBgpEdgeRouterOVNRoutes(router *kubeovnv1.BgpEdgeRouter, af int, lrName, lrpName, bfdIP string, nextHops map[string]string, sources set.Set[string]) error { - if len(nextHops) == 0 { + if !cachedAdvertisement.DeletionTimestamp.IsZero() { + c.delBgpEdgeRouterAdvertisementQueue.Add(key) return nil } - externalIDs := map[string]string{ - ovs.ExternalIDVendor: util.CniTypeName, - ovs.ExternalIDBgpEdgeRouter: fmt.Sprintf("%s/%s", router.Namespace, router.Name), - "af": strconv.Itoa(af), - } - bfdList, err := c.OVNNbClient.FindBFD(externalIDs) - if err != nil { - klog.Error(err) - return err - } + klog.Infof("reconciling bgp-edge-router-advertisement %s", key) + advertisement := cachedAdvertisement.DeepCopy() - // reconcile OVN port group - ports := set.New[string]() - for _, selector := range router.Spec.Selectors { - sel := labels.Everything() - if selector.NamespaceSelector != nil { - if sel, err = metav1.LabelSelectorAsSelector(selector.NamespaceSelector); err != nil { - err = fmt.Errorf("failed to create label selector for namespace selector %#v: %w", selector.NamespaceSelector, err) - klog.Error(err) - return err - } - } - namespaces, err := c.namespacesLister.List(sel) + if controllerutil.AddFinalizer(advertisement, util.KubeOVNControllerFinalizer) { + updatedAdvertisement, err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouterAdvertisements(advertisement.Namespace). + Update(context.Background(), advertisement, metav1.UpdateOptions{}) if err != nil { - err = fmt.Errorf("failed to list namespaces with selector %s: %w", sel, err) + err = fmt.Errorf("failed to add finalizer for bgp-edge-router %s/%s: %w", advertisement.Namespace, advertisement.Name, err) klog.Error(err) return err } - sel = labels.Everything() - if selector.PodSelector != nil { - if sel, err = metav1.LabelSelectorAsSelector(selector.PodSelector); err != nil { - err = fmt.Errorf("failed to create label selector for pod selector %#v: %w", selector.PodSelector, err) - klog.Error(err) - return err - } - } - for _, ns := range namespaces { - pods, err := c.podsLister.Pods(ns.Name).List(sel) - if err != nil { - err = fmt.Errorf("failed to list pods with selector %s in namespace %s: %w", sel, ns.Name, err) - klog.Error(err) - return err - } - for _, pod := range pods { - if pod.Spec.HostNetwork || - pod.Annotations[util.AllocatedAnnotation] != "true" || - pod.Annotations[util.LogicalRouterAnnotation] != router.VPC(c.config.ClusterRouter) || - !isPodAlive(pod) { - continue - } - podName := c.getNameByPod(pod) - ports.Insert(ovs.PodNameToPortName(podName, pod.Namespace, util.OvnProvider)) - } - } - } - key := cache.MetaObjectToName(router).String() - pgName := berPortGroupName(key) - if err = c.OVNNbClient.CreatePortGroup(pgName, externalIDs); err != nil { - err = fmt.Errorf("failed to create port group %s: %w", pgName, err) - klog.Error(err) - return err - } - if err = c.OVNNbClient.PortGroupSetPorts(pgName, ports.UnsortedList()); err != nil { - err = fmt.Errorf("failed to set ports of port group %s: %w", pgName, err) - klog.Error(err) - return err + advertisement = updatedAdvertisement } - // reconcile OVN address set - asName := berAddressSetName(key, af) - if err = c.OVNNbClient.CreateAddressSet(asName, externalIDs); err != nil { - err = fmt.Errorf("failed to create address set %s: %w", asName, err) - klog.Error(err) - return err - } - if err = c.OVNNbClient.AddressSetUpdateAddress(asName, sources.SortedList()...); err != nil { - err = fmt.Errorf("failed to update address set %s: %w", asName, err) + // reconcile the bgp edge router workload and get the route sources for later OVN resources reconciliation + deploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(advertisement.Spec.BgpEdgeRouter) + if err != nil { klog.Error(err) return err } - // reconcile OVN BFD entries - bfdIDs := set.New[string]() - staleBFDIDs := set.New[string]() - bfdDstIPs := set.New(slices.Collect(maps.Values(nextHops))...) - bfdMap := make(map[string]string, bfdDstIPs.Len()) - for _, bfd := range bfdList { - if bfdIP == "" || bfd.LogicalPort != lrpName || !bfdDstIPs.Has(bfd.DstIP) { - staleBFDIDs.Insert(bfd.UUID) - } - if bfdIP == "" || (bfd.LogicalPort == lrpName && bfdDstIPs.Has(bfd.DstIP)) { - // TODO: update min_rx, min_tx and multiplier - if bfdIP != "" { - bfdIDs.Insert(bfd.UUID) - bfdMap[bfd.DstIP] = bfd.UUID - } - bfdDstIPs.Delete(bfd.DstIP) - } - } - if bfdIP != "" { - for _, dstIP := range bfdDstIPs.UnsortedList() { - bfd, err := c.OVNNbClient.CreateBFD(lrpName, dstIP, int(router.Spec.BFD.MinRX), int(router.Spec.BFD.MinTX), int(router.Spec.BFD.Multiplier), externalIDs) - if err != nil { - klog.Error(err) - return err - } - bfdIDs.Insert(bfd.UUID) - bfdMap[bfd.DstIP] = bfd.UUID - } - } - - // reconcile LR policy - if router.Spec.TrafficPolicy == kubeovnv1.TrafficPolicyLocal { - rules := make(map[string]string, len(nextHops)) - for nodeName, nexthop := range nextHops { - node, err := c.nodesLister.Get(nodeName) - if err != nil { - if k8serrors.IsNotFound(err) { - continue - } - klog.Errorf("failed to get node %s: %v", nodeName, err) - return err - } - portName := node.Annotations[util.PortNameAnnotation] - if portName == "" { - err = fmt.Errorf("node %s does not have port name annotation", nodeName) - klog.Error(err) - return err - } - localPgName := strings.ReplaceAll(portName, "-", ".") - rules[fmt.Sprintf("ip%d.src == $%s_ip%d && ip%d.src == $%s_ip%d", af, localPgName, af, af, pgName, af)] = nexthop - rules[fmt.Sprintf("ip%d.src == $%s_ip%d && ip%d.src == $%s", af, localPgName, af, af, asName)] = nexthop - } - policies, err := c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayLocalPolicyPriority, externalIDs, false) - if err != nil { - klog.Error(err) - return err - } - // update/delete existing policies - for _, policy := range policies { - nexthop := rules[policy.Match] - bfdSessions := set.New(bfdMap[nexthop]).Delete("") - if nexthop == "" { - if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { - err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) - klog.Error(err) - return err - } - } else { - var changed bool - if len(policy.Nexthops) != 1 || policy.Nexthops[0] != nexthop { - policy.Nexthops = []string{nexthop} - changed = true - } - if !bfdSessions.Equal(set.New(policy.BFDSessions...)) { - policy.BFDSessions = bfdSessions.UnsortedList() - changed = true - } - if changed { - if err = c.OVNNbClient.UpdateLogicalRouterPolicy(policy, &policy.Nexthops, &policy.BFDSessions); err != nil { - err = fmt.Errorf("failed to update logical router policy %s: %w", policy.UUID, err) - klog.Error(err) - return err - } - } - } - delete(rules, policy.Match) - } - // create new policies - for match, nexthop := range rules { - if err = c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayLocalPolicyPriority, match, - ovnnb.LogicalRouterPolicyActionReroute, []string{nexthop}, []string{bfdMap[nexthop]}, externalIDs); err != nil { - klog.Error(err) - return err - } - } - } else { - if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, util.EgressGatewayLocalPolicyPriority, externalIDs); err != nil { - klog.Error(err) - return err - } + ready := util.DeploymentIsReady(deploy) + if !ready { + readyErr := fmt.Sprintf("Deployment %s is not ready", deploy.Kind, deploy.Name) + klog.Error(readyErr) + return fmt.Errorf(readyErr) } - policies, err := c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayPolicyPriority, externalIDs, false) + // get the pods of the deployment to collect the pod IPs + podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) if err != nil { + err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) klog.Error(err) return err } - matches := set.New( - fmt.Sprintf("ip%d.src == $%s_ip%d", af, pgName, af), - fmt.Sprintf("ip%d.src == $%s", af, asName), - ) - bfdIPs := set.New(slices.Collect(maps.Values(nextHops))...) - bfdSessions := bfdIDs.UnsortedList() - for _, policy := range policies { - if matches.Has(policy.Match) { - if !bfdIPs.Equal(set.New(policy.Nexthops...)) || !bfdIDs.Equal(set.New(policy.BFDSessions...)) { - policy.Nexthops, policy.BFDSessions = bfdIPs.UnsortedList(), bfdSessions - if err = c.OVNNbClient.UpdateLogicalRouterPolicy(policy, &policy.Nexthops, &policy.BFDSessions); err != nil { - err = fmt.Errorf("failed to update bfd sessions of logical router policy %s: %w", policy.UUID, err) - klog.Error(err) - return err - } - } - matches.Delete(policy.Match) - continue - } - if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { - err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) - klog.Error(err) - return err - } - } - for _, match := range matches.UnsortedList() { - if err = c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayPolicyPriority, match, - ovnnb.LogicalRouterPolicyActionReroute, bfdIPs.UnsortedList(), bfdSessions, externalIDs); err != nil { - klog.Error(err) - return err - } - } - if router.Spec.BFD.Enabled { - // drop traffic if no nexthop is available - if policies, err = c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayDropPolicyPriority, externalIDs, false); err != nil { - klog.Error(err) - return err - } - matches = set.New( - fmt.Sprintf("ip%d.src == $%s_ip%d", af, pgName, af), - fmt.Sprintf("ip%d.src == $%s", af, asName), - ) - for _, policy := range policies { - if matches.Has(policy.Match) { - matches.Delete(policy.Match) - continue - } - if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { - err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) - klog.Error(err) - return err - } - } - for _, match := range matches.UnsortedList() { - if err = c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayDropPolicyPriority, match, - ovnnb.LogicalRouterPolicyActionDrop, nil, nil, externalIDs); err != nil { - klog.Error(err) - return err - } - } - } else if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, util.EgressGatewayDropPolicyPriority, externalIDs); err != nil { + pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) + if err != nil { + err = fmt.Errorf("failed to list pods of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) klog.Error(err) return err } - for _, bfdID := range staleBFDIDs.UnsortedList() { - if err = c.OVNNbClient.DeleteBFD(bfdID); err != nil { - err = fmt.Errorf("failed to delete bfd %s: %w", bfdID, err) - klog.Error(err) - return err - } - } - - return nil -} - -func berMergeNodeSelector(nodeSelector []kubeovnv1.BgpEdgeRouterNodeSelector) *corev1.NodeSelector { - if len(nodeSelector) == 0 { - return nil - } - - result := &corev1.NodeSelector{ - NodeSelectorTerms: make([]corev1.NodeSelectorTerm, len(nodeSelector)), - } - for i, selector := range nodeSelector { - result.NodeSelectorTerms[i] = corev1.NodeSelectorTerm{ - MatchExpressions: make([]corev1.NodeSelectorRequirement, len(selector.MatchExpressions), len(selector.MatchLabels)+len(selector.MatchExpressions)), - MatchFields: make([]corev1.NodeSelectorRequirement, len(selector.MatchFields)), - } - for j := range selector.MatchExpressions { - selector.MatchExpressions[j].DeepCopyInto(&result.NodeSelectorTerms[i].MatchExpressions[j]) - } - for _, key := range slices.Sorted(maps.Keys(selector.MatchLabels)) { - result.NodeSelectorTerms[i].MatchExpressions = append(result.NodeSelectorTerms[i].MatchExpressions, corev1.NodeSelectorRequirement{ - Key: key, - Operator: corev1.NodeSelectorOpIn, - Values: []string{selector.MatchLabels[key]}, - }) - } - for j := range selector.MatchFields { - selector.MatchFields[j].DeepCopyInto(&result.NodeSelectorTerms[i].MatchFields[j]) + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue } } - return result -} - -func bgpEdgeRouterInitContainerEnv(af int, internalGateway, externalGateway string, forwardSrc set.Set[string]) ([]corev1.EnvVar, error) { - if internalGateway == "" { - return nil, nil - } - - return []corev1.EnvVar{{ - Name: fmt.Sprintf("INTERNAL_GATEWAY_IPV%d", af), - Value: internalGateway, - }, { - Name: fmt.Sprintf("EXTERNAL_GATEWAY_IPV%d", af), - Value: externalGateway, - }, { - Name: fmt.Sprintf("NO_SNAT_SOURCES_IPV%d", af), - Value: strings.Join(forwardSrc.SortedList(), ","), - }}, nil -} + klog.Infof("finished reconciling bgp-edge-router-advertisement %s", key) -func bgpEdgeRouterContainerBFDD(image, bfdIP string, minTX, minRX, multiplier int32) corev1.Container { - return corev1.Container{ - Name: "bfdd", - Image: image, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"bash", "/kube-ovn/start-bfdd.sh"}, - Env: []corev1.EnvVar{{ - Name: "POD_IPS", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIPs", - }, - }, - }, { - Name: "BFD_PEER_IPS", - Value: bfdIP, - }, { - Name: "BFD_MIN_TX", - Value: strconv.Itoa(int(minTX)), - }, { - Name: "BFD_MIN_RX", - Value: strconv.Itoa(int(minRX)), - }, { - Name: "BFD_MULTI", - Value: strconv.Itoa(int(multiplier)), - }}, - // wait for the BFD process to be running and initialize the BFD configuration - StartupProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - Exec: &corev1.ExecAction{ - Command: []string{"bash", "/kube-ovn/bfdd-prestart.sh"}, - }, - }, - InitialDelaySeconds: 1, - FailureThreshold: 1, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - Exec: &corev1.ExecAction{ - Command: []string{"bfdd-control", "status"}, - }, - }, - InitialDelaySeconds: 1, - PeriodSeconds: 5, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - Exec: &corev1.ExecAction{ - Command: []string{"bfdd-control", "status"}, - }, - }, - InitialDelaySeconds: 3, - PeriodSeconds: 3, - FailureThreshold: 1, - }, - SecurityContext: &corev1.SecurityContext{ - Privileged: ptr.To(false), - RunAsUser: ptr.To[int64](65534), - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"NET_ADMIN", "NET_BIND_SERVICE", "NET_RAW"}, - Drop: []corev1.Capability{"ALL"}, - }, - }, - VolumeMounts: []corev1.VolumeMount{{ - Name: "usr-local-sbin", - MountPath: "/usr/local/sbin", - }}, - } + return nil } -func (c *Controller) handleDelBgpEdgeRouter(key string) error { +func (c *Controller) handleDelBgpEdgeRouterAdvertisement(key string) error { ns, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) @@ -734,208 +201,111 @@ func (c *Controller) handleDelBgpEdgeRouter(key string) error { c.bgpEdgeRouterKeyMutex.LockKey(key) defer func() { _ = c.bgpEdgeRouterKeyMutex.UnlockKey(key) }() - cachedGateway, err := c.bgpEdgeRouterLister.BgpEdgeRouters(ns).Get(name) + cachedAdvertisement, err := c.bgpEdgeRouterAdvertisementLister.BgpEdgeRouterAdvertisements(ns).Get(name) if err != nil { if !k8serrors.IsNotFound(err) { - err = fmt.Errorf("failed to get bgp-edge-router %s: %w", key, err) klog.Error(err) return err } return nil } - klog.Infof("handle deleting bgp-edge-router %s", key) - if err = c.cleanOVNForBgpEdgeRouter(key, cachedGateway.Spec.VPC); err != nil { - klog.Error(err) - return err - } - - router := cachedGateway.DeepCopy() - if controllerutil.RemoveFinalizer(router, util.KubeOVNControllerFinalizer) { - if _, err = c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouters(router.Namespace). - Update(context.Background(), router, metav1.UpdateOptions{}); err != nil { - err = fmt.Errorf("failed to remove finalizer from bgp-edge-router %s: %w", key, err) - klog.Error(err) - } + if !cachedAdvertisement.DeletionTimestamp.IsZero() { + c.delBgpEdgeRouterAdvertisementQueue.Add(key) + return nil } - return nil -} - -func (c *Controller) cleanOVNForBgpEdgeRouter(key, lrName string) error { - externalIDs := map[string]string{ - ovs.ExternalIDVendor: util.CniTypeName, - ovs.ExternalIDBgpEdgeRouter: key, - } + klog.Infof("reconciling bgp-edge-router-advertisement %s", key) + advertisement := cachedAdvertisement.DeepCopy() - bfdList, err := c.OVNNbClient.FindBFD(externalIDs) + // reconcile the bgp edge router workload and get the route sources for later OVN resources reconciliation + deploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(advertisement.Spec.BgpEdgeRouter) if err != nil { klog.Error(err) return err } - for _, bfd := range bfdList { - if err = c.OVNNbClient.DeleteBFD(bfd.UUID); err != nil { - klog.Error(err) - return err - } - } - - if lrName == "" { - lrName = c.config.ClusterRouter - } - if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, -1, externalIDs); err != nil { - klog.Error(err) - return err - } - if err = c.OVNNbClient.DeletePortGroup(berPortGroupName(key)); err != nil { - klog.Error(err) - return err - } - for _, af := range [...]int{4, 6} { - if err = c.OVNNbClient.DeleteAddressSet(berAddressSetName(key, af)); err != nil { - klog.Error(err) - return err - } - } - - return nil -} - -func berPortGroupName(key string) string { - hash := util.Sha256Hash([]byte(key)) - return "BER." + hash[:12] -} -func berAddressSetName(key string, af int) string { - hash := util.Sha256Hash([]byte(key)) - return fmt.Sprintf("BER.%s.ipv%d", hash[:12], af) -} - -func (c *Controller) handlePodEventForBgpEdgeRouter(pod *corev1.Pod) error { - if !pod.DeletionTimestamp.IsZero() || pod.Annotations[util.AllocatedAnnotation] != "true" { - return nil - } - vpc := pod.Annotations[util.LogicalRouterAnnotation] - if vpc == "" { - return nil + ready := util.DeploymentIsReady(deploy) + if !ready { + readyErr := fmt.Sprintf("Deployment %s is not ready", deploy.Kind, deploy.Name) + klog.Error(readyErr) + return fmt.Errorf(readyErr) } - - ns, err := c.namespacesLister.Get(pod.Namespace) + // get the pods of the deployment to collect the pod IPs + podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) if err != nil { - if k8serrors.IsNotFound(err) { - return nil - } - klog.Errorf("failed to get namespace %s: %v", pod.Namespace, err) - utilruntime.HandleError(err) + err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) return err } - router, err := c.bgpEdgeRouterLister.List(labels.Everything()) + pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) if err != nil { - klog.Errorf("failed to list bgp edge router: %v", err) - utilruntime.HandleError(err) + err = fmt.Errorf("failed to list pods of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) return err } - for _, ber := range router { - if ber.VPC(c.config.ClusterRouter) != vpc { + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { continue } + klog.Infof("handle deleting bgp-edge-router-advertisement %s", key) + if err = c.cleanBgpEdgeRouterAdvertisementRule(key, pod.Name, advertisement.Spec.Subnet); err != nil { + klog.Error(err) + return err + } + } - for _, selector := range ber.Spec.Selectors { - if selector.NamespaceSelector != nil && !util.ObjectMatchesLabelSelector(ns, selector.NamespaceSelector) { - continue - } - if selector.PodSelector != nil && !util.ObjectMatchesLabelSelector(pod, selector.PodSelector) { - continue - } - c.addOrUpdateBgpEdgeRouterQueue.Add(cache.MetaObjectToName(ber).String()) + advertisement = cachedAdvertisement.DeepCopy() + if controllerutil.RemoveFinalizer(advertisement, util.KubeOVNControllerFinalizer) { + if _, err = c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouterAdvertisements(advertisement.Namespace). + Update(context.Background(), advertisement, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to remove finalizer from bgp-edge-router-advertisement %s: %w", key, err) + klog.Error(err) } } return nil } -func bgpEdgeRouterContainerBGP(speakerImage, routerName string, speakerParams *kubeovnv1.BgpEdgeRouterBGPConfig) (*corev1.Container, error) { - if speakerImage == "" { - return nil, errors.New("BGP speaker image must be specified") - } - if speakerParams == nil { - return nil, errors.New("BGP config must not be nil") - } - if speakerParams.ASN == 0 { - return nil, errors.New("ASN not set, but must be non-zero value") - } - if speakerParams.RemoteASN == 0 { - return nil, errors.New("remote ASN not set, but must be non-zero value") - } - if len(speakerParams.Neighbors) == 0 { - return nil, errors.New("no BGP neighbors specified") - } +func (c *Controller) cleanBgpEdgeRouterAdvertisementRule(key, podName string, subnetNames []string) error { + // externalIDs := map[string]string{ + // ovs.ExternalIDVendor: util.CniTypeName, + // ovs.ExternalIDBgpEdgeRouter: key, + // } - args := []string{} - if speakerParams.RouterID != "" { - args = append(args, "--router-id="+speakerParams.RouterID) - } - if speakerParams.Password != "" { - args = append(args, "--auth-password="+speakerParams.Password) - } - if speakerParams.EnableGracefulRestart { - args = append(args, "--graceful-restart") - } - if speakerParams.HoldTime != (metav1.Duration{}) { - args = append(args, "--holdtime="+speakerParams.HoldTime.Duration.String()) + if podName == "" { + err := fmt.Errorf("failed to get pod name %s", podName) + klog.Error(err) + return err } - - args = append(args, fmt.Sprintf("--cluster-as=%d", speakerParams.ASN)) - args = append(args, fmt.Sprintf("--neighbor-as=%d", speakerParams.RemoteASN)) - - var neighIPv4, neighIPv6 []string - for _, neighbor := range speakerParams.Neighbors { - switch util.CheckProtocol(neighbor) { - case kubeovnv1.ProtocolIPv4: - neighIPv4 = append(neighIPv4, neighbor) - case kubeovnv1.ProtocolIPv6: - neighIPv6 = append(neighIPv6, neighbor) - default: - return nil, fmt.Errorf("unsupported protocol for peer %s", neighbor) + for _, subnetName := range subnetNames { + if subnet, err := c.subnetsLister.Get(subnetName); err != nil { + err = fmt.Errorf("failed to get subnet %s: %w", subnetName, err) + klog.Error(err) + return err + } else { + if subnet.Spec.CIDRBlock != key { + //delete bgp advertised ipblock + } + klog.Infof("cleaning bgp-edge-router-advertisement %s for subnet %s", key, subnet.Name) } - } - if len(neighIPv4) > 0 { - args = append(args, "--neighbor-address="+strings.Join(neighIPv4, ",")) - } - if len(neighIPv6) > 0 { - args = append(args, "--neighbor-ipv6-address="+strings.Join(neighIPv6, ",")) - } - args = append(args, speakerParams.ExtraArgs...) - - container := &corev1.Container{ - Name: "bgp-router-speaker", - Image: speakerImage, - Command: []string{"/kube-ovn/kube-ovn-speaker"}, - ImagePullPolicy: corev1.PullIfNotPresent, - Env: []corev1.EnvVar{ - { - Name: "EGRESS_GATEWAY_NAME", - Value: routerName, - }, - { - Name: "POD_IP", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIP", - }, - }, - }, - }, - Args: args, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "kube-ovn-logs", - MountPath: "/var/log/kube-ovn", - }, - }, } + // if err = c.OVNNbClient.DeleteLogicalRouterPolicies(podName, -1, externalIDs); err != nil { + // klog.Error(err) + // return err + // } + // if err = c.OVNNbClient.DeletePortGroup(berPortGroupName(key)); err != nil { + // klog.Error(err) + // return err + // } + // for _, af := range [...]int{4, 6} { + // if err = c.OVNNbClient.DeleteAddressSet(berAddressSetName(key, af)); err != nil { + // klog.Error(err) + // return err + // } + // } - return container, nil + return nil } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index e648e274166..14b9a340856 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -113,6 +113,13 @@ type Controller struct { delBgpEdgeRouterQueue workqueue.TypedRateLimitingInterface[string] bgpEdgeRouterKeyMutex keymutex.KeyMutex + bgpEdgeRouterAdvertisementLister kubeovnlister.BgpEdgeRouterAdvertisementLister + bgpEdgeRouterAdvertisementSynced cache.InformerSynced + addBgpEdgeRouterAdvertisementQueue workqueue.TypedRateLimitingInterface[string] + updateBgpEdgeRouterAdvertisementQueue workqueue.TypedRateLimitingInterface[string] + delBgpEdgeRouterAdvertisementQueue workqueue.TypedRateLimitingInterface[string] + bgpEdgeRouterAdvertisementKeyMutex keymutex.KeyMutex + switchLBRuleLister kubeovnlister.SwitchLBRuleLister switchLBRuleSynced cache.InformerSynced addSwitchLBRuleQueue workqueue.TypedRateLimitingInterface[string] @@ -372,6 +379,7 @@ func Run(ctx context.Context, config *Configuration) { vpcNatGatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcNatGateways() vpcEgressGatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcEgressGateways() bgpEdgeRouterInformer := kubeovnInformerFactory.Kubeovn().V1().BgpEdgeRouters() + bgpEdgeRouterAdvertisementInformer := kubeovnInformerFactory.Kubeovn().V1().BgpEdgeRouterAdvertisements() subnetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets() ippoolInformer := kubeovnInformerFactory.Kubeovn().V1().IPPools() ipInformer := kubeovnInformerFactory.Kubeovn().V1().IPs() @@ -443,6 +451,13 @@ func Run(ctx context.Context, config *Configuration) { delBgpEdgeRouterQueue: newTypedRateLimitingQueue("DeleteBgpEdgeRouter", custCrdRateLimiter), bgpEdgeRouterKeyMutex: keymutex.NewHashed(numKeyLocks), + bgpEdgeRouterAdvertisementLister: bgpEdgeRouterAdvertisementInformer.Lister(), + bgpEdgeRouterAdvertisementSynced: bgpEdgeRouterAdvertisementInformer.Informer().HasSynced, + addBgpEdgeRouterAdvertisementQueue: newTypedRateLimitingQueue("AddBgpEdgeRouterAdvertisement", custCrdRateLimiter), + updateBgpEdgeRouterAdvertisementQueue: newTypedRateLimitingQueue("UpdateBgpEdgeRouterAdvertisement", custCrdRateLimiter), + delBgpEdgeRouterAdvertisementQueue: newTypedRateLimitingQueue("DeleteBgpEdgeRouterAdvertisement", custCrdRateLimiter), + bgpEdgeRouterAdvertisementKeyMutex: keymutex.NewHashed(numKeyLocks), + subnetsLister: subnetInformer.Lister(), subnetSynced: subnetInformer.Informer().HasSynced, addOrUpdateSubnetQueue: newTypedRateLimitingQueue[string]("AddSubnet", nil), @@ -1240,6 +1255,9 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(runWorker("delete vpc egress gateway", c.delVpcEgressGatewayQueue, c.handleDelVpcEgressGateway), time.Second, ctx.Done()) go wait.Until(runWorker("add/update bgp edge router", c.addOrUpdateBgpEdgeRouterQueue, c.handleAddOrUpdateBgpEdgeRouter), time.Second, ctx.Done()) go wait.Until(runWorker("delete bgp edge router", c.delBgpEdgeRouterQueue, c.handleDelBgpEdgeRouter), time.Second, ctx.Done()) + go wait.Until(runWorker("add bgp edge router advertisement", c.addBgpEdgeRouterAdvertisementQueue, c.handleAddBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) + go wait.Until(runWorker("update bgp edge router advertisement", c.updateBgpEdgeRouterAdvertisementQueue, c.handleUpdateBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) + go wait.Until(runWorker("delete bgp edge router advertisement", c.delBgpEdgeRouterAdvertisementQueue, c.handleDelBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) go wait.Until(runWorker("update fip for vpc nat gateway", c.updateVpcFloatingIPQueue, c.handleUpdateVpcFloatingIP), time.Second, ctx.Done()) go wait.Until(runWorker("update eip for vpc nat gateway", c.updateVpcEipQueue, c.handleUpdateVpcEip), time.Second, ctx.Done()) go wait.Until(runWorker("update dnat for vpc nat gateway", c.updateVpcDnatQueue, c.handleUpdateVpcDnat), time.Second, ctx.Done()) diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go index 9e340d64d9c..cc4c44b59dc 100644 --- a/pkg/ovs/ovn.go +++ b/pkg/ovs/ovn.go @@ -46,9 +46,10 @@ const ( OVSDBWaitTimeout = 0 - ExternalIDVendor = "vendor" - ExternalIDVpcEgressGateway = "vpc-egress-gateway" - ExternalIDBgpEdgeRouter = "bgp-edge-router" + ExternalIDVendor = "vendor" + ExternalIDVpcEgressGateway = "vpc-egress-gateway" + ExternalIDBgpEdgeRouter = "bgp-edge-router" + ExternalIDBgpEdgeRouterAdvertisement = "bgp-edge-router-advertisement" ) // NewLegacyClient init a legacy ovn client diff --git a/pkg/util/const.go b/pkg/util/const.go index 8566235d53b..c7772f07db7 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -113,9 +113,10 @@ const ( NetworkPolicyLogAnnotation = "ovn.kubernetes.io/enable_log" ACLActionsLogAnnotation = "ovn.kubernetes.io/log_acl_actions" - VpcEgressGatewayLabel = "ovn.kubernetes.io/vpc-egress-gateway" - BgpEdgeRouterLabel = "ovn.kubernetes.io/bgp-edge-router" - GenerateHashAnnotation = "ovn.kubernetes.io/generate-hash" + VpcEgressGatewayLabel = "ovn.kubernetes.io/vpc-egress-gateway" + BgpEdgeRouterLabel = "ovn.kubernetes.io/bgp-edge-router" + BgpEdgeRouterAdvertisementLabel = "ovn.kubernetes.io/bgp-edge-router-advertisement" + GenerateHashAnnotation = "ovn.kubernetes.io/generate-hash" VpcLastName = "ovn.kubernetes.io/last_vpc_name" VpcLastPolicies = "ovn.kubernetes.io/last_policies" From 62a920ba506e2de6d3e38d5df87051b4ce6b2f5e Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Wed, 30 Jul 2025 01:28:06 -0700 Subject: [PATCH 17/66] bgp edge router advertisement modification. --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 239 +++--------------- .../v1/bgp-edge-router-advertisement.go | 8 +- .../bgp_edge_router_advertisement.go | 218 +++++++++------- pkg/controller/controller.go | 8 + 4 files changed, 174 insertions(+), 299 deletions(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index 0b9857e6770..6993708a6ff 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3599,11 +3599,14 @@ spec: listKind: BgpEdgeRouterAdvertisementList scope: Namespaced versions: - - additionalPrinterColumns: + - name: v1 + served: true + storage: true + additionalPrinterColumns: - jsonPath: .spec.subnet name: SUBNET type: string - - jsonPath: .spec.bgpedgerouter + - jsonPath: .spec.bgpEdgeRouter name: BGP-EDGE-ROUTER type: string - jsonPath: .status.ready @@ -3612,60 +3615,64 @@ spec: - jsonPath: .metadata.creationTimestamp name: AGE type: date - name: v1 - served: true - storage: true subresources: status: {} scale: - # specReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Spec.Replicas. specReplicasPath: .spec.replicas - # statusReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Replicas. statusReplicasPath: .status.replicas - # labelSelectorPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Selector. labelSelectorPath: .status.labelSelector schema: openAPIV3Schema: type: object properties: - status: + spec: + type: object properties: - replicas: - type: integer - format: int32 - labelSelector: + subnet: + type: array + items: + type: string + bgpEdgeRouter: type: string + required: + - subnet + - bgpEdgeRouter + status: + type: object + properties: conditions: + type: array items: + type: object properties: lastTransitionTime: - format: date-time type: string - lastUpdateTime: format: date-time + lastUpdateTime: type: string + format: date-time message: - maxLength: 32768 type: string + maxLength: 32768 observedGeneration: + type: integer format: int64 minimum: 0 - type: integer reason: - maxLength: 1024 + type: string minLength: 1 + maxLength: 1024 pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string status: + type: string enum: - "True" - "False" - Unknown - type: string type: + type: string maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string required: - lastTransitionTime - lastUpdateTime @@ -3673,200 +3680,12 @@ spec: - reason - status - type - type: object - type: array x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map - internalIPs: - items: - type: string - type: array - externalIPs: - items: - type: string - type: array - phase: - type: string - default: Pending - enum: - - Pending - - Processing - - Completed ready: type: boolean default: false - workload: - type: object - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - nodes: - type: array - items: - type: string required: - conditions - - phase - type: object - spec: - type: object - required: - - externalSubnet - x-kubernetes-validations: - - rule: "!has(self.internalIPs) || size(self.internalIPs) == 0 || size(self.internalIPs) >= self.replicas" - message: 'Size of Internal IPs MUST be equal to or greater than Replicas' - fieldPath: ".internalIPs" - - rule: "!has(self.externalIPs) || size(self.externalIPs) == 0 || size(self.externalIPs) >= self.replicas" - message: 'Size of External IPs MUST be equal to or greater than Replicas' - fieldPath: ".externalIPs" - - rule: "size(self.policies) != 0 || size(self.selectors) != 0" - message: 'Each BGP Edeg Router MUST have at least one policy or selector' - properties: - - - - selectors: - type: array - items: - type: object - properties: - namespaceSelector: - type: object - properties: - matchLabels: - additionalProperties: - type: string - type: object - matchExpressions: - type: array - items: - type: object - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - x-kubernetes-validations: - - rule: "size(self.matchLabels) != 0 || size(self.matchExpressions) != 0" - message: 'Each namespace selector MUST have at least one matchLabels or matchExpressions' - podSelector: - type: object - properties: - matchLabels: - additionalProperties: - type: string - type: object - matchExpressions: - type: array - items: - type: object - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - x-kubernetes-validations: - - rule: "size(self.matchLabels) != 0 || size(self.matchExpressions) != 0" - message: 'Each pod selector MUST have at least one matchLabels or matchExpressions' - policies: - type: array - items: - type: object - properties: - snat: - type: boolean - default: false - ipBlocks: - type: array - x-kubernetes-list-type: set - items: - type: string - anyOf: - - format: ipv4 - - format: ipv6 - - format: cidr - subnets: - type: array - x-kubernetes-list-type: set - items: - type: string - minLength: 1 - x-kubernetes-validations: - - rule: "size(self.ipBlocks) != 0 || size(self.subnets) != 0" - message: 'Each policy MUST have at least one ipBlock or subnet' - nodeSelector: - type: array - items: - type: object - properties: - matchLabels: - additionalProperties: - type: string - type: object - matchExpressions: - type: array - items: - type: object - properties: - key: - type: string - operator: - type: string - enum: - - In - - NotIn - - Exists - - DoesNotExist - - Gt - - Lt - values: - type: array - x-kubernetes-list-type: set - items: - type: string - required: - - key - - operator - matchFields: - type: array - items: - type: object - properties: - key: - type: string - operator: - type: string - enum: - - In - - NotIn - - Exists - - DoesNotExist - - Gt - - Lt - values: - type: array - x-kubernetes-list-type: set - items: - type: string - required: - - key - - operator + - ready \ No newline at end of file diff --git a/pkg/apis/kubeovn/v1/bgp-edge-router-advertisement.go b/pkg/apis/kubeovn/v1/bgp-edge-router-advertisement.go index 61ba91b1462..480b2087acf 100644 --- a/pkg/apis/kubeovn/v1/bgp-edge-router-advertisement.go +++ b/pkg/apis/kubeovn/v1/bgp-edge-router-advertisement.go @@ -23,7 +23,8 @@ type BgpEdgeRouterAdvertisement struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata"` - Spec BgpEdgeRouterAdvertisementSpec `json:"spec"` + Spec BgpEdgeRouterAdvertisementSpec `json:"spec"` + Status BgpEdgeRouterAdvertisementStatus `json:"status"` } // If the BgpEdgeRouter has no VPC specified in the spec, it will return the default VPC name @@ -43,3 +44,8 @@ type BgpEdgeRouterAdvertisementSpec struct { Subnet []string `json:"subnet,omitempty"` BgpEdgeRouter string `json:"bgpEdgeRouter,omitempty"` } + +type BgpEdgeRouterAdvertisementStatus struct { + Ready bool `json:"ready"` + Conditions Conditions `json:"conditions,omitempty"` +} diff --git a/pkg/controller/bgp_edge_router_advertisement.go b/pkg/controller/bgp_edge_router_advertisement.go index 7d4d9db8c1d..8f8a063ead0 100644 --- a/pkg/controller/bgp_edge_router_advertisement.go +++ b/pkg/controller/bgp_edge_router_advertisement.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -33,10 +34,6 @@ func (c *Controller) enqueueDeleteBgpEdgeRouterAdvertisement(obj any) { c.delBgpEdgeRouterAdvertisementQueue.Add(key) } -func bgpEdgeRouterAdvertisementWorkloadLabels(bgpEdgeRouterAdvertisementName string) map[string]string { - return map[string]string{"app": "bgp-edge-router-advertisement", util.BgpEdgeRouterLabel: bgpEdgeRouterAdvertisementName} -} - func (c *Controller) handleAddBgpEdgeRouterAdvertisement(key string) error { ns, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { @@ -61,6 +58,11 @@ func (c *Controller) handleAddBgpEdgeRouterAdvertisement(key string) error { return nil } + if _, err := c.initBgpEdgeRouterAdvertisementStatus(cachedAdvertisement); err != nil { + klog.Error(err) + return err + } + klog.Infof("reconciling bgp-edge-router-advertisement %s", key) advertisement := cachedAdvertisement.DeepCopy() @@ -75,37 +77,19 @@ func (c *Controller) handleAddBgpEdgeRouterAdvertisement(key string) error { advertisement = updatedAdvertisement } - // reconcile the bgp edge router workload and get the route sources for later OVN resources reconciliation - deploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(advertisement.Spec.BgpEdgeRouter) - if err != nil { - klog.Error(err) - return err - } - - ready := util.DeploymentIsReady(deploy) - if !ready { - readyErr := fmt.Sprintf("Deployment %s is not ready", deploy.Kind, deploy.Name) - klog.Error(readyErr) - return fmt.Errorf(readyErr) - } - // get the pods of the deployment to collect the pod IPs - podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) - if err != nil { - err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) - klog.Error(err) - return err - } - - pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) - if err != nil { - err = fmt.Errorf("failed to list pods of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + if pods, err := c.validateBgpEdgeRouterAdvertisement(advertisement); err != nil || pods == nil { klog.Error(err) return err - } - - for _, pod := range pods { - if len(pod.Status.PodIPs) == 0 { - continue + } else { + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + klog.Infof("handle deleting bgp-edge-router-advertisement %s", key) + if err = c.addBgpEdgeRouterAdvertisementRule(key, pod.Name, advertisement.Spec.Subnet); err != nil { + klog.Error(err) + return err + } } } @@ -161,9 +145,14 @@ func (c *Controller) handleUpdateBgpEdgeRouterAdvertisement(key string) error { ready := util.DeploymentIsReady(deploy) if !ready { - readyErr := fmt.Sprintf("Deployment %s is not ready", deploy.Kind, deploy.Name) + advertisement.Status.Ready = false + msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) + // advertisement.Status.Conditions.SetCondition(kubeovnv1.Ready, corev1.ConditionFalse, "Processing", msg, advertisement.Generation) + advertisement.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "BgpEdgeRouterNotEnabled", msg, advertisement.Generation) + _, _ = c.updatebgpEdgeRouterAdvertisementStatus(advertisement) + readyErr := fmt.Sprintf("Kind %s, Deployment %s is not ready", deploy.Kind, deploy.Name) klog.Error(readyErr) - return fmt.Errorf(readyErr) + return fmt.Errorf("%s", readyErr) } // get the pods of the deployment to collect the pod IPs podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) @@ -218,42 +207,19 @@ func (c *Controller) handleDelBgpEdgeRouterAdvertisement(key string) error { klog.Infof("reconciling bgp-edge-router-advertisement %s", key) advertisement := cachedAdvertisement.DeepCopy() - // reconcile the bgp edge router workload and get the route sources for later OVN resources reconciliation - deploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(advertisement.Spec.BgpEdgeRouter) - if err != nil { - klog.Error(err) - return err - } - - ready := util.DeploymentIsReady(deploy) - if !ready { - readyErr := fmt.Sprintf("Deployment %s is not ready", deploy.Kind, deploy.Name) - klog.Error(readyErr) - return fmt.Errorf(readyErr) - } - // get the pods of the deployment to collect the pod IPs - podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) - if err != nil { - err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) - klog.Error(err) - return err - } - - pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) - if err != nil { - err = fmt.Errorf("failed to list pods of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + if pods, err := c.validateBgpEdgeRouterAdvertisement(advertisement); err != nil || pods == nil { klog.Error(err) return err - } - - for _, pod := range pods { - if len(pod.Status.PodIPs) == 0 { - continue - } - klog.Infof("handle deleting bgp-edge-router-advertisement %s", key) - if err = c.cleanBgpEdgeRouterAdvertisementRule(key, pod.Name, advertisement.Spec.Subnet); err != nil { - klog.Error(err) - return err + } else { + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + klog.Infof("handle deleting bgp-edge-router-advertisement %s", key) + if err = c.cleanBgpEdgeRouterAdvertisementRule(key, pod.Name, advertisement.Spec.Subnet); err != nil { + klog.Error(err) + return err + } } } @@ -269,10 +235,6 @@ func (c *Controller) handleDelBgpEdgeRouterAdvertisement(key string) error { } func (c *Controller) cleanBgpEdgeRouterAdvertisementRule(key, podName string, subnetNames []string) error { - // externalIDs := map[string]string{ - // ovs.ExternalIDVendor: util.CniTypeName, - // ovs.ExternalIDBgpEdgeRouter: key, - // } if podName == "" { err := fmt.Errorf("failed to get pod name %s", podName) @@ -285,27 +247,107 @@ func (c *Controller) cleanBgpEdgeRouterAdvertisementRule(key, podName string, su klog.Error(err) return err } else { - if subnet.Spec.CIDRBlock != key { + if subnet.Spec.CIDRBlock != "" { + //delete bgp advertised ipblock + } + klog.Infof("cleaning bgp-edge-router-advertisement %s for subnet %s", key, subnet.Name) + } + + } + + return nil +} + +func (c *Controller) addBgpEdgeRouterAdvertisementRule(key, podName string, subnetNames []string) error { + + if podName == "" { + err := fmt.Errorf("failed to get pod name %s", podName) + klog.Error(err) + return err + } + for _, subnetName := range subnetNames { + if subnet, err := c.subnetsLister.Get(subnetName); err != nil { + err = fmt.Errorf("failed to get subnet %s: %w", subnetName, err) + klog.Error(err) + return err + } else { + if subnet.Spec.CIDRBlock != "" { //delete bgp advertised ipblock } klog.Infof("cleaning bgp-edge-router-advertisement %s for subnet %s", key, subnet.Name) } } - // if err = c.OVNNbClient.DeleteLogicalRouterPolicies(podName, -1, externalIDs); err != nil { - // klog.Error(err) - // return err - // } - // if err = c.OVNNbClient.DeletePortGroup(berPortGroupName(key)); err != nil { - // klog.Error(err) - // return err - // } - // for _, af := range [...]int{4, 6} { - // if err = c.OVNNbClient.DeleteAddressSet(berAddressSetName(key, af)); err != nil { - // klog.Error(err) - // return err - // } - // } return nil } + +func (c *Controller) validateBgpEdgeRouterAdvertisement(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement) ([]*corev1.Pod, error) { + + deploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(advertisement.Spec.BgpEdgeRouter) + if err != nil { + advertisement.Status.Ready = false + msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) + advertisement.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "BgpEdgeRouterDeployNotFound", msg, advertisement.Generation) + _, _ = c.updatebgpEdgeRouterAdvertisementStatus(advertisement) + klog.Error(err) + return nil, err + } + + ready := util.DeploymentIsReady(deploy) + if !ready { + advertisement.Status.Ready = false + msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) + advertisement.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "BgpEdgeRouterNotEnabled", msg, advertisement.Generation) + _, _ = c.updatebgpEdgeRouterAdvertisementStatus(advertisement) + readyErr := fmt.Sprintf("Kind %s, Deployment %s is not ready", deploy.Kind, deploy.Name) + klog.Error(readyErr) + return nil, fmt.Errorf("%s", readyErr) + } + // get the pods of the deployment to collect the pod IPs + podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) + if err != nil { + err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return nil, err + } + + pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) + if err != nil { + err = fmt.Errorf("failed to list pods of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return nil, err + } + + if ready { + advertisement.Status.Ready = true + advertisement.Status.Conditions.SetReady("ReconcileSuccess", advertisement.Generation) + if _, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement); err != nil { + return pods, err + } + } + + return pods, nil +} + +func (c *Controller) initBgpEdgeRouterAdvertisementStatus(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) { + var err error + advertisement, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement) + return advertisement, err +} + +func (c *Controller) updatebgpEdgeRouterAdvertisementStatus(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) { + if len(advertisement.Status.Conditions) == 0 { + advertisement.Status.Conditions.SetCondition(kubeovnv1.Init, corev1.ConditionUnknown, "Processing", "", advertisement.Generation) + } + + updateAdvertisement, err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouterAdvertisements(advertisement.Namespace). + UpdateStatus(context.Background(), advertisement, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to update status of bgp-edge-router %s/%s: %w", advertisement.Namespace, advertisement.Name, err) + klog.Error(err) + return nil, err + } + + return updateAdvertisement, nil +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 14b9a340856..93bb17c1328 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -808,6 +808,14 @@ func Run(ctx context.Context, config *Configuration) { util.LogFatalAndExit(err, "failed to add bgp edge router event handler") } + if _, err = bgpEdgeRouterAdvertisementInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddBgpEdgeRouterAdvertisement, + UpdateFunc: controller.enqueueUpdateBgpEdgeRouterAdvertisement, + DeleteFunc: controller.enqueueDeleteBgpEdgeRouterAdvertisement, + }); err != nil { + util.LogFatalAndExit(err, "failed to add bgp edge router advertisement event handler") + } + if _, err = vpcEgressGatewayInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueAddVpcEgressGateway, UpdateFunc: controller.enqueueUpdateVpcEgressGateway, From 8d4475c0d7ad4599a8ad7ff9beb884e246d01a21 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Thu, 31 Jul 2025 01:22:44 -0700 Subject: [PATCH 18/66] bgp edge router modification --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 37 +-- dist/images/update-bgp-route.sh | 312 ++++++++++++++++++ pkg/controller/bgp_edge_router.go | 8 + .../bgp_edge_router_advertisement.go | 280 ++++++++++------ pkg/controller/controller.go | 14 +- pkg/util/const.go | 2 +- 6 files changed, 528 insertions(+), 125 deletions(-) create mode 100644 dist/images/update-bgp-route.sh diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index 6993708a6ff..75c5f7b74cd 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3599,10 +3599,7 @@ spec: listKind: BgpEdgeRouterAdvertisementList scope: Namespaced versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: + - additionalPrinterColumns: - jsonPath: .spec.subnet name: SUBNET type: string @@ -3615,12 +3612,11 @@ spec: - jsonPath: .metadata.creationTimestamp name: AGE type: date + name: v1 + served: true + storage: true subresources: status: {} - scale: - specReplicasPath: .spec.replicas - statusReplicasPath: .status.replicas - labelSelectorPath: .status.labelSelector schema: openAPIV3Schema: type: object @@ -3630,6 +3626,8 @@ spec: properties: subnet: type: array + minItems: 1 + x-kubernetes-list-type: set items: type: string bgpEdgeRouter: @@ -3638,41 +3636,38 @@ spec: - subnet - bgpEdgeRouter status: - type: object properties: conditions: - type: array items: - type: object properties: lastTransitionTime: - type: string format: date-time - lastUpdateTime: type: string + lastUpdateTime: format: date-time - message: type: string + message: maxLength: 32768 + type: string observedGeneration: - type: integer format: int64 minimum: 0 + type: integer reason: - type: string - minLength: 1 maxLength: 1024 + minLength: 1 pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - status: type: string + status: enum: - "True" - "False" - Unknown - type: type: string + type: maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string required: - lastTransitionTime - lastUpdateTime @@ -3680,6 +3675,8 @@ spec: - reason - status - type + type: object + type: array x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map @@ -3688,4 +3685,4 @@ spec: default: false required: - conditions - - ready \ No newline at end of file + type: object diff --git a/dist/images/update-bgp-route.sh b/dist/images/update-bgp-route.sh new file mode 100644 index 00000000000..bc322eb5f09 --- /dev/null +++ b/dist/images/update-bgp-route.sh @@ -0,0 +1,312 @@ +#!/usr/bin/env bash +# update-bgp-route.sh +# shellcheck disable=SC2086,SC2155 + + +set -euo pipefail + + +GOBGP_BIN=${GOBGP_BIN:-$(command -v gobgp || true)} +[[ -z "$GOBGP_BIN" ]] && { echo "gobgp binary not found" >&2; exit 1; } + + +die() { echo "ERROR: $*" >&2; exit 1; } + + +external_iface="net1" +external_ipv4=$(ip addr show dev "${external_iface}" | grep 'inet ' | awk '{print $2}' | cut -d'/' -f1) +[[ -z "$external_ipv4" ]] && die "cannot determine external IPv4 address" + + +exec_cmd() { + "$@" || die "failed: $*" +} + + +check_inited() { + $GOBGP_BIN global rib &>/dev/null \ + || die "gobgp global RIB not initialized (did you 'gobgp global'?)" +} + + +add_announced_route() { + check_inited + echo "Adding routes..." + for cidr in "$@"; do + if [[ $cidr == *:* ]]; then + family_flag="-a ipv6" + else + family_flag="-a ipv4" + fi + echo " + Adding: $cidr" + exec_cmd $GOBGP_BIN global rib $family_flag add \ + "$cidr" nexthop "$external_ipv4" origin igp + done + echo "" +} + + +del_announced_route() { + check_inited + echo "Deleting routes..." + for cidr in "$@"; do + if [[ $cidr == *:* ]]; then + family_flag="-a ipv6" + else + family_flag="-a ipv4" + fi + echo " - Deleting: $cidr" + exec_cmd $GOBGP_BIN global rib $family_flag del \ + "$cidr" nexthop "$external_ipv4" origin igp + done + echo "" +} + + +list_announced_route() { + check_inited + + echo "=== BGP Global RIB ===" + echo "External Interface: $external_iface" + echo "External IPv4: $external_ipv4" + echo "" + + # Show IPv4 routes + echo "--- IPv4 Routes ---" + if $GOBGP_BIN global rib -a ipv4 2>/dev/null | grep -q "Network"; then + $GOBGP_BIN global rib -a ipv4 + else + echo "No IPv4 routes found" + fi + + echo "" + + # Show IPv6 routes + echo "--- IPv6 Routes ---" + if $GOBGP_BIN global rib -a ipv6 2>/dev/null | grep -q "Network"; then + $GOBGP_BIN global rib -a ipv6 + else + echo "No IPv6 routes found" + fi + + echo "" + + # Show routes that match our external IP as next-hop + echo "--- Routes with Next-Hop $external_ipv4 ---" + local found_matching=false + + # Check IPv4 routes with matching next-hop + if $GOBGP_BIN global rib -a ipv4 2>/dev/null | awk -v nh="$external_ipv4" 'NR==1 || $3 == nh' | grep -v "Network" | grep -q .; then + echo "IPv4 routes with next-hop $external_ipv4:" + $GOBGP_BIN global rib -a ipv4 | awk -v nh="$external_ipv4" 'NR==1 || $3 == nh' + found_matching=true + fi + + # Check IPv6 routes with matching next-hop + if $GOBGP_BIN global rib -a ipv6 2>/dev/null | awk -v nh="$external_ipv4" 'NR==1 || $3 == nh' | grep -v "Network" | grep -q .; then + echo "IPv6 routes with next-hop $external_ipv4:" + $GOBGP_BIN global rib -a ipv6 | awk -v nh="$external_ipv4" 'NR==1 || $3 == nh' + found_matching=true + fi + + if [[ "$found_matching" == false ]]; then + echo "No routes found with next-hop $external_ipv4" + fi + + echo "" + echo "==========================================" + echo "" +} + + +parse_sequential_args() { + local operations=() + local operation_args=() + + # Parse arguments and store operations in order + for arg in "$@"; do + case "$arg" in + add_announced_route=*|add_announce_routes=*) + operations+=("add") + operation_args+=("${arg#*=}") + ;; + del_announced_route=*|del_announce_routes=*) + operations+=("del") + operation_args+=("${arg#*=}") + ;; + list_announced_route) + operations+=("list") + operation_args+=("") + ;; + *) + echo "Unknown argument: $arg" >&2 + usage + ;; + esac + done + + # Execute operations in order + for i in "${!operations[@]}"; do + case "${operations[$i]}" in + list) + list_announced_route + ;; + del) + # Parse comma-separated CIDRs + IFS=',' read -ra del_cidrs <<< "${operation_args[$i]}" + # Remove leading and trailing spaces + for j in "${!del_cidrs[@]}"; do + del_cidrs[$j]=$(echo "${del_cidrs[$j]}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + done + del_announced_route "${del_cidrs[@]}" + ;; + add) + # Parse comma-separated CIDRs + IFS=',' read -ra add_cidrs <<< "${operation_args[$i]}" + # Remove leading and trailing spaces + for j in "${!add_cidrs[@]}"; do + add_cidrs[$j]=$(echo "${add_cidrs[$j]}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + done + add_announced_route "${add_cidrs[@]}" + ;; + esac + done +} + + +parse_key_value_args() { + local add_routes="" + local del_routes="" + local list_routes=false + + # find key=value + for arg in "$@"; do + case "$arg" in + add_announced_route=*|add_announce_routes=*) + add_routes="${arg#*=}" # extract after = values + ;; + del_announced_route=*|del_announce_routes=*) + del_routes="${arg#*=}" + ;; + list_announced_route) + list_routes=true + ;; + *) + echo "Unknown argument: $arg" >&2 + usage + ;; + esac + done + + if [[ "$list_routes" == true ]]; then + list_announced_route + return 0 + fi + + if [[ -n "$del_routes" ]]; then + echo "Processing del_announced_route: $del_routes" + # change cidrs to array + IFS=',' read -ra del_cidrs <<< "$del_routes" + # remove leading and trailing spaces + for i in "${!del_cidrs[@]}"; do + del_cidrs[$i]=$(echo "${del_cidrs[$i]}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + done + del_announced_route "${del_cidrs[@]}" + fi + + if [[ -n "$add_routes" ]]; then + echo "Processing add_announced_route: $add_routes" + # change cidrs to array + IFS=',' read -ra add_cidrs <<< "$add_routes" + # remove leading and trailing spaces from each CIDR + for i in "${!add_cidrs[@]}"; do + add_cidrs[$i]=$(echo "${add_cidrs[$i]}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + done + add_announced_route "${add_cidrs[@]}" + fi +} + + +usage() { + cat >&2 < [CIDR ...] + 2. Key-Value Arguments: $0 add_announced_route=CIDR1,CIDR2 [del_announced_route=CIDR3,CIDR4] [list_announced_route] + 3. Sequential Processing: $0 list_announced_route del_announce_routes=CIDR1,CIDR2 add_announce_routes=CIDR3,CIDR4 list_announced_route + + +Examples: + $0 add_announced_route 10.100.0.0/24 192.168.1.0/24 + $0 del_announced_route 10.100.0.0/24 192.168.1.0/24 + $0 list_announced_route + $0 add_announced_route=10.100.0.0/24,10.100.1.0/24 del_announced_route=10.0.0.0/24,10.0.1.0/24 + $0 list_announced_route del_announce_routes=10.100.0.0/24,10.100.1.0/24 add_announce_routes=10.0.0.0/24,10.0.1.0/24 list_announced_route + +Note: Both 'del_announced_route=' and 'del_announce_routes=' are supported (same for add operations) +EOF + exit 1 +} + + +has_sequential_processing() { + local has_list=false + local has_operations=false + + for arg in "$@"; do + case "$arg" in + list_announced_route) + has_list=true + ;; + add_announced_route=*|add_announce_routes=*|del_announced_route=*|del_announce_routes=*) + has_operations=true + ;; + esac + done + + # Return true if we have list + operations (sequential processing) + [[ "$has_list" == true && "$has_operations" == true ]] +} + + +has_key_value_args() { + for arg in "$@"; do + case "$arg" in + *=*|list_announced_route) + return 0 # key=value or list command + ;; + esac + done + return 1 # key=value not found +} + + +# main entry point +main() { + # if no arguments are provided, show usage + [[ $# -eq 0 ]] && usage + + # check if we need sequential processing (list + operations) + if has_sequential_processing "$@"; then + parse_sequential_args "$@" + return 0 + fi + + # check if key=value arguments are used + if has_key_value_args "$@"; then + parse_key_value_args "$@" + return 0 + fi + + [[ $# -lt 1 ]] && usage + + local op=$1; shift + case "$op" in + add_announced_route) add_announced_route "$@" ;; + del_announced_route) del_announced_route "$@" ;; + list_announced_route) list_announced_route ;; + *) usage ;; + esac +} + + +main "$@" \ No newline at end of file diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go index 60d8d93fbf7..9008c3b72c8 100644 --- a/pkg/controller/bgp_edge_router.go +++ b/pkg/controller/bgp_edge_router.go @@ -1142,6 +1142,14 @@ func bgpEdgeRouterContainerBGP(speakerImage, routerName string, speakerParams *k MountPath: "/var/log/kube-ovn", }, }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](0), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_BIND_SERVICE", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, } return container, nil diff --git a/pkg/controller/bgp_edge_router_advertisement.go b/pkg/controller/bgp_edge_router_advertisement.go index 8f8a063ead0..23453bb1d35 100644 --- a/pkg/controller/bgp_edge_router_advertisement.go +++ b/pkg/controller/bgp_edge_router_advertisement.go @@ -2,7 +2,9 @@ package controller import ( "context" + "errors" "fmt" + "strings" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -16,22 +18,55 @@ import ( "github.com/kubeovn/kube-ovn/pkg/util" ) +type updateVerObject struct { + key string + oldVer *kubeovnv1.BgpEdgeRouterAdvertisement + newVer *kubeovnv1.BgpEdgeRouterAdvertisement +} + func (c *Controller) enqueueAddBgpEdgeRouterAdvertisement(obj any) { key := cache.MetaObjectToName(obj.(*kubeovnv1.BgpEdgeRouterAdvertisement)).String() klog.V(3).Infof("enqueue add bgp-edge-router-advertisement %s", key) c.addBgpEdgeRouterAdvertisementQueue.Add(key) } -func (c *Controller) enqueueUpdateBgpEdgeRouterAdvertisement(_, newObj any) { +func (c *Controller) enqueueUpdateBgpEdgeRouterAdvertisement(oldObj, newObj any) { key := cache.MetaObjectToName(newObj.(*kubeovnv1.BgpEdgeRouterAdvertisement)).String() klog.V(3).Infof("enqueue update bgp-edge-router-advertisement %s", key) - c.updateBgpEdgeRouterAdvertisementQueue.Add(key) + + if oldObj == nil || newObj == nil { + klog.Warningf("enqueue update bgp-edge-router-advertisement %s, but old object is nil", key) + return + } + + oldRouter := oldObj.(*kubeovnv1.BgpEdgeRouterAdvertisement) + newRouter := newObj.(*kubeovnv1.BgpEdgeRouterAdvertisement) + updateVer := &updateVerObject{ + key: key, + oldVer: oldRouter, + newVer: newRouter, + } + + c.updateBgpEdgeRouterAdvertisementQueue.Add(updateVer) } func (c *Controller) enqueueDeleteBgpEdgeRouterAdvertisement(obj any) { + var berAd *kubeovnv1.BgpEdgeRouterAdvertisement + switch t := obj.(type) { + case *kubeovnv1.BgpEdgeRouterAdvertisement: + berAd = t + case cache.DeletedFinalStateUnknown: + if v, ok := t.Obj.(*kubeovnv1.BgpEdgeRouterAdvertisement); ok { + berAd = v + } + } + if berAd == nil { + klog.Warning("enqueueDeleteBgpEdgeRouterAdvertisement: object is not BgpEdgeRouterAdvertisement") + return + } key := cache.MetaObjectToName(obj.(*kubeovnv1.BgpEdgeRouterAdvertisement)).String() klog.V(3).Infof("enqueue delete bgp-edge-router-advertisement %s", key) - c.delBgpEdgeRouterAdvertisementQueue.Add(key) + c.deleteBgpEdgeRouterAdvertisementQueue.Add(key) } func (c *Controller) handleAddBgpEdgeRouterAdvertisement(key string) error { @@ -54,9 +89,10 @@ func (c *Controller) handleAddBgpEdgeRouterAdvertisement(key string) error { } if !cachedAdvertisement.DeletionTimestamp.IsZero() { - c.delBgpEdgeRouterAdvertisementQueue.Add(key) + c.deleteBgpEdgeRouterAdvertisementQueue.Add(key) return nil } + klog.V(3).Infof("debug bgp-edge-router-advertisement %s", cachedAdvertisement.Name) if _, err := c.initBgpEdgeRouterAdvertisementStatus(cachedAdvertisement); err != nil { klog.Error(err) @@ -77,28 +113,35 @@ func (c *Controller) handleAddBgpEdgeRouterAdvertisement(key string) error { advertisement = updatedAdvertisement } - if pods, err := c.validateBgpEdgeRouterAdvertisement(advertisement); err != nil || pods == nil { + pods, err := c.validateBgpEdgeRouterAdvertisement(advertisement) + if err != nil || pods == nil { klog.Error(err) return err - } else { - for _, pod := range pods { - if len(pod.Status.PodIPs) == 0 { - continue - } - klog.Infof("handle deleting bgp-edge-router-advertisement %s", key) - if err = c.addBgpEdgeRouterAdvertisementRule(key, pod.Name, advertisement.Spec.Subnet); err != nil { - klog.Error(err) - return err - } + } + + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + klog.Infof("handle adding bgp-edge-router-advertisement %s", key) + if err = c.addOrDeleteBgpEdgeRouterAdvertisementRule("add", key, pod, advertisement.Spec.Subnet); err != nil { + klog.Error(err) + return err } } + advertisement.Status.Conditions.SetReady("ReconcileSuccess", advertisement.Generation) + if _, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement); err != nil { + return err + } klog.Infof("finished reconciling bgp-edge-router-advertisement %s", key) return nil } -func (c *Controller) handleUpdateBgpEdgeRouterAdvertisement(key string) error { +func (c *Controller) handleUpdateBgpEdgeRouterAdvertisement(updatedObj *updateVerObject) error { + key := updatedObj.key + ns, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) @@ -118,53 +161,15 @@ func (c *Controller) handleUpdateBgpEdgeRouterAdvertisement(key string) error { } if !cachedAdvertisement.DeletionTimestamp.IsZero() { - c.delBgpEdgeRouterAdvertisementQueue.Add(key) + c.deleteBgpEdgeRouterAdvertisementQueue.Add(key) return nil } klog.Infof("reconciling bgp-edge-router-advertisement %s", key) advertisement := cachedAdvertisement.DeepCopy() - if controllerutil.AddFinalizer(advertisement, util.KubeOVNControllerFinalizer) { - updatedAdvertisement, err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouterAdvertisements(advertisement.Namespace). - Update(context.Background(), advertisement, metav1.UpdateOptions{}) - if err != nil { - err = fmt.Errorf("failed to add finalizer for bgp-edge-router %s/%s: %w", advertisement.Namespace, advertisement.Name, err) - klog.Error(err) - return err - } - advertisement = updatedAdvertisement - } - - // reconcile the bgp edge router workload and get the route sources for later OVN resources reconciliation - deploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(advertisement.Spec.BgpEdgeRouter) - if err != nil { - klog.Error(err) - return err - } - - ready := util.DeploymentIsReady(deploy) - if !ready { - advertisement.Status.Ready = false - msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) - // advertisement.Status.Conditions.SetCondition(kubeovnv1.Ready, corev1.ConditionFalse, "Processing", msg, advertisement.Generation) - advertisement.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "BgpEdgeRouterNotEnabled", msg, advertisement.Generation) - _, _ = c.updatebgpEdgeRouterAdvertisementStatus(advertisement) - readyErr := fmt.Sprintf("Kind %s, Deployment %s is not ready", deploy.Kind, deploy.Name) - klog.Error(readyErr) - return fmt.Errorf("%s", readyErr) - } - // get the pods of the deployment to collect the pod IPs - podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) - if err != nil { - err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) - klog.Error(err) - return err - } - - pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) - if err != nil { - err = fmt.Errorf("failed to list pods of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + pods, err := c.validateBgpEdgeRouterAdvertisement(advertisement) + if err != nil || pods == nil { klog.Error(err) return err } @@ -173,8 +178,17 @@ func (c *Controller) handleUpdateBgpEdgeRouterAdvertisement(key string) error { if len(pod.Status.PodIPs) == 0 { continue } + klog.Infof("handle adding bgp-edge-router-advertisement %s", key) + if err = c.updateBgpEdgeRouterAdvertisementRule(key, pod, updatedObj.oldVer, updatedObj.newVer); err != nil { + klog.Error(err) + return err + } } + advertisement.Status.Conditions.SetReady("ReconcileSuccess", advertisement.Generation) + if _, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement); err != nil { + return err + } klog.Infof("finished reconciling bgp-edge-router-advertisement %s", key) return nil @@ -199,27 +213,23 @@ func (c *Controller) handleDelBgpEdgeRouterAdvertisement(key string) error { return nil } - if !cachedAdvertisement.DeletionTimestamp.IsZero() { - c.delBgpEdgeRouterAdvertisementQueue.Add(key) - return nil - } - klog.Infof("reconciling bgp-edge-router-advertisement %s", key) advertisement := cachedAdvertisement.DeepCopy() - if pods, err := c.validateBgpEdgeRouterAdvertisement(advertisement); err != nil || pods == nil { + pods, err := c.validateBgpEdgeRouterAdvertisement(advertisement) + if err != nil || pods == nil { klog.Error(err) return err - } else { - for _, pod := range pods { - if len(pod.Status.PodIPs) == 0 { - continue - } - klog.Infof("handle deleting bgp-edge-router-advertisement %s", key) - if err = c.cleanBgpEdgeRouterAdvertisementRule(key, pod.Name, advertisement.Spec.Subnet); err != nil { - klog.Error(err) - return err - } + } + + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + klog.Infof("handle deleting bgp-edge-router-advertisement %s", key) + if err = c.addOrDeleteBgpEdgeRouterAdvertisementRule("del", key, pod, advertisement.Spec.Subnet); err != nil { + klog.Error(err) + return err } } @@ -231,59 +241,96 @@ func (c *Controller) handleDelBgpEdgeRouterAdvertisement(key string) error { klog.Error(err) } } + + advertisement.Status.Conditions.SetReady("ReconcileSuccess", advertisement.Generation) + if _, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement); err != nil { + return err + } + klog.Infof("finished reconciling bgp-edge-router-advertisement %s", key) + return nil } -func (c *Controller) cleanBgpEdgeRouterAdvertisementRule(key, podName string, subnetNames []string) error { - - if podName == "" { - err := fmt.Errorf("failed to get pod name %s", podName) +func (c *Controller) updateBgpEdgeRouterAdvertisementRule(key string, pod *corev1.Pod, oldBerAd, newBerAd *kubeovnv1.BgpEdgeRouterAdvertisement) error { + if pod.Name == "" { + err := fmt.Errorf("failed to get pod name %s", pod.Name) klog.Error(err) return err } - for _, subnetName := range subnetNames { - if subnet, err := c.subnetsLister.Get(subnetName); err != nil { + var oldSubnetArray []string + var newSubnetArray []string + + for _, subnetName := range oldBerAd.Spec.Subnet { + var subnet *kubeovnv1.Subnet + var err error + if subnet, err = c.subnetsLister.Get(subnetName); err != nil { + err = fmt.Errorf("failed to get subnet %s: %w", subnetName, err) + klog.Error(err) + return err + } + if subnet.Spec.CIDRBlock != "" { + oldSubnetArray = append(oldSubnetArray, subnet.Spec.CIDRBlock) + } + klog.Infof("cleaning bgp-edge-router-advertisement %s for subnet %s", key, subnet.Name) + } + + for _, subnetName := range newBerAd.Spec.Subnet { + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { err = fmt.Errorf("failed to get subnet %s: %w", subnetName, err) klog.Error(err) return err - } else { - if subnet.Spec.CIDRBlock != "" { - //delete bgp advertised ipblock - } - klog.Infof("cleaning bgp-edge-router-advertisement %s for subnet %s", key, subnet.Name) } + if subnet.Spec.CIDRBlock != "" { + newSubnetArray = append(newSubnetArray, subnet.Spec.CIDRBlock) + } + klog.Infof("cleaning bgp-edge-router-advertisement %s for subnet %s", key, subnet.Name) + } + if err := c.execUpdateBgpRoute(pod, oldSubnetArray, newSubnetArray); err != nil { + klog.Error(err) + return err } return nil } -func (c *Controller) addBgpEdgeRouterAdvertisementRule(key, podName string, subnetNames []string) error { - - if podName == "" { - err := fmt.Errorf("failed to get pod name %s", podName) +func (c *Controller) addOrDeleteBgpEdgeRouterAdvertisementRule(op, key string, pod *corev1.Pod, subnetNames []string) error { + if pod.Name == "" { + err := fmt.Errorf("failed to get pod name %s", pod.Name) klog.Error(err) return err } + SubnetCidrArray := []string{} for _, subnetName := range subnetNames { - if subnet, err := c.subnetsLister.Get(subnetName); err != nil { + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { err = fmt.Errorf("failed to get subnet %s: %w", subnetName, err) klog.Error(err) return err - } else { - if subnet.Spec.CIDRBlock != "" { - //delete bgp advertised ipblock - } - klog.Infof("cleaning bgp-edge-router-advertisement %s for subnet %s", key, subnet.Name) } + if subnet.Spec.CIDRBlock != "" { + SubnetCidrArray = append(SubnetCidrArray, subnet.Spec.CIDRBlock) + } + klog.Infof("cleaning bgp-edge-router-advertisement %s for subnet %s", key, subnet.Name) + } + if op == "add" { + if err := c.execUpdateBgpRoute(pod, nil, SubnetCidrArray); err != nil { + klog.Error(err) + return err + } + } else { + if err := c.execUpdateBgpRoute(pod, SubnetCidrArray, nil); err != nil { + klog.Error(err) + return err + } } return nil } func (c *Controller) validateBgpEdgeRouterAdvertisement(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement) ([]*corev1.Pod, error) { - deploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(advertisement.Spec.BgpEdgeRouter) if err != nil { advertisement.Status.Ready = false @@ -321,10 +368,6 @@ func (c *Controller) validateBgpEdgeRouterAdvertisement(advertisement *kubeovnv1 if ready { advertisement.Status.Ready = true - advertisement.Status.Conditions.SetReady("ReconcileSuccess", advertisement.Generation) - if _, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement); err != nil { - return pods, err - } } return pods, nil @@ -351,3 +394,42 @@ func (c *Controller) updatebgpEdgeRouterAdvertisementStatus(advertisement *kubeo return updateAdvertisement, nil } + +func (c *Controller) execUpdateBgpRoute(pod *corev1.Pod, oldCidrs, newCidrs []string) error { + // add_announced_route + cmdArs := []string{} + if len(oldCidrs) > 0 { + cmdArs = append(cmdArs, "del_announced_route="+strings.Join(oldCidrs, ",")) + } + if len(newCidrs) > 0 { + cmdArs = append(cmdArs, "add_announced_route="+strings.Join(newCidrs, ",")) + } + cmdArs = append(cmdArs, "list_announced_route") + cmd := fmt.Sprintf("bash /kube-ovn/update-bgp-route.sh %s", strings.Join(cmdArs, " ")) + + klog.Infof("exec command : %s", cmd) + stdOutput, errOutput, err := util.ExecuteCommandInContainer(c.config.KubeClient, c.config.KubeRestConfig, pod.Namespace, pod.Name, "bgp-router-speaker", []string{"/bin/bash", "-c", cmd}...) + if err != nil { + if len(errOutput) > 0 { + klog.Errorf("failed to ExecuteCommandInContainer, errOutput: %v", errOutput) + } + if len(stdOutput) > 0 { + klog.Infof("failed to ExecuteCommandInContainer, stdOutput: %v", stdOutput) + } + klog.Error(err) + return err + } + + if len(stdOutput) > 0 { + klog.Infof("ExecuteCommandInContainer stdOutput: %v", stdOutput) + } + + if len(errOutput) > 0 { + klog.Errorf("failed to ExecuteCommandInContainer errOutput: %v", errOutput) + return errors.New(errOutput) + } + + // list the current rule and check if the routes are updated + + return nil +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 93bb17c1328..a70032fae16 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -116,8 +116,8 @@ type Controller struct { bgpEdgeRouterAdvertisementLister kubeovnlister.BgpEdgeRouterAdvertisementLister bgpEdgeRouterAdvertisementSynced cache.InformerSynced addBgpEdgeRouterAdvertisementQueue workqueue.TypedRateLimitingInterface[string] - updateBgpEdgeRouterAdvertisementQueue workqueue.TypedRateLimitingInterface[string] - delBgpEdgeRouterAdvertisementQueue workqueue.TypedRateLimitingInterface[string] + updateBgpEdgeRouterAdvertisementQueue workqueue.TypedRateLimitingInterface[*updateVerObject] + deleteBgpEdgeRouterAdvertisementQueue workqueue.TypedRateLimitingInterface[string] bgpEdgeRouterAdvertisementKeyMutex keymutex.KeyMutex switchLBRuleLister kubeovnlister.SwitchLBRuleLister @@ -454,8 +454,8 @@ func Run(ctx context.Context, config *Configuration) { bgpEdgeRouterAdvertisementLister: bgpEdgeRouterAdvertisementInformer.Lister(), bgpEdgeRouterAdvertisementSynced: bgpEdgeRouterAdvertisementInformer.Informer().HasSynced, addBgpEdgeRouterAdvertisementQueue: newTypedRateLimitingQueue("AddBgpEdgeRouterAdvertisement", custCrdRateLimiter), - updateBgpEdgeRouterAdvertisementQueue: newTypedRateLimitingQueue("UpdateBgpEdgeRouterAdvertisement", custCrdRateLimiter), - delBgpEdgeRouterAdvertisementQueue: newTypedRateLimitingQueue("DeleteBgpEdgeRouterAdvertisement", custCrdRateLimiter), + updateBgpEdgeRouterAdvertisementQueue: newTypedRateLimitingQueue[*updateVerObject]("UpdateBgpEdgeRouterAdvertisement", nil), + deleteBgpEdgeRouterAdvertisementQueue: newTypedRateLimitingQueue("DeleteBgpEdgeRouterAdvertisement", custCrdRateLimiter), bgpEdgeRouterAdvertisementKeyMutex: keymutex.NewHashed(numKeyLocks), subnetsLister: subnetInformer.Lister(), @@ -1168,6 +1168,10 @@ func (c *Controller) shutdown() { c.addOrUpdateBgpEdgeRouterQueue.ShutDown() c.delBgpEdgeRouterQueue.ShutDown() + c.addBgpEdgeRouterAdvertisementQueue.ShutDown() + c.updateBgpEdgeRouterAdvertisementQueue.ShutDown() + c.deleteBgpEdgeRouterAdvertisementQueue.ShutDown() + if c.config.EnableLb { c.addSwitchLBRuleQueue.ShutDown() c.delSwitchLBRuleQueue.ShutDown() @@ -1265,7 +1269,7 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(runWorker("delete bgp edge router", c.delBgpEdgeRouterQueue, c.handleDelBgpEdgeRouter), time.Second, ctx.Done()) go wait.Until(runWorker("add bgp edge router advertisement", c.addBgpEdgeRouterAdvertisementQueue, c.handleAddBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) go wait.Until(runWorker("update bgp edge router advertisement", c.updateBgpEdgeRouterAdvertisementQueue, c.handleUpdateBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) - go wait.Until(runWorker("delete bgp edge router advertisement", c.delBgpEdgeRouterAdvertisementQueue, c.handleDelBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) + go wait.Until(runWorker("delete bgp edge router advertisement", c.deleteBgpEdgeRouterAdvertisementQueue, c.handleDelBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) go wait.Until(runWorker("update fip for vpc nat gateway", c.updateVpcFloatingIPQueue, c.handleUpdateVpcFloatingIP), time.Second, ctx.Done()) go wait.Until(runWorker("update eip for vpc nat gateway", c.updateVpcEipQueue, c.handleUpdateVpcEip), time.Second, ctx.Done()) go wait.Until(runWorker("update dnat for vpc nat gateway", c.updateVpcDnatQueue, c.handleUpdateVpcDnat), time.Second, ctx.Done()) diff --git a/pkg/util/const.go b/pkg/util/const.go index c7772f07db7..21bd6fd877f 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -115,7 +115,7 @@ const ( VpcEgressGatewayLabel = "ovn.kubernetes.io/vpc-egress-gateway" BgpEdgeRouterLabel = "ovn.kubernetes.io/bgp-edge-router" - BgpEdgeRouterAdvertisementLabel = "ovn.kubernetes.io/bgp-edge-router-advertisement" + BgpEdgeRouterAdvertisementLabel = "ovn.kubernetes.io/bgp-edge-router-advertisement" GenerateHashAnnotation = "ovn.kubernetes.io/generate-hash" VpcLastName = "ovn.kubernetes.io/last_vpc_name" From 5240e6017ccc4a1fa80e675862bbdb1c7f38767d Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Thu, 31 Jul 2025 22:31:02 -0700 Subject: [PATCH 19/66] bgp edge router advertisement resync logic added. --- dist/images/update-bgp-route.sh | 105 +++++++--- .../bgp_edge_router_advertisement.go | 193 ++++++++++++++++++ pkg/controller/controller.go | 1 + 3 files changed, 272 insertions(+), 27 deletions(-) diff --git a/dist/images/update-bgp-route.sh b/dist/images/update-bgp-route.sh index bc322eb5f09..3fb58f21b69 100644 --- a/dist/images/update-bgp-route.sh +++ b/dist/images/update-bgp-route.sh @@ -2,33 +2,26 @@ # update-bgp-route.sh # shellcheck disable=SC2086,SC2155 - set -euo pipefail - GOBGP_BIN=${GOBGP_BIN:-$(command -v gobgp || true)} [[ -z "$GOBGP_BIN" ]] && { echo "gobgp binary not found" >&2; exit 1; } - die() { echo "ERROR: $*" >&2; exit 1; } - external_iface="net1" external_ipv4=$(ip addr show dev "${external_iface}" | grep 'inet ' | awk '{print $2}' | cut -d'/' -f1) [[ -z "$external_ipv4" ]] && die "cannot determine external IPv4 address" - exec_cmd() { "$@" || die "failed: $*" } - check_inited() { $GOBGP_BIN global rib &>/dev/null \ || die "gobgp global RIB not initialized (did you 'gobgp global'?)" } - add_announced_route() { check_inited echo "Adding routes..." @@ -45,7 +38,6 @@ add_announced_route() { echo "" } - del_announced_route() { check_inited echo "Deleting routes..." @@ -62,6 +54,53 @@ del_announced_route() { echo "" } +flush_announced_route() { + check_inited + echo "Flushing all routes with next-hop $external_ipv4..." + + local routes_to_delete=() + local found_routes=false + + # Get IPv4 routes with matching next-hop + local ipv4_routes + if ipv4_routes=$($GOBGP_BIN global rib -a ipv4 2>/dev/null | grep "$external_ipv4" | awk '{print $2}' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[0-9]+$'); then + while IFS= read -r route; do + if [[ -n "$route" ]]; then + routes_to_delete+=("ipv4:$route") + found_routes=true + fi + done <<< "$ipv4_routes" + fi + + # Get IPv6 routes with matching next-hop + local ipv6_routes + if ipv6_routes=$($GOBGP_BIN global rib -a ipv6 2>/dev/null | grep "$external_ipv4" | awk '{print $2}' | grep -E '^[0-9a-fA-F:]+/[0-9]+$'); then + while IFS= read -r route; do + if [[ -n "$route" ]]; then + routes_to_delete+=("ipv6:$route") + found_routes=true + fi + done <<< "$ipv6_routes" + fi + + if [[ "$found_routes" == false ]]; then + echo " No routes found with next-hop $external_ipv4" + echo "" + return 0 + fi + + # Delete all found routes + for route_entry in "${routes_to_delete[@]}"; do + local family="${route_entry%%:*}" + local cidr="${route_entry#*:}" + echo " - Flushing: $cidr ($family)" + exec_cmd $GOBGP_BIN global rib -a "$family" del \ + "$cidr" nexthop "$external_ipv4" origin igp + done + + echo "Flushed ${#routes_to_delete[@]} routes with next-hop $external_ipv4" + echo "" +} list_announced_route() { check_inited @@ -118,7 +157,6 @@ list_announced_route() { echo "" } - parse_sequential_args() { local operations=() local operation_args=() @@ -134,6 +172,10 @@ parse_sequential_args() { operations+=("del") operation_args+=("${arg#*=}") ;; + flush_announced_route) + operations+=("flush") + operation_args+=("") + ;; list_announced_route) operations+=("list") operation_args+=("") @@ -151,6 +193,9 @@ parse_sequential_args() { list) list_announced_route ;; + flush) + flush_announced_route + ;; del) # Parse comma-separated CIDRs IFS=',' read -ra del_cidrs <<< "${operation_args[$i]}" @@ -173,11 +218,11 @@ parse_sequential_args() { done } - parse_key_value_args() { local add_routes="" local del_routes="" local list_routes=false + local flush_routes=false # find key=value for arg in "$@"; do @@ -188,6 +233,9 @@ parse_key_value_args() { del_announced_route=*|del_announce_routes=*) del_routes="${arg#*=}" ;; + flush_announced_route) + flush_routes=true + ;; list_announced_route) list_routes=true ;; @@ -198,6 +246,10 @@ parse_key_value_args() { esac done + if [[ "$flush_routes" == true ]]; then + flush_announced_route + fi + if [[ "$list_routes" == true ]]; then list_announced_route return 0 @@ -226,35 +278,36 @@ parse_key_value_args() { fi } - usage() { cat >&2 < [CIDR ...] - 2. Key-Value Arguments: $0 add_announced_route=CIDR1,CIDR2 [del_announced_route=CIDR3,CIDR4] [list_announced_route] - 3. Sequential Processing: $0 list_announced_route del_announce_routes=CIDR1,CIDR2 add_announce_routes=CIDR3,CIDR4 list_announced_route - + 1. Traditional: $0 [CIDR ...] + 2. Key-Value Arguments: $0 add_announced_route=CIDR1,CIDR2 [del_announced_route=CIDR3,CIDR4] [flush_announced_route] [list_announced_route] + 3. Sequential Processing: $0 list_announced_route del_announce_routes=CIDR1,CIDR2 add_announce_routes=CIDR3,CIDR4 flush_announced_route list_announced_route Examples: $0 add_announced_route 10.100.0.0/24 192.168.1.0/24 $0 del_announced_route 10.100.0.0/24 192.168.1.0/24 + $0 flush_announced_route $0 list_announced_route $0 add_announced_route=10.100.0.0/24,10.100.1.0/24 del_announced_route=10.0.0.0/24,10.0.1.0/24 - $0 list_announced_route del_announce_routes=10.100.0.0/24,10.100.1.0/24 add_announce_routes=10.0.0.0/24,10.0.1.0/24 list_announced_route + $0 flush_announced_route list_announced_route + $0 list_announced_route flush_announced_route add_announce_routes=10.0.0.0/24,10.0.1.0/24 list_announced_route -Note: Both 'del_announced_route=' and 'del_announce_routes=' are supported (same for add operations) +Note: + - flush_announced_route removes ALL routes with next-hop $external_ipv4 + - Both 'del_announced_route=' and 'del_announce_routes=' are supported (same for add operations) EOF exit 1 } - has_sequential_processing() { local has_list=false local has_operations=false for arg in "$@"; do case "$arg" in - list_announced_route) + list_announced_route|flush_announced_route) has_list=true ;; add_announced_route=*|add_announce_routes=*|del_announced_route=*|del_announce_routes=*) @@ -263,29 +316,27 @@ has_sequential_processing() { esac done - # Return true if we have list + operations (sequential processing) + # Return true if we have list/flush + operations (sequential processing) [[ "$has_list" == true && "$has_operations" == true ]] } - has_key_value_args() { for arg in "$@"; do case "$arg" in - *=*|list_announced_route) - return 0 # key=value or list command + *=*|list_announced_route|flush_announced_route) + return 0 # key=value or list/flush command ;; esac done return 1 # key=value not found } - # main entry point main() { # if no arguments are provided, show usage [[ $# -eq 0 ]] && usage - # check if we need sequential processing (list + operations) + # check if we need sequential processing (list/flush + operations) if has_sequential_processing "$@"; then parse_sequential_args "$@" return 0 @@ -298,15 +349,15 @@ main() { fi [[ $# -lt 1 ]] && usage - + # TODO list announced routes network which nexthop is external_ipv4 local op=$1; shift case "$op" in add_announced_route) add_announced_route "$@" ;; del_announced_route) del_announced_route "$@" ;; + flush_announced_route) flush_announced_route ;; list_announced_route) list_announced_route ;; *) usage ;; esac } - main "$@" \ No newline at end of file diff --git a/pkg/controller/bgp_edge_router_advertisement.go b/pkg/controller/bgp_edge_router_advertisement.go index 23453bb1d35..de43afeb6b9 100644 --- a/pkg/controller/bgp_edge_router_advertisement.go +++ b/pkg/controller/bgp_edge_router_advertisement.go @@ -4,11 +4,16 @@ import ( "context" "errors" "fmt" + "regexp" + "runtime/debug" + "slices" + "sort" "strings" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" @@ -433,3 +438,191 @@ func (c *Controller) execUpdateBgpRoute(pod *corev1.Pod, oldCidrs, newCidrs []st return nil } + +func (c *Controller) resyncBgpEdgeRouterAdvertisement() { + defer func() { + if r := recover(); r != nil { + klog.Errorf("panic in resyncBgpEdgeRouterAdvertisement: %v\n%s", r, debug.Stack()) + } + }() + klog.Info("resync bgp edge router") + // resync all vpc edge routers + berAds, err := c.bgpEdgeRouterAdvertisementLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list vpc edge routers: %v", err) + return + } + + for _, berAd := range berAds { + // Check router.Spec.BGP.AdvertisedRoutes same with pods bgp advertised routes + if err := c.syncAdvertisedRoutes(berAd); err != nil { + klog.Errorf("failed to sync advertised routes for vpc edge router %s: %v", berAd.Name, err) + continue + } + klog.Infof("resync vpc edge router %s", berAd.Name) + } +} + +func (c *Controller) syncAdvertisedRoutes(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement) error { + key := cache.MetaObjectToName(advertisement).String() + + c.bgpEdgeRouterAdvertisementKeyMutex.LockKey(key) + defer func() { _ = c.bgpEdgeRouterAdvertisementKeyMutex.UnlockKey(key) }() + + if !advertisement.DeletionTimestamp.IsZero() { + c.deleteBgpEdgeRouterAdvertisementQueue.Add(key) + return nil + } + klog.Infof("reconciling bgp-edge-router %s", key) + // Deep copy because we might mutate Status below. + cachedAdvertisement := advertisement.DeepCopy() + + pods, err := c.validateBgpEdgeRouterAdvertisement(cachedAdvertisement) + if err != nil || pods == nil { + klog.Error(err) + return err + } + cidrBlock, err := c.getSubnetCidrBlock(cachedAdvertisement) + if err != nil { + klog.Error(err) + return err + } + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + podCidr, err := c.execGetBgpRoute(pod) + if err != nil { + return err + } + klog.Infof("current router advertised routes: %v", cidrBlock) + klog.Infof("router pod %s/%s advertised routes: %v", pod.Namespace, pod.Name, podCidr) + routesDiff := !slicesEqual(podCidr, cidrBlock) + if routesDiff { + if err := c.execUpdateBgpRoute(pod, podCidr, cidrBlock); err != nil { + return err + } + klog.Infof("synced advertised routes for bgp-router-speaker %s pod %s/%s", key, pod.Namespace, pod.Name) + } + } + + klog.Infof("finished sync bgp-edge-router %s advertised routes", key) + return nil +} + +func (c *Controller) execGetBgpRoute(routerPod *corev1.Pod) ([]string, error) { + cmd := "bash /kube-ovn/update-bgp-route.sh list_announced_route" + klog.Infof("exec command : %s", cmd) + stdOutput, errOutput, err := util.ExecuteCommandInContainer(c.config.KubeClient, c.config.KubeRestConfig, routerPod.Namespace, routerPod.Name, "bgp-router-speaker", []string{"/bin/bash", "-c", cmd}...) + if err != nil { + if len(errOutput) > 0 { + klog.Errorf("failed to ExecuteCommandInContainer, errOutput: %v", errOutput) + } + klog.Error(err) + return nil, err + } + + if len(stdOutput) > 0 { + klog.Infof("ExecuteCommandInContainer stdOutput: %v", stdOutput) + } + if len(errOutput) > 0 { + klog.Errorf("failed to ExecuteCommandInContainer errOutput: %v", errOutput) + return nil, errors.New(errOutput) + } + + // Parse the output to extract announced routes + announcedRoutes, err := c.parseBgpAnnouncedRoutes(stdOutput) + if err != nil { + klog.Errorf("failed to parse BGP announced routes: %v", err) + return nil, err + } + + return announcedRoutes, nil +} + +func (c *Controller) getSubnetCidrBlock(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement) ([]string, error) { + var cirdBlock []string + for _, subnetName := range advertisement.Spec.Subnet { + var subnet *kubeovnv1.Subnet + var err error + subnet, err = c.subnetsLister.Get(subnetName) + if err != nil { + err = fmt.Errorf("failed to get subnet %s: %w", subnetName, err) + klog.Error(err) + return nil, err + } + if subnet.Spec.CIDRBlock != "" { + cirdBlock = append(cirdBlock, subnet.Spec.CIDRBlock) + } + } + return cirdBlock, nil +} + +func slicesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + + // Create copies and sort them + aCopy := make([]string, len(a)) + bCopy := make([]string, len(b)) + copy(aCopy, a) + copy(bCopy, b) + + sort.Strings(aCopy) + sort.Strings(bCopy) + + return slices.Equal(aCopy, bCopy) +} + +func (c *Controller) parseBgpAnnouncedRoutes(output string) ([]string, error) { + var routes []string + + // Look for the specific section with next-hop routes + lines := strings.Split(output, "\n") + inTargetSection := false + foundRoutesSection := false + + // Regex to match route lines starting with "*>" followed by CIDR + routeRegex := regexp.MustCompile(`^\*>\s+(\d+\.\d+\.\d+\.\d+/\d+)`) + + for _, line := range lines { + line = strings.TrimSpace(line) + + // Start parsing when we find the target section with any IP address + if strings.Contains(line, "--- Routes with Next-Hop") && strings.Contains(line, "---") { + inTargetSection = true + continue + } + + // Look for the IPv4 routes subsection + if inTargetSection && strings.Contains(line, "IPv4 routes with next-hop") { + foundRoutesSection = true + continue + } + + // Stop parsing if we hit another section starting with "---" or "===" + if inTargetSection && foundRoutesSection && (strings.HasPrefix(line, "---") || strings.HasPrefix(line, "===")) { + break + } + + // Skip header lines (Network, Next Hop, AS_PATH, etc.) + if inTargetSection && (strings.Contains(line, "Network") && strings.Contains(line, "Next Hop")) { + continue + } + + // Parse route lines in the target section that start with "*>" + if inTargetSection && foundRoutesSection && routeRegex.MatchString(line) { + matches := routeRegex.FindStringSubmatch(line) + if len(matches) > 1 { + routes = append(routes, matches[1]) + } + } + } + + if len(routes) == 0 { + return nil, errors.New("no announced routes found in BGP output") + } + + return routes, nil +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index a70032fae16..25f41f7abbf 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -1270,6 +1270,7 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(runWorker("add bgp edge router advertisement", c.addBgpEdgeRouterAdvertisementQueue, c.handleAddBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) go wait.Until(runWorker("update bgp edge router advertisement", c.updateBgpEdgeRouterAdvertisementQueue, c.handleUpdateBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) go wait.Until(runWorker("delete bgp edge router advertisement", c.deleteBgpEdgeRouterAdvertisementQueue, c.handleDelBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) + go wait.Until(c.resyncBgpEdgeRouterAdvertisement, 60*time.Second, ctx.Done()) go wait.Until(runWorker("update fip for vpc nat gateway", c.updateVpcFloatingIPQueue, c.handleUpdateVpcFloatingIP), time.Second, ctx.Done()) go wait.Until(runWorker("update eip for vpc nat gateway", c.updateVpcEipQueue, c.handleUpdateVpcEip), time.Second, ctx.Done()) go wait.Until(runWorker("update dnat for vpc nat gateway", c.updateVpcDnatQueue, c.handleUpdateVpcDnat), time.Second, ctx.Done()) From b2d908ffa35e44c90ff57c36dbcaf36fe8624ed7 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Thu, 31 Jul 2025 22:39:59 -0700 Subject: [PATCH 20/66] ber-ad resync recover logic delete --- pkg/controller/bgp_edge_router_advertisement.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/controller/bgp_edge_router_advertisement.go b/pkg/controller/bgp_edge_router_advertisement.go index de43afeb6b9..4720fba60ae 100644 --- a/pkg/controller/bgp_edge_router_advertisement.go +++ b/pkg/controller/bgp_edge_router_advertisement.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "regexp" - "runtime/debug" "slices" "sort" "strings" @@ -440,11 +439,6 @@ func (c *Controller) execUpdateBgpRoute(pod *corev1.Pod, oldCidrs, newCidrs []st } func (c *Controller) resyncBgpEdgeRouterAdvertisement() { - defer func() { - if r := recover(); r != nil { - klog.Errorf("panic in resyncBgpEdgeRouterAdvertisement: %v\n%s", r, debug.Stack()) - } - }() klog.Info("resync bgp edge router") // resync all vpc edge routers berAds, err := c.bgpEdgeRouterAdvertisementLister.List(labels.Everything()) From 386c544fa154118bf5fe76e492b3912e8c518570 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 4 Aug 2025 20:37:01 -0700 Subject: [PATCH 21/66] BgpEdgeRouterAdrvertisementExpansion typho error --- .../versioned/typed/kubeovn/v1/bgpedgerouteradvertisement.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouteradvertisement.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouteradvertisement.go index 40d7d2f16ef..c0ee6dcd3d9 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouteradvertisement.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouteradvertisement.go @@ -51,7 +51,7 @@ type BgpEdgeRouterAdvertisementInterface interface { GetScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, options metav1.GetOptions) (*autoscalingv1.Scale, error) UpdateScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (*autoscalingv1.Scale, error) - BgpEdgeRouterAdrvertisementExpansion + BgpEdgeRouterAdvertisementExpansion } // bgpEdgeRouterAdvertisements implements BgpEdgeRouterAdvertisementInterface From 86338edd0ba4dbf13097a2e1451586b222dc3d4f Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 4 Aug 2025 20:39:59 -0700 Subject: [PATCH 22/66] EdgeRouterMode, RouteServerClient mode added --- pkg/apis/kubeovn/v1/bgp-edge-router.go | 2 ++ pkg/speaker/config.go | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/apis/kubeovn/v1/bgp-edge-router.go b/pkg/apis/kubeovn/v1/bgp-edge-router.go index 34cfb99af3e..8c397c5f8c8 100644 --- a/pkg/apis/kubeovn/v1/bgp-edge-router.go +++ b/pkg/apis/kubeovn/v1/bgp-edge-router.go @@ -156,4 +156,6 @@ type BgpEdgeRouterBGPConfig struct { Password string `json:"password"` EnableGracefulRestart bool `json:"enableGracefulRestart"` ExtraArgs []string `json:"extraArgs"` + EdgeRouterMode bool `json:"edgeRouterMode"` + RouteServerClient bool `json:"routeServerClient"` } diff --git a/pkg/speaker/config.go b/pkg/speaker/config.go index ebf7c756636..07b63d99295 100644 --- a/pkg/speaker/config.go +++ b/pkg/speaker/config.go @@ -66,9 +66,10 @@ type Configuration struct { KubeClient kubernetes.Interface KubeOvnClient clientset.Interface - PprofPort int32 - LogPerm string - EdgeRouterMode bool + PprofPort int32 + LogPerm string + EdgeRouterMode bool + RouteServerClient bool } func ParseFlags() (*Configuration, error) { @@ -97,6 +98,7 @@ func ParseFlags() (*Configuration, error) { argEnableMetrics = pflag.BoolP("enable-metrics", "", true, "Whether to support metrics query") argLogPerm = pflag.String("log-perm", "640", "The permission for the log file") argEdgeRouterMode = pflag.BoolP("edge-router-mode", "", false, "Make the BGP speaker announce inside subnet and get routes from the outside, work as edge router") + argRouteServerClient = pflag.BoolP("route-server-client", "", false, "Make the BGP speaker policy route, work as route server client") ) klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) klog.InitFlags(klogFlags) @@ -165,6 +167,7 @@ func ParseFlags() (*Configuration, error) { EnableMetrics: *argEnableMetrics, LogPerm: *argLogPerm, EdgeRouterMode: *argEdgeRouterMode, + RouteServerClient: *argRouteServerClient, } if *argNeighborAddress != "" { From 219bf4f04b1f1a354db173ddc3958fc25242c9f8 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Thu, 7 Aug 2025 19:01:26 -0700 Subject: [PATCH 23/66] gobgp config crd and controller logic added --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 161 +++++++ pkg/apis/kubeovn/v1/gobgp-config.go | 70 +++ pkg/apis/kubeovn/v1/register.go | 2 + pkg/apis/kubeovn/v1/zz_generated.deepcopy.go | 113 +++++ .../typed/kubeovn/v1/fake/fake_gobgpconfig.go | 81 ++++ .../kubeovn/v1/fake/fake_kubeovn_client.go | 6 +- .../typed/kubeovn/v1/generated_expansion.go | 4 +- .../versioned/typed/kubeovn/v1/gobgpconfig.go | 103 +++++ .../typed/kubeovn/v1/kubeovn_client.go | 5 + .../kubeovn/v1/gobgpconfig.go | 89 ++++ .../externalversions/kubeovn/v1/interface.go | 9 +- .../listers/kubeovn/v1/bgpedgerouter.go | 1 + .../listers/kubeovn/v1/expansion_generated.go | 8 + pkg/client/listers/kubeovn/v1/gobgpconfig.go | 70 +++ pkg/controller/bgp_edge_router.go | 7 +- .../bgp_edge_router_advertisement.go | 38 +- pkg/controller/controller.go | 33 +- pkg/controller/gobgp_config.go | 428 ++++++++++++++++++ pkg/ovs/ovn.go | 1 + pkg/util/const.go | 1 + 20 files changed, 1206 insertions(+), 24 deletions(-) create mode 100644 pkg/apis/kubeovn/v1/gobgp-config.go create mode 100644 pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_gobgpconfig.go create mode 100644 pkg/client/clientset/versioned/typed/kubeovn/v1/gobgpconfig.go create mode 100644 pkg/client/informers/externalversions/kubeovn/v1/gobgpconfig.go create mode 100644 pkg/client/listers/kubeovn/v1/gobgpconfig.go create mode 100644 pkg/controller/gobgp_config.go diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index 75c5f7b74cd..b5bae517441 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3686,3 +3686,164 @@ spec: required: - conditions type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gobgp-configs.kubeovn.io +spec: + group: kubeovn.io + names: + plural: gobgp-configs + singular: gobgp-config + shortNames: + - bgp-config + kind: GobgpConfig + listKind: GobgpConfigList + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: bgp-edge-router + type: string + jsonPath: .spec.bgpEdgeRouter + - name: neighbor-address + type: string + jsonPath: .spec.neighbors[*].address + - name: toReceive-mode + type: string + jsonPath: .spec.neighbors[*].toReceive.allowed.mode + - name: toReceive-prefixes + type: string + jsonPath: .spec.neighbors[*].toReceive.allowed.prefixes + - name: toAdvertise-mode + type: string + jsonPath: .spec.neighbors[*].toAdvertise.allowed.mode + - name: toAdvertise-prefixes + type: string + jsonPath: .spec.neighbors[*].toAdvertise.allowed.prefixes + - name: READY + type: boolean + jsonPath: .status.ready + - name: AGE + type: date + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + required: + - bgpEdgeRouter + - neighbors + properties: + bgpEdgeRouter: + type: string + neighbors: + type: array + items: + type: object + required: + - address + - toAdvertise + - toReceive + properties: + address: + type: string + pattern: '^(([0-9]{1,3}\.){3}[0-9]{1,3}|[0-9a-fA-F:]+)$' + toAdvertise: + type: object + required: + - allowed + properties: + allowed: + type: object + required: + - mode + properties: + mode: + type: string + prefixes: + type: array + x-kubernetes-list-type: set + items: + type: string + pattern: '^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/\d{1,2})?|([0-9a-fA-F:]+(\/\d{1,3})?))$' + x-kubernetes-validations: + - rule: "self.mode != 'filtered' || (has(self.prefixes) && size(self.prefixes) > 0)" + message: "prefixes must be set and non-empty when mode is 'filtered'" + toReceive: + type: object + required: + - allowed + properties: + allowed: + type: object + required: + - mode + properties: + mode: + type: string + prefixes: + type: array + x-kubernetes-list-type: set + items: + type: string + pattern: '^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/\d{1,2})?|([0-9a-fA-F:]+(\/\d{1,3})?))$' + x-kubernetes-validations: + - rule: "self.mode != 'filtered' || (has(self.prefixes) && size(self.prefixes) > 0)" + message: "prefixes must be set and non-empty when mode is 'filtered'" + status: + type: object + properties: + ready: + type: boolean + default: false + conditions: + type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - type + items: + type: object + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + properties: + type: + type: string + maxLength: 316 + pattern: '^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$' + status: + type: string + enum: + - "True" + - "False" + - "Unknown" + reason: + type: string + minLength: 1 + maxLength: 1024 + pattern: '^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$' + message: + type: string + maxLength: 32768 + lastTransitionTime: + type: string + format: date-time + lastUpdateTime: + type: string + format: date-time + observedGeneration: + type: integer + format: int64 + minimum: 0 \ No newline at end of file diff --git a/pkg/apis/kubeovn/v1/gobgp-config.go b/pkg/apis/kubeovn/v1/gobgp-config.go new file mode 100644 index 00000000000..a42ba10f5d7 --- /dev/null +++ b/pkg/apis/kubeovn/v1/gobgp-config.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0 +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type GobgpConfigList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []GobgpConfig `json:"items"` +} + +// +genclient +// +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale +// +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +resourceName=gobgp-configs +// bgp edge router advertisement is used to forward the egress traffic from the VPC to the external network +type GobgpConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + + Spec GobgpConfigSpec `json:"spec"` + Status GobgpConfigStatus `json:"status"` +} + +// If the GobgpConfig has no VPC specified in the spec, it will return the default VPC name +func (g *GobgpConfig) Subnet(subnets []string) []string { + if len(subnets) != 0 { + return subnets + } + return nil +} + +// Ready returns true if the BgpEdgeRouter has been processed successfully and is ready to serve traffic +// func (g *BgpEdgeRouterAdvertisement) Ready() bool { +// return g.Status.Ready && g.Status.Conditions.IsReady(g.Generation) +// } + +type GobgpConfigSpec struct { + BgpEdgeRouterName string `json:"bgpEdgeRouterName"` + Neighbors []Neighbors `json:"neighbors,omitempty"` +} + +type GobgpConfigStatus struct { + Ready bool `json:"ready"` + Conditions Conditions `json:"conditions,omitempty"` +} + +type Neighbors struct { + Address string `json:"address"` + ToAdvertise ToAdvertise `json:"toAdvertise"` + ToReceive ToReceive `json:"toReceive"` +} + +type ToAdvertise struct { + Allowed Allowed `json:"allowed"` +} + +type ToReceive struct { + Allowed Allowed `json:"allowed"` +} + +type Allowed struct { + Mode string `json:"mode,omitempty"` + Prefixes []string `json:"prefixes,omitempty"` +} diff --git a/pkg/apis/kubeovn/v1/register.go b/pkg/apis/kubeovn/v1/register.go index 53758d2be91..d6cee08fd91 100644 --- a/pkg/apis/kubeovn/v1/register.go +++ b/pkg/apis/kubeovn/v1/register.go @@ -77,6 +77,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &BgpEdgeRouterList{}, &BgpEdgeRouterAdvertisement{}, &BgpEdgeRouterAdvertisementList{}, + &GobgpConfig{}, + &GobgpConfigList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go index 51b49ad5962..baa7bc636ee 100644 --- a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go +++ b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go @@ -3194,3 +3194,116 @@ func (in *BgpEdgeRouterAdvertisementSpec) DeepCopy() *BgpEdgeRouterAdvertisement in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GobgpConfig) DeepCopyInto(out *GobgpConfig) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGateway. +func (in *GobgpConfig) DeepCopy() *GobgpConfig { + if in == nil { + return nil + } + out := new(GobgpConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GobgpConfig) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GobgpConfigList) DeepCopyInto(out *GobgpConfigList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GobgpConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayList. +func (in *GobgpConfigList) DeepCopy() *GobgpConfigList { + if in == nil { + return nil + } + out := new(GobgpConfigList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GobgpConfigList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GobgpConfigSpec) DeepCopyInto(out *GobgpConfigSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcDNSSpec. +func (in *GobgpConfigSpec) DeepCopy() *GobgpConfigSpec { + if in == nil { + return nil + } + out := new(GobgpConfigSpec) + in.DeepCopyInto(out) + return out +} + +// // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +// func (in *ToAdvertise) DeepCopyInto(out *ToAdvertise) { +// *out = *in +// if in != nil { +// in, out := &in.Allowed, &out.Allowed +// out = new(Allowed) +// (in).DeepCopyInto(out) +// } + +// return +// } + +// func (in *ToReceive) DeepCopyInto(out *ToReceive) { +// *out = *in +// if in != nil { +// in, out := &in.Allowed, &out.Allowed +// out = new(Allowed) +// (in).DeepCopyInto(out) +// } + +// return +// } + +// func (in *Allowed) DeepCopyInto(out *Allowed) { +// *out = *in +// if in.Mode != "" { +// in, out := &in.Mode, &out.Mode +// *out = *in +// } +// if in.Prefixes != nil { +// in, out := &in.Prefixes, &out.Prefixes +// *out = make([]string, len(*in)) +// copy(*out, *in) +// } +// return +// } diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_gobgpconfig.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_gobgpconfig.go new file mode 100644 index 00000000000..e8c05c5449f --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_gobgpconfig.go @@ -0,0 +1,81 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + context "context" + + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gentype "k8s.io/client-go/gentype" + testing "k8s.io/client-go/testing" +) + +// fakeGobgpConfigs implements GobgpConfigInterface +type fakeGobgpConfigs struct { + *gentype.FakeClientWithList[*v1.GobgpConfig, *v1.GobgpConfigList] + Fake *FakeKubeovnV1 +} + +func newFakeGobgpConfigs(fake *FakeKubeovnV1, namespace string) kubeovnv1.GobgpConfigInterface { + return &fakeGobgpConfigs{ + gentype.NewFakeClientWithList[*v1.GobgpConfig, *v1.GobgpConfigList]( + fake.Fake, + namespace, + v1.SchemeGroupVersion.WithResource("gobgp-configs"), + v1.SchemeGroupVersion.WithKind("GobgpConfig"), + func() *v1.GobgpConfig { return &v1.GobgpConfig{} }, + func() *v1.GobgpConfigList { return &v1.GobgpConfigList{} }, + func(dst, src *v1.GobgpConfigList) { dst.ListMeta = src.ListMeta }, + func(list *v1.GobgpConfigList) []*v1.GobgpConfig { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1.GobgpConfigList, items []*v1.GobgpConfig) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} + +// GetScale takes name of the gobgpConfig, and returns the corresponding scale object, and an error if there is any. +func (c *fakeGobgpConfigs) GetScale(ctx context.Context, gobgpConfigName string, options metav1.GetOptions) (result *autoscalingv1.Scale, err error) { + emptyResult := &autoscalingv1.Scale{} + obj, err := c.Fake. + Invokes(testing.NewGetSubresourceActionWithOptions(c.Resource(), c.Namespace(), "scale", gobgpConfigName, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*autoscalingv1.Scale), err +} + +// UpdateScale takes the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *fakeGobgpConfigs) UpdateScale(ctx context.Context, gobgpConfigName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (result *autoscalingv1.Scale, err error) { + emptyResult := &autoscalingv1.Scale{} + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceActionWithOptions(c.Resource(), "scale", c.Namespace(), scale, opts), &autoscalingv1.Scale{}) + + if obj == nil { + return emptyResult, err + } + return obj.(*autoscalingv1.Scale), err +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go index e2e632cf062..dec025c5dd7 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go @@ -120,8 +120,12 @@ func (c *FakeKubeovnV1) BgpEdgeRouterAdvertisements(namespace string) v1.BgpEdge return newFakeBgpEdgeRouterAdvertisements(c, namespace) } +func (c *FakeKubeovnV1) GobgpConfigs(namespace string) v1.GobgpConfigInterface { + return newFakeGobgpConfigs(c, namespace) +} + // RESTClient returns a RESTClient that is used to communicate -// with API server by this client implementation. +// with API server by this client implementation.p func (c *FakeKubeovnV1) RESTClient() rest.Interface { var ret *rest.RESTClient return ret diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go index 9299dea7c90..f487e31b210 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go @@ -62,4 +62,6 @@ type VpcNatGatewayExpansion interface{} type BgpEdgeRouterExpansion interface{} -type BgpEdgeRouterAdrvertisementExpansion interface{} +type BgpEdgeRouterAdvertisementExpansion interface{} + +type GobgpConfigExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/gobgpconfig.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/gobgpconfig.go new file mode 100644 index 00000000000..3e31d04a744 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/gobgpconfig.go @@ -0,0 +1,103 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + scheme "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" + autoscalingv1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// GobgpConfigsGetter has a method to return a GobgpConfigInterface. +// A group's client should implement this interface. +type GobgpConfigsGetter interface { + GobgpConfigs(namespace string) GobgpConfigInterface +} + +// GobgpConfigInterface has methods to work with gobgpConfig resources. +type GobgpConfigInterface interface { + Create(ctx context.Context, gobgpConfig *kubeovnv1.GobgpConfig, opts metav1.CreateOptions) (*kubeovnv1.GobgpConfig, error) + Update(ctx context.Context, gobgpConfig *kubeovnv1.GobgpConfig, opts metav1.UpdateOptions) (*kubeovnv1.GobgpConfig, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, gobgpConfig *kubeovnv1.GobgpConfig, opts metav1.UpdateOptions) (*kubeovnv1.GobgpConfig, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*kubeovnv1.GobgpConfig, error) + List(ctx context.Context, opts metav1.ListOptions) (*kubeovnv1.GobgpConfigList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *kubeovnv1.GobgpConfig, err error) + GetScale(ctx context.Context, gobgpConfigName string, options metav1.GetOptions) (*autoscalingv1.Scale, error) + UpdateScale(ctx context.Context, gobgpConfigName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (*autoscalingv1.Scale, error) + + GobgpConfigExpansion +} + +// gobgpConfigs implements GobgpConfigInterface +type gobgpConfigs struct { + *gentype.ClientWithList[*kubeovnv1.GobgpConfig, *kubeovnv1.GobgpConfigList] +} + +// newGobgpConfigs returns a GobgpConfigs +func newGobgpConfigs(c *KubeovnV1Client, namespace string) *gobgpConfigs { + return &gobgpConfigs{ + gentype.NewClientWithList[*kubeovnv1.GobgpConfig, *kubeovnv1.GobgpConfigList]( + "gobgp-configs", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *kubeovnv1.GobgpConfig { return &kubeovnv1.GobgpConfig{} }, + func() *kubeovnv1.GobgpConfigList { return &kubeovnv1.GobgpConfigList{} }, + ), + } +} + +// GetScale takes name of the GobgpConfig, and returns the corresponding autoscalingv1.Scale object, and an error if there is any. +func (c *gobgpConfigs) GetScale(ctx context.Context, gobgpConfigName string, options metav1.GetOptions) (result *autoscalingv1.Scale, err error) { + result = &autoscalingv1.Scale{} + err = c.GetClient().Get(). + Namespace(c.GetNamespace()). + Resource("gobgp-configs"). + Name(gobgpConfigName). + SubResource("scale"). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// UpdateScale takes the top resource name and the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *gobgpConfigs) UpdateScale(ctx context.Context, gobgpConfigName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (result *autoscalingv1.Scale, err error) { + result = &autoscalingv1.Scale{} + err = c.GetClient().Put(). + Namespace(c.GetNamespace()). + Resource("gobgp-configs"). + Name(gobgpConfigName). + SubResource("scale"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(scale). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go index 540e754cfd7..de04c5416b9 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go @@ -51,6 +51,7 @@ type KubeovnV1Interface interface { VpcNatGatewaysGetter BgpEdgeRoutersGetter BgpEdgeRouterAdvertisementsGetter + GobgpConfigsGetter } // KubeovnV1Client is used to interact with features provided by the kubeovn.io group. @@ -150,6 +151,10 @@ func (c *KubeovnV1Client) BgpEdgeRouterAdvertisements(namespace string) BgpEdgeR return newBgpEdgeRouterAdvertisements(c, namespace) } +func (c *KubeovnV1Client) GobgpConfigs(namespace string) GobgpConfigInterface { + return newGobgpConfigs(c, namespace) +} + // NewForConfig creates a new KubeovnV1Client for the given config. // NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), // where httpClient was generated with rest.HTTPClientFor(c). diff --git a/pkg/client/informers/externalversions/kubeovn/v1/gobgpconfig.go b/pkg/client/informers/externalversions/kubeovn/v1/gobgpconfig.go new file mode 100644 index 00000000000..317ad6d1ffb --- /dev/null +++ b/pkg/client/informers/externalversions/kubeovn/v1/gobgpconfig.go @@ -0,0 +1,89 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + time "time" + + apiskubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + versioned "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions/internalinterfaces" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// GobgpConfigInformer provides access to a shared informer and lister for +type GobgpConfigInformer interface { + Informer() cache.SharedIndexInformer + Lister() kubeovnv1.GobgpConfigLister +} + +type gobgpConfigInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewBgpEdgeRouterAdvertisementInformer constructs a new informer for BgpEdgeRouter type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewGobgpConfigInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredGobgpConfigInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredGobgpConfigInformer constructs a new informer for GobgpConfig type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredGobgpConfigInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().GobgpConfigs(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().GobgpConfigs(namespace).Watch(context.TODO(), options) + }, + }, + &apiskubeovnv1.GobgpConfig{}, + resyncPeriod, + indexers, + ) +} + +func (f *gobgpConfigInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredGobgpConfigInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *gobgpConfigInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apiskubeovnv1.GobgpConfig{}, f.defaultInformer) +} + +func (f *gobgpConfigInformer) Lister() kubeovnv1.GobgpConfigLister { + return kubeovnv1.NewGobgpConfigLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/kubeovn/v1/interface.go b/pkg/client/informers/externalversions/kubeovn/v1/interface.go index 57dcc7394c9..deda5a9333c 100644 --- a/pkg/client/informers/externalversions/kubeovn/v1/interface.go +++ b/pkg/client/informers/externalversions/kubeovn/v1/interface.go @@ -68,8 +68,10 @@ type Interface interface { VpcNatGateways() VpcNatGatewayInformer // BgpEdgeRouters returns a BgpEdgeRouterInformer. BgpEdgeRouters() BgpEdgeRouterInformer - // BgpEdgeRouters returns a BgpEdgeRouterInformer. + // BgpEdgeRouterAdvertisements returns a BgpEdgeRouterAdvertisementInformer. BgpEdgeRouterAdvertisements() BgpEdgeRouterAdvertisementInformer + // GobgpConfigs returns a GobgpConfigInformer. + GobgpConfigs() GobgpConfigInformer } type version struct { @@ -197,3 +199,8 @@ func (v *version) BgpEdgeRouters() BgpEdgeRouterInformer { func (v *version) BgpEdgeRouterAdvertisements() BgpEdgeRouterAdvertisementInformer { return &bgpEdgeRouterAdvertisementInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } + +// BgpEdgeRouters returns a BgpEdgeRouterInformer. +func (v *version) GobgpConfigs() GobgpConfigInformer { + return &gobgpConfigInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/client/listers/kubeovn/v1/bgpedgerouter.go b/pkg/client/listers/kubeovn/v1/bgpedgerouter.go index c7c9bf2d6cd..70b4b99d211 100644 --- a/pkg/client/listers/kubeovn/v1/bgpedgerouter.go +++ b/pkg/client/listers/kubeovn/v1/bgpedgerouter.go @@ -42,6 +42,7 @@ type BgpEdgeRouterLister interface { // List lists all BgpEdgeRouters in the indexer. // Objects returned here must be treated as read-only. List(selector labels.Selector) (ret []*kubeovnv1.BgpEdgeRouter, err error) + // Get(name string) (*kubeovnv1.BgpEdgeRouter, error) // BgpEdgeRouters returns an object that can list and get BgpEdgeRouters. BgpEdgeRouters(namespace string) BgpEdgeRouterNamespaceLister BgpEdgeRouterListerExpansion diff --git a/pkg/client/listers/kubeovn/v1/expansion_generated.go b/pkg/client/listers/kubeovn/v1/expansion_generated.go index c3f95d5071d..6af94fabcb7 100644 --- a/pkg/client/listers/kubeovn/v1/expansion_generated.go +++ b/pkg/client/listers/kubeovn/v1/expansion_generated.go @@ -121,3 +121,11 @@ type BgpEdgeRouterAdvertisementListerExpansion interface{} // BgpEdgeRouterNamespaceListerExpansion allows custom methods to be added to // BgpEdgeRouterNamespaceLister. type BgpEdgeRouterAdvertisementNamespaceListerExpansion interface{} + +// BgpEdgeRouterListerExpansion allows custom methods to be added to +// BgpEdgeRouterLister. +type GobgpConfigListerExpansion interface{} + +// BgpEdgeRouterNamespaceListerExpansion allows custom methods to be added to +// BgpEdgeRouterNamespaceLister. +type GobgpConfigNamespaceListerExpansion interface{} diff --git a/pkg/client/listers/kubeovn/v1/gobgpconfig.go b/pkg/client/listers/kubeovn/v1/gobgpconfig.go new file mode 100644 index 00000000000..fbf3974c660 --- /dev/null +++ b/pkg/client/listers/kubeovn/v1/gobgpconfig.go @@ -0,0 +1,70 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// GobgpConfigLister helps list GobgpConfigs. +// All objects returned here must be treated as read-only. +type GobgpConfigLister interface { + // List lists all GobgpConfigs in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*kubeovnv1.GobgpConfig, err error) + // GobgpConfigs returns an object that can list and get GobgpConfigs. + GobgpConfigs(namespace string) GobgpConfigNamespaceLister + GobgpConfigListerExpansion +} + +// gobgpConfigLister implements the gobgpConfigLister interface. +type gobgpConfigLister struct { + listers.ResourceIndexer[*kubeovnv1.GobgpConfig] +} + +// NewGobgpConfigLister returns a new gobgpConfigLister. +func NewGobgpConfigLister(indexer cache.Indexer) GobgpConfigLister { + return &gobgpConfigLister{listers.New[*kubeovnv1.GobgpConfig](indexer, kubeovnv1.Resource("gobgpconfig"))} +} + +// gobgpConfigs returns an object that can list and get gobgpConfigs. +func (s *gobgpConfigLister) GobgpConfigs(namespace string) GobgpConfigNamespaceLister { + return gobgpConfigNamespaceLister{listers.NewNamespaced[*kubeovnv1.GobgpConfig](s.ResourceIndexer, namespace)} +} + +// gobgpConfigNamespaceLister helps list and get gobgpConfigs. +// All objects returned here must be treated as read-only. +type GobgpConfigNamespaceLister interface { + // List lists all gobgpConfigs in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*kubeovnv1.GobgpConfig, err error) + // Get retrieves the GobgpConfig from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*kubeovnv1.GobgpConfig, error) + GobgpConfigNamespaceListerExpansion +} + +// gobgpConfigNamespaceLister implements the gobgpConfigNamespaceLister +// interface. +type gobgpConfigNamespaceLister struct { + listers.ResourceIndexer[*kubeovnv1.GobgpConfig] +} diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go index 9008c3b72c8..947be6975be 100644 --- a/pkg/controller/bgp_edge_router.go +++ b/pkg/controller/bgp_edge_router.go @@ -1092,7 +1092,12 @@ func bgpEdgeRouterContainerBGP(speakerImage, routerName string, speakerParams *k if speakerParams.HoldTime != (metav1.Duration{}) { args = append(args, "--holdtime="+speakerParams.HoldTime.Duration.String()) } - + if speakerParams.EdgeRouterMode { + args = append(args, "--edge-router-mode=true") + } + if speakerParams.RouteServerClient { + args = append(args, "--route-server-client=true") + } args = append(args, fmt.Sprintf("--cluster-as=%d", speakerParams.ASN)) args = append(args, fmt.Sprintf("--neighbor-as=%d", speakerParams.RemoteASN)) diff --git a/pkg/controller/bgp_edge_router_advertisement.go b/pkg/controller/bgp_edge_router_advertisement.go index 4720fba60ae..8e6d15aa2a1 100644 --- a/pkg/controller/bgp_edge_router_advertisement.go +++ b/pkg/controller/bgp_edge_router_advertisement.go @@ -334,6 +334,25 @@ func (c *Controller) addOrDeleteBgpEdgeRouterAdvertisementRule(op, key string, p return nil } +func (c *Controller) resyncBgpRules() { + klog.Info("resync bgp edge router") + // resync all bgp edge routers + bgpEdgeRouterAds, err := c.bgpEdgeRouterAdvertisementLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list bgp edge routers: %v", err) + return + } + + for _, bgpEdgeRouterAd := range bgpEdgeRouterAds { + // Check router.Spec.BGP.AdvertisedRoutes same with pods bgp advertised routes + if err := c.syncAdvertisedRoutes(bgpEdgeRouterAd); err != nil { + klog.Errorf("failed to sync advertised routes for bgp edge router %s: %v", bgpEdgeRouterAd.Name, err) + continue + } + klog.Infof("resync bgp edge router %s", bgpEdgeRouterAd.Name) + } +} + func (c *Controller) validateBgpEdgeRouterAdvertisement(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement) ([]*corev1.Pod, error) { deploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(advertisement.Spec.BgpEdgeRouter) if err != nil { @@ -438,25 +457,6 @@ func (c *Controller) execUpdateBgpRoute(pod *corev1.Pod, oldCidrs, newCidrs []st return nil } -func (c *Controller) resyncBgpEdgeRouterAdvertisement() { - klog.Info("resync bgp edge router") - // resync all vpc edge routers - berAds, err := c.bgpEdgeRouterAdvertisementLister.List(labels.Everything()) - if err != nil { - klog.Errorf("failed to list vpc edge routers: %v", err) - return - } - - for _, berAd := range berAds { - // Check router.Spec.BGP.AdvertisedRoutes same with pods bgp advertised routes - if err := c.syncAdvertisedRoutes(berAd); err != nil { - klog.Errorf("failed to sync advertised routes for vpc edge router %s: %v", berAd.Name, err) - continue - } - klog.Infof("resync vpc edge router %s", berAd.Name) - } -} - func (c *Controller) syncAdvertisedRoutes(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement) error { key := cache.MetaObjectToName(advertisement).String() diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 25f41f7abbf..6d05280ea8b 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -120,6 +120,13 @@ type Controller struct { deleteBgpEdgeRouterAdvertisementQueue workqueue.TypedRateLimitingInterface[string] bgpEdgeRouterAdvertisementKeyMutex keymutex.KeyMutex + gobgpConfigLister kubeovnlister.GobgpConfigLister + gobgpConfigSynced cache.InformerSynced + addGobgpConfigQueue workqueue.TypedRateLimitingInterface[string] + updateGobgpConfigQueue workqueue.TypedRateLimitingInterface[*updateVerGobgpConfigObject] + deleteGobgpConfigQueue workqueue.TypedRateLimitingInterface[string] + gobgpConfigKeyMutex keymutex.KeyMutex + switchLBRuleLister kubeovnlister.SwitchLBRuleLister switchLBRuleSynced cache.InformerSynced addSwitchLBRuleQueue workqueue.TypedRateLimitingInterface[string] @@ -380,6 +387,7 @@ func Run(ctx context.Context, config *Configuration) { vpcEgressGatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcEgressGateways() bgpEdgeRouterInformer := kubeovnInformerFactory.Kubeovn().V1().BgpEdgeRouters() bgpEdgeRouterAdvertisementInformer := kubeovnInformerFactory.Kubeovn().V1().BgpEdgeRouterAdvertisements() + gobgpConfigInformer := kubeovnInformerFactory.Kubeovn().V1().GobgpConfigs() subnetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets() ippoolInformer := kubeovnInformerFactory.Kubeovn().V1().IPPools() ipInformer := kubeovnInformerFactory.Kubeovn().V1().IPs() @@ -458,6 +466,13 @@ func Run(ctx context.Context, config *Configuration) { deleteBgpEdgeRouterAdvertisementQueue: newTypedRateLimitingQueue("DeleteBgpEdgeRouterAdvertisement", custCrdRateLimiter), bgpEdgeRouterAdvertisementKeyMutex: keymutex.NewHashed(numKeyLocks), + gobgpConfigLister: gobgpConfigInformer.Lister(), + gobgpConfigSynced: gobgpConfigInformer.Informer().HasSynced, + addGobgpConfigQueue: newTypedRateLimitingQueue("AddGobgpConfig", custCrdRateLimiter), + updateGobgpConfigQueue: newTypedRateLimitingQueue[*updateVerGobgpConfigObject]("UpdateGobgpConfig", nil), + deleteGobgpConfigQueue: newTypedRateLimitingQueue("DeleteGobgpConfig", custCrdRateLimiter), + gobgpConfigKeyMutex: keymutex.NewHashed(numKeyLocks), + subnetsLister: subnetInformer.Lister(), subnetSynced: subnetInformer.Informer().HasSynced, addOrUpdateSubnetQueue: newTypedRateLimitingQueue[string]("AddSubnet", nil), @@ -816,6 +831,14 @@ func Run(ctx context.Context, config *Configuration) { util.LogFatalAndExit(err, "failed to add bgp edge router advertisement event handler") } + if _, err = gobgpConfigInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddGobgpConfig, + UpdateFunc: controller.enqueueUpdateGobgpConfig, + DeleteFunc: controller.enqueueDeleteGobgpConfig, + }); err != nil { + util.LogFatalAndExit(err, "failed to add gobgp config event handler") + } + if _, err = vpcEgressGatewayInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueAddVpcEgressGateway, UpdateFunc: controller.enqueueUpdateVpcEgressGateway, @@ -1172,6 +1195,10 @@ func (c *Controller) shutdown() { c.updateBgpEdgeRouterAdvertisementQueue.ShutDown() c.deleteBgpEdgeRouterAdvertisementQueue.ShutDown() + c.addGobgpConfigQueue.ShutDown() + c.updateGobgpConfigQueue.ShutDown() + c.deleteGobgpConfigQueue.ShutDown() + if c.config.EnableLb { c.addSwitchLBRuleQueue.ShutDown() c.delSwitchLBRuleQueue.ShutDown() @@ -1270,7 +1297,11 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(runWorker("add bgp edge router advertisement", c.addBgpEdgeRouterAdvertisementQueue, c.handleAddBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) go wait.Until(runWorker("update bgp edge router advertisement", c.updateBgpEdgeRouterAdvertisementQueue, c.handleUpdateBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) go wait.Until(runWorker("delete bgp edge router advertisement", c.deleteBgpEdgeRouterAdvertisementQueue, c.handleDelBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) - go wait.Until(c.resyncBgpEdgeRouterAdvertisement, 60*time.Second, ctx.Done()) + go wait.Until(c.resyncBgpRules, 60*time.Second, ctx.Done()) + go wait.Until(runWorker("add bgp edge router advertisement", c.addGobgpConfigQueue, c.handleAddGobgpConfig), time.Second, ctx.Done()) + go wait.Until(runWorker("update bgp edge router advertisement", c.updateGobgpConfigQueue, c.handleUpdateGobgpConfig), time.Second, ctx.Done()) + go wait.Until(runWorker("delete bgp edge router advertisement", c.deleteGobgpConfigQueue, c.handleDelGobgpConfig), time.Second, ctx.Done()) + go wait.Until(runWorker("update fip for vpc nat gateway", c.updateVpcFloatingIPQueue, c.handleUpdateVpcFloatingIP), time.Second, ctx.Done()) go wait.Until(runWorker("update eip for vpc nat gateway", c.updateVpcEipQueue, c.handleUpdateVpcEip), time.Second, ctx.Done()) go wait.Until(runWorker("update dnat for vpc nat gateway", c.updateVpcDnatQueue, c.handleUpdateVpcDnat), time.Second, ctx.Done()) diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go new file mode 100644 index 00000000000..56da45a010f --- /dev/null +++ b/pkg/controller/gobgp_config.go @@ -0,0 +1,428 @@ +package controller + +import ( + "context" + "fmt" + "slices" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +type updateVerGobgpConfigObject struct { + key string + oldVer *kubeovnv1.GobgpConfig + newVer *kubeovnv1.GobgpConfig +} + +func (c *Controller) enqueueAddGobgpConfig(obj any) { + key := cache.MetaObjectToName(obj.(*kubeovnv1.GobgpConfig)).String() + klog.V(3).Infof("enqueue add gobgp-configuration %s", key) + c.addGobgpConfigQueue.Add(key) +} + +func (c *Controller) enqueueUpdateGobgpConfig(oldObj, newObj any) { + key := cache.MetaObjectToName(newObj.(*kubeovnv1.GobgpConfig)).String() + klog.V(3).Infof("enqueue update gobgp-configuration %s", key) + + if oldObj == nil || newObj == nil { + klog.Warningf("enqueue update gobgp-configuration %s, but old object is nil", key) + return + } + + oldGobgpConfig := oldObj.(*kubeovnv1.GobgpConfig) + newGobgpConfig := newObj.(*kubeovnv1.GobgpConfig) + updateConfigVer := &updateVerGobgpConfigObject{ + key: key, + oldVer: oldGobgpConfig, + newVer: newGobgpConfig, + } + + c.updateGobgpConfigQueue.Add(updateConfigVer) +} + +func (c *Controller) enqueueDeleteGobgpConfig(obj any) { + var gobgpConfig *kubeovnv1.GobgpConfig + switch t := obj.(type) { + case *kubeovnv1.GobgpConfig: + gobgpConfig = t + case cache.DeletedFinalStateUnknown: + if v, ok := t.Obj.(*kubeovnv1.GobgpConfig); ok { + gobgpConfig = v + } + } + if gobgpConfig == nil { + klog.Warning("enqueueDeleteGobgpConfig: object is not GobgpConfig") + return + } + key := cache.MetaObjectToName(obj.(*kubeovnv1.GobgpConfig)).String() + klog.V(3).Infof("enqueue delete gobgp-config %s", key) + c.deleteGobgpConfigQueue.Add(key) +} + +func (c *Controller) handleAddGobgpConfig(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.gobgpConfigKeyMutex.LockKey(key) + defer func() { _ = c.gobgpConfigKeyMutex.UnlockKey(key) }() + + cachedGobgpConfig, err := c.gobgpConfigLister.GobgpConfigs(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + if !cachedGobgpConfig.DeletionTimestamp.IsZero() { + c.deleteGobgpConfigQueue.Add(key) + return nil + } + + klog.V(3).Infof("debug gobgp-config %s", cachedGobgpConfig.Name) + + if _, err := c.initGobgpConfigStatus(cachedGobgpConfig); err != nil { + klog.Error(err) + return err + } + + klog.Infof("reconciling gobgp-configuration %s", key) + gobgpConfig := cachedGobgpConfig.DeepCopy() + + if controllerutil.AddFinalizer(gobgpConfig, util.KubeOVNControllerFinalizer) { + updatedGobgpConfig, err := c.config.KubeOvnClient.KubeovnV1().GobgpConfigs(gobgpConfig.Namespace). + Update(context.Background(), + gobgpConfig, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to add finalizer for gobgp-configuration %s/%s: %w", gobgpConfig.Namespace, gobgpConfig.Name, err) + klog.Error(err) + return err + } + gobgpConfig = updatedGobgpConfig + } + + pods, err := c.validateGobgpConfig(gobgpConfig) + if err != nil || pods == nil { + klog.Error(err) + return err + } + + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + klog.Infof("handle adding gobgp-config %s", key) + // if err = c.addOrDeleteGobgpConfigRule("add", key, pod, gobgpConfig); err != nil { + // klog.Error(err) + // return err + // } + } + + gobgpConfig.Status.Conditions.SetReady("ReconcileSuccess", gobgpConfig.Generation) + if _, err = c.updateGobgpConfigStatus(gobgpConfig); err != nil { + return err + } + klog.Infof("finished reconciling gobgp-config %s", key) + + return nil +} + +func (c *Controller) handleUpdateGobgpConfig(updatedObj *updateVerGobgpConfigObject) error { + key := updatedObj.key + + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.gobgpConfigKeyMutex.LockKey(key) + defer func() { _ = c.gobgpConfigKeyMutex.UnlockKey(key) }() + + cachedGobgpConfig, err := c.gobgpConfigLister.GobgpConfigs(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + if !cachedGobgpConfig.DeletionTimestamp.IsZero() { + c.deleteGobgpConfigQueue.Add(key) + return nil + } + + klog.Infof("reconciling gobgp-configs %s", key) + gobgpConfig := cachedGobgpConfig.DeepCopy() + + pods, err := c.validateGobgpConfig(gobgpConfig) + if err != nil || pods == nil { + klog.Error(err) + return err + } + + // for _, pod := range pods { + // if len(pod.Status.PodIPs) == 0 { + // continue + // } + // klog.Infof("handle adding gobgp-configs %s", key) + // if err = c.updateGobgpConfigRule(key, pod, updatedObj.oldVer, updatedObj.newVer); err != nil { + // klog.Error(err) + // return err + // } + // } + + gobgpConfig.Status.Conditions.SetReady("ReconcileSuccess", gobgpConfig.Generation) + if _, err = c.updateGobgpConfigStatus(gobgpConfig); err != nil { + return err + } + klog.Infof("finished reconciling gobgp-configs %s", key) + + return nil +} + +func (c *Controller) handleDelGobgpConfig(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.gobgpConfigKeyMutex.LockKey(key) + defer func() { _ = c.gobgpConfigKeyMutex.UnlockKey(key) }() + + cachedGobgpConfig, err := c.gobgpConfigLister.GobgpConfigs(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + klog.Infof("reconciling gobgp-configs %s", key) + gobgpConfig := cachedGobgpConfig.DeepCopy() + + pods, err := c.validateGobgpConfig(gobgpConfig) + if err != nil || pods == nil { + klog.Error(err) + return err + } + + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + klog.Infof("handle deleting gobgp-configs %s", key) + // if err = c.addOrDeleteGobgpConfigRule("del", key, pod, gobgpConfig); err != nil { + // klog.Error(err) + // return err + // } + } + + gobgpConfig = cachedGobgpConfig.DeepCopy() + if controllerutil.RemoveFinalizer(gobgpConfig, util.KubeOVNControllerFinalizer) { + if _, err = c.config.KubeOvnClient.KubeovnV1().GobgpConfigs(gobgpConfig.Namespace). + Update(context.Background(), gobgpConfig, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to remove finalizer from gobgp-configs %s: %w", key, err) + klog.Error(err) + } + } + + gobgpConfig.Status.Conditions.SetReady("ReconcileSuccess", gobgpConfig.Generation) + if _, err = c.updateGobgpConfigStatus(gobgpConfig); err != nil { + return err + } + klog.Infof("finished reconciling gobgp-config %s", key) + + return nil +} + +// func (c *Controller) updateGobgpConfigRule(key string, pod *corev1.Pod, oldGobgpConfig, newGobgpConfig *kubeovnv1.GobgpConfig) error { +// if pod.Name == "" { +// err := fmt.Errorf("failed to get pod name %s", pod.Name) +// klog.Error(err) +// return err +// } +// // klog.Infof("update gobgp-config %s %s %s", key, oldGobgpConfig, newGobgpConfig) + +// // var oldSubnetArray []string +// // var newSubnetArray []string + +// // for _, neighbor := range oldGobgpConfig.Spec.Neighbors { +// // toAdvertisePrefixes := neighbor.ToAdvertise.Allowed.Prefixes +// // toReceivePrefixes := neighbor.ToReceive.Allowed.Prefixes + +// // klog.Infof("cleaning gobgp-config %s for subnet %s", key, prefix) +// // } + +// // for _, prefix := range newGobgpConfig.Spec.ToAdvertise.Allowed.Prefixes { +// // subnet, err := c.subnetsLister.Get(prefix) +// // if err != nil { +// // err = fmt.Errorf("failed to get subnet %s: %w", prefix, err) +// // klog.Error(err) +// // return err +// // } +// // if subnet.Spec.CIDRBlock != "" { +// // newSubnetArray = append(newSubnetArray, subnet.Spec.CIDRBlock) +// // } +// // klog.Infof("cleaning gobgp-config %s for subnet %s", key, subnet.Name) +// // } + +// // if err := c.execUpdateBgpRoute(pod, oldSubnetArray, newSubnetArray); err != nil { +// // klog.Error(err) +// // return err +// // } + +// return nil +// } + +// func (c *Controller) addOrDeleteGobgpConfigRule(op, key string, pod *corev1.Pod, gobgpConfig *kubeovnv1.GobgpConfig) error { +// neighbors := gobgpConfig.Spec.Neighbors +// klog.Infof("add delete gobgp-config %s", key) + +// subnetCidrArray := []string{} +// for _, neighbor := range neighbors { +// // address := neighbor.Address +// toAdvertise := neighbor.ToAdvertise.Allowed +// toReceive := neighbor.ToReceive.Allowed +// if toAdvertise.Mode == "all" { +// // toAdvertisePrefixes := toAdvertise.Prefixes +// // toReceivePrefixes := toReceive.Prefixes +// // Exec gobgpConfig command to advertise prefixes +// } else { +// // toAdvertisePrefixes := toAdvertise.Prefixes +// // toReceivePrefixes := toReceive.Prefixes +// // Exec gobgpConfig command to advertise prefixes +// } + +// if toReceive.Mode == "all" { +// // toAdvertisePrefixes := toAdvertise.Prefixes +// // toReceivePrefixes := toReceive.Prefixes +// // Exec gobgpConfig command to advertise prefixes +// } else { +// // toAdvertisePrefixes := toAdvertise.Prefixes +// // toReceivePrefixes := toReceive.Prefixes +// // Exec gobgpConfig command to advertise prefixes +// } +// } + +// if op == "add" { +// if err := c.execUpdateBgpRoute(pod, nil, subnetCidrArray); err != nil { +// klog.Error(err) +// return err +// } +// } else { +// if err := c.execUpdateBgpRoute(pod, subnetCidrArray, nil); err != nil { +// klog.Error(err) +// return err +// } +// } + +// return nil +// } + +func (c *Controller) validateGobgpConfig(gobgpConfig *kubeovnv1.GobgpConfig) ([]*corev1.Pod, error) { + ber, err := c.bgpEdgeRouterLister.BgpEdgeRouters(gobgpConfig.Namespace).Get(gobgpConfig.Spec.BgpEdgeRouterName) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil, fmt.Errorf("bgp edge router %s not found: %w", gobgpConfig.Spec.BgpEdgeRouterName, err) + } + } + berNeighbors := ber.Spec.BGP.Neighbors + gobgpNeighbors := gobgpConfig.Spec.Neighbors + neighborFlag := false + for _, gNeighbor := range gobgpNeighbors { + if containsNeighbor(berNeighbors, gNeighbor.Address) { + neighborFlag = true + break + } + } + + if !neighborFlag { + err = fmt.Errorf("no matching neighbor found in BgpEdgeRouter %s for GobgpConfig %s", gobgpConfig.Spec.BgpEdgeRouterName, gobgpConfig.Name) + klog.Error(err) + return nil, err + } + + deploy, err := c.berDeploymentsLister.Deployments(gobgpConfig.Namespace).Get(gobgpConfig.Spec.BgpEdgeRouterName) + if err != nil { + gobgpConfig.Status.Ready = false + msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) + gobgpConfig.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "BgpEdgeRouterDeployNotFound", msg, gobgpConfig.Generation) + _, _ = c.updateGobgpConfigStatus(gobgpConfig) + klog.Error(err) + return nil, err + } + + ready := util.DeploymentIsReady(deploy) + if !ready { + gobgpConfig.Status.Ready = false + msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) + gobgpConfig.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "BgpEdgeRouterNotEnabled", msg, gobgpConfig.Generation) + _, _ = c.updateGobgpConfigStatus(gobgpConfig) + readyErr := fmt.Sprintf("Kind %s, Deployment %s is not ready", deploy.Kind, deploy.Name) + klog.Error(readyErr) + return nil, fmt.Errorf("%s", readyErr) + } + // get the pods of the deployment to collect the pod IPs + podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) + if err != nil { + err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return nil, err + } + + pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) + if err != nil { + err = fmt.Errorf("failed to list pods of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return nil, err + } + + if ready { + gobgpConfig.Status.Ready = true + } + + return pods, nil +} + +func (c *Controller) initGobgpConfigStatus(gobgpConfig *kubeovnv1.GobgpConfig) (*kubeovnv1.GobgpConfig, error) { + var err error + gobgpConfig, err = c.updateGobgpConfigStatus(gobgpConfig) + return gobgpConfig, err +} + +func (c *Controller) updateGobgpConfigStatus(gobgpConfig *kubeovnv1.GobgpConfig) (*kubeovnv1.GobgpConfig, error) { + if len(gobgpConfig.Status.Conditions) == 0 { + gobgpConfig.Status.Conditions.SetCondition(kubeovnv1.Init, corev1.ConditionUnknown, "Processing", "", gobgpConfig.Generation) + } + + updatedGobgpConfig, err := c.config.KubeOvnClient.KubeovnV1().GobgpConfigs(gobgpConfig.Namespace). + UpdateStatus(context.Background(), gobgpConfig, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to update status of gobgp-config %s/%s: %w", gobgpConfig.Namespace, gobgpConfig.Name, err) + klog.Error(err) + return nil, err + } + + return updatedGobgpConfig, nil +} + +func containsNeighbor(neighbors []string, address string) bool { + return slices.Contains(neighbors, address) +} diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go index cc4c44b59dc..4038692ae81 100644 --- a/pkg/ovs/ovn.go +++ b/pkg/ovs/ovn.go @@ -50,6 +50,7 @@ const ( ExternalIDVpcEgressGateway = "vpc-egress-gateway" ExternalIDBgpEdgeRouter = "bgp-edge-router" ExternalIDBgpEdgeRouterAdvertisement = "bgp-edge-router-advertisement" + ExternalIDGobgpConfig = "gobgp-config" ) // NewLegacyClient init a legacy ovn client diff --git a/pkg/util/const.go b/pkg/util/const.go index 21bd6fd877f..ee5655cefab 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -116,6 +116,7 @@ const ( VpcEgressGatewayLabel = "ovn.kubernetes.io/vpc-egress-gateway" BgpEdgeRouterLabel = "ovn.kubernetes.io/bgp-edge-router" BgpEdgeRouterAdvertisementLabel = "ovn.kubernetes.io/bgp-edge-router-advertisement" + GobgpConfigLabel = "ovn.kubernetes.io/gobgp-config" GenerateHashAnnotation = "ovn.kubernetes.io/generate-hash" VpcLastName = "ovn.kubernetes.io/last_vpc_name" From ae1939bf59fd15617008cb7106760d02725bda9c Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Thu, 7 Aug 2025 21:27:43 -0700 Subject: [PATCH 24/66] GobgpConfig's bgpEdgeRouter namespace key is added. --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index b5bae517441..d98f4d7b74a 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3743,7 +3743,12 @@ spec: - neighbors properties: bgpEdgeRouter: - type: string + type: object + properties: + name: + type: string + namespace: + type: string neighbors: type: array items: From ca6c3f756c4f978fc95521ba094461f2101a1c60 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Thu, 7 Aug 2025 21:33:41 -0700 Subject: [PATCH 25/66] listing gobgp-config needs ber namespace and added that logic --- pkg/apis/kubeovn/v1/gobgp-config.go | 12 ++++++++++-- pkg/controller/gobgp_config.go | 8 ++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/apis/kubeovn/v1/gobgp-config.go b/pkg/apis/kubeovn/v1/gobgp-config.go index a42ba10f5d7..48c6cb9f447 100644 --- a/pkg/apis/kubeovn/v1/gobgp-config.go +++ b/pkg/apis/kubeovn/v1/gobgp-config.go @@ -41,8 +41,8 @@ func (g *GobgpConfig) Subnet(subnets []string) []string { // } type GobgpConfigSpec struct { - BgpEdgeRouterName string `json:"bgpEdgeRouterName"` - Neighbors []Neighbors `json:"neighbors,omitempty"` + BgpEdgeRouterInfo BgpEdgeRouterInfo `json:"bgpEdgeRouterName"` + Neighbors []Neighbors `json:"neighbors,omitempty"` } type GobgpConfigStatus struct { @@ -50,6 +50,14 @@ type GobgpConfigStatus struct { Conditions Conditions `json:"conditions,omitempty"` } +type BgpEdgeRouterInfo struct { + Name string `json:"name"` + Namespace string `json:"namespace"` +} + +// Neighbors defines the BGP neighbors configuration +// +k8s:openapi-gen=true +// +genclient:nonNamespaced type Neighbors struct { Address string `json:"address"` ToAdvertise ToAdvertise `json:"toAdvertise"` diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index 56da45a010f..291079733b9 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -337,10 +337,10 @@ func (c *Controller) handleDelGobgpConfig(key string) error { // } func (c *Controller) validateGobgpConfig(gobgpConfig *kubeovnv1.GobgpConfig) ([]*corev1.Pod, error) { - ber, err := c.bgpEdgeRouterLister.BgpEdgeRouters(gobgpConfig.Namespace).Get(gobgpConfig.Spec.BgpEdgeRouterName) + ber, err := c.bgpEdgeRouterLister.BgpEdgeRouters(gobgpConfig.Spec.BgpEdgeRouterInfo.Namespace).Get(gobgpConfig.Spec.BgpEdgeRouterInfo.Name) if err != nil { if k8serrors.IsNotFound(err) { - return nil, fmt.Errorf("bgp edge router %s not found: %w", gobgpConfig.Spec.BgpEdgeRouterName, err) + return nil, fmt.Errorf("bgp edge router %s not found: %w", gobgpConfig.Spec.BgpEdgeRouterInfo.Name, err) } } berNeighbors := ber.Spec.BGP.Neighbors @@ -354,12 +354,12 @@ func (c *Controller) validateGobgpConfig(gobgpConfig *kubeovnv1.GobgpConfig) ([] } if !neighborFlag { - err = fmt.Errorf("no matching neighbor found in BgpEdgeRouter %s for GobgpConfig %s", gobgpConfig.Spec.BgpEdgeRouterName, gobgpConfig.Name) + err = fmt.Errorf("no matching neighbor found in BgpEdgeRouter %s for GobgpConfig %s", gobgpConfig.Spec.BgpEdgeRouterInfo.Name, gobgpConfig.Name) klog.Error(err) return nil, err } - deploy, err := c.berDeploymentsLister.Deployments(gobgpConfig.Namespace).Get(gobgpConfig.Spec.BgpEdgeRouterName) + deploy, err := c.berDeploymentsLister.Deployments(gobgpConfig.Spec.BgpEdgeRouterInfo.Namespace).Get(gobgpConfig.Spec.BgpEdgeRouterInfo.Name) if err != nil { gobgpConfig.Status.Ready = false msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) From 031a6783099161de32c5f57b761fb530e1d64fb6 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Sun, 10 Aug 2025 22:42:36 -0700 Subject: [PATCH 26/66] update-bgp-policy.sh added. --- dist/images/update-bgp-policy.sh | 205 +++++++++++++++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 dist/images/update-bgp-policy.sh diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh new file mode 100644 index 00000000000..522c2921231 --- /dev/null +++ b/dist/images/update-bgp-policy.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# update-bgp-policy.sh +# shellcheck disable=SC2086,SC2155 + +set -euo pipefail + +GOBGP_BIN=${GOBGP_BIN:-$(command -v gobgp || true)} +[[ -z "$GOBGP_BIN" ]] && { echo "ERROR: gobgp binary not found" >&2; exit 1; } + +die() { echo "ERROR: $*" >&2; exit 1; } + +usage() { + cat >&2 < + $0 flush-neighbor-policy + $0 flush-prefix-in + $0 flush-prefix-out + $0 add-prefix + +Examples: + $0 set-neighbor-policy 1.1.1.1 + $0 flush-neighbor-policy 1.1.1.1 + $0 flush-prefix-in 1.1.1.1 + $0 flush-prefix-out 1.1.1.1 + $0 add-prefix in 1.1.1.1 "0.0.0.0/0 0..32","1.1.1.0/24","10.0.0.0/8 16..32" +EOF + exit 1 +} + +# Validate IPv4 format (simple regex) +validate_ip() { + local ip=$1 + if [[ ! $ip =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then + die "Invalid IPv4 address: $ip" + fi +} + +exec_cmd() { "$@" || die "failed: $*"; } + +set_neighbor_policy() { + local nbr_ip=$1; validate_ip "$nbr_ip" + local prefix_in="prefix-${nbr_ip}-in" + local prefix_out="prefix-${nbr_ip}-out" + local nbr_name="neighbor-${nbr_ip}" + local stmt_in="stmt-${nbr_ip}-in" + local stmt_out="stmt-${nbr_ip}-out" + local policy_in="policy-${nbr_ip}-in" + local policy_out="policy-${nbr_ip}-out" + + echo "=== Setting policy for neighbor $nbr_ip ===" + echo "-> Creating prefix-lists" + exec_cmd $GOBGP_BIN policy prefix add $prefix_in 0.0.0.0/0 0..32 + exec_cmd $GOBGP_BIN policy prefix add $prefix_out 0.0.0.0/0 0..32 + + echo "-> Defining neighbor" + exec_cmd $GOBGP_BIN policy neighbor add $nbr_name $nbr_ip + + echo "-> Building inbound statement" + exec_cmd $GOBGP_BIN policy statement add $stmt_in + exec_cmd $GOBGP_BIN policy statement $stmt_in add action accept + exec_cmd $GOBGP_BIN policy statement $stmt_in add condition prefix $prefix_in + exec_cmd $GOBGP_BIN policy statement $stmt_in add condition neighbor $nbr_name + + echo "-> Building outbound statement" + exec_cmd $GOBGP_BIN policy statement add $stmt_out + exec_cmd $GOBGP_BIN policy statement $stmt_out add action accept + exec_cmd $GOBGP_BIN policy statement $stmt_out add condition prefix $prefix_out + exec_cmd $GOBGP_BIN policy statement $stmt_out add condition neighbor $nbr_name + + echo "-> Assembling policies" + exec_cmd $GOBGP_BIN policy add $policy_in $stmt_in + exec_cmd $GOBGP_BIN policy add $policy_out $stmt_out + + echo "-> Applying to global" + exec_cmd $GOBGP_BIN global policy import add $policy_in + exec_cmd $GOBGP_BIN global policy export add $policy_out + + echo "=== Policy set successfully for $nbr_ip ===" +} + +flush_neighbor_policy() { + local nbr_ip=$1; validate_ip "$nbr_ip" + local prefix_in="prefix-${nbr_ip}-in" + local prefix_out="prefix-${nbr_ip}-out" + local nbr_name="neighbor-${nbr_ip}" + local stmt_in="stmt-${nbr_ip}-in" + local stmt_out="stmt-${nbr_ip}-out" + local policy_in="policy-${nbr_ip}-in" + local policy_out="policy-${nbr_ip}-out" + + echo "=== Flushing policy for neighbor $nbr_ip ===" + echo "-> Removing from global policies" + exec_cmd $GOBGP_BIN global policy import del $policy_in + exec_cmd $GOBGP_BIN global policy export del $policy_out + + echo "-> Removing policies" + exec_cmd $GOBGP_BIN policy del $policy_in + exec_cmd $GOBGP_BIN policy del $policy_out + + echo "-> Removing statements" + exec_cmd $GOBGP_BIN policy statement del $stmt_in + exec_cmd $GOBGP_BIN policy statement del $stmt_out + + echo "-> Removing neighbor definition" + exec_cmd $GOBGP_BIN policy neighbor del $nbr_name + + echo "-> Removing prefix-lists" + exec_cmd $GOBGP_BIN policy prefix del $prefix_in + exec_cmd $GOBGP_BIN policy prefix del $prefix_out + + echo "=== Policy flushed successfully for $nbr_ip ===" +} + +flush_prefix_in() { + local nbr_ip=$1; validate_ip "$nbr_ip" + local prefix_name="prefix-${nbr_ip}-in" + + echo "=== Flushing all entries from $prefix_name ===" + $GOBGP_BIN policy prefix $prefix_name 2>/dev/null \ + | awk 'NR>1 && NF>=2 { print $(NF-1), $NF }' \ + | while read -r iprange mask; do + echo "-> Deleting: $iprange $mask" + exec_cmd $GOBGP_BIN policy prefix del $prefix_name $iprange $mask + done + + echo "=== All entries removed from $prefix_name ===" +} + +flush_prefix_out() { + local nbr_ip=$1; validate_ip "$nbr_ip" + local prefix_name="prefix-${nbr_ip}-out" + + echo "=== Flushing all entries from $prefix_name ===" + $GOBGP_BIN policy prefix $prefix_name 2>/dev/null \ + | awk 'NR>1 && NF>=2 { print $(NF-1), $NF }' \ + | while read -r iprange mask; do + echo "-> Deleting: $iprange $mask" + exec_cmd $GOBGP_BIN policy prefix del $prefix_name $iprange $mask + done + + echo "=== All entries removed from $prefix_name ===" +} + +add_prefix() { + local dir=$1; shift + local nbr_ip=$1; shift + validate_ip "$nbr_ip" + [[ $dir != in && $dir != out ]] && die "Direction must be 'in' or 'out'" + local prefix_name="prefix-${nbr_ip}-${dir}" + + # split comma-separated list in first argument after IP + IFS=',' read -ra entries <<< "$*" + + echo "=== Adding prefixes to $prefix_name ===" + for entry in "${entries[@]}"; do + entry="${entry%\"}" + entry="${entry#\"}" + entry="${entry##*( )}" + entry="${entry%%*( )}" + + if [[ $entry =~ ^([^[:space:]]+)[[:space:]]+(.+)$ ]]; then + local ip_pref=${BASH_REMATCH[1]} + local mask=${BASH_REMATCH[2]} + echo "-> Adding: $ip_pref $mask" + exec_cmd $GOBGP_BIN policy prefix add $prefix_name $ip_pref $mask + else + echo "-> Adding: $entry" + exec_cmd $GOBGP_BIN policy prefix add $prefix_name $entry + fi + done + echo "=== Done ===" +} + +main() { + [[ $# -lt 1 ]] && usage + case "$1" in + set-neighbor-policy) + [[ $# -ne 2 ]] && usage + set_neighbor_policy "$2" + ;; + flush-neighbor-policy) + [[ $# -ne 2 ]] && usage + flush_neighbor_policy "$2" + ;; + flush-prefix-in) + [[ $# -ne 2 ]] && usage + flush_prefix_in "$2" + ;; + flush-prefix-out) + [[ $# -ne 2 ]] && usage + flush_prefix_out "$2" + ;; + add-prefix) + [[ $# -lt 4 ]] && usage + shift + add_prefix "$@" + ;; + *) + usage + ;; + esac +} + +main "$@" \ No newline at end of file From 53ffc33e28268eafed68e17c8ad47408bd78cf2e Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Tue, 12 Aug 2025 00:02:10 -0700 Subject: [PATCH 27/66] [fix] for bgp router id null --- pkg/controller/bgp_edge_router.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go index 947be6975be..99e132121c0 100644 --- a/pkg/controller/bgp_edge_router.go +++ b/pkg/controller/bgp_edge_router.go @@ -1139,6 +1139,14 @@ func bgpEdgeRouterContainerBGP(speakerImage, routerName string, speakerParams *k }, }, }, + { + Name: "MULTI_NET_STATUS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.annotations['k8s.v1.cni.cncf.io/networks-status']", + }, + }, + }, }, Args: args, VolumeMounts: []corev1.VolumeMount{ From 80fe78cc7ca79fbb91be57942cea65f24bef795f Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Tue, 12 Aug 2025 01:01:48 -0700 Subject: [PATCH 28/66] gobgp-config batch process --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 4 +- dist/images/update-bgp-policy.sh | 105 +++++++-- pkg/apis/kubeovn/v1/gobgp-config.go | 2 +- pkg/controller/gobgp_config.go | 229 +++++++++++--------- 4 files changed, 214 insertions(+), 126 deletions(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index d98f4d7b74a..76605f4320b 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3710,7 +3710,7 @@ spec: additionalPrinterColumns: - name: bgp-edge-router type: string - jsonPath: .spec.bgpEdgeRouter + jsonPath: .spec.bgpEdgeRouterInfo.name - name: neighbor-address type: string jsonPath: .spec.neighbors[*].address @@ -3742,7 +3742,7 @@ spec: - bgpEdgeRouter - neighbors properties: - bgpEdgeRouter: + bgpEdgeRouterInfo: type: object properties: name: diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index 522c2921231..ff3dc598605 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# update-bgp-policy.sh +# update-bgp-policy.sh - Hybrid Version # shellcheck disable=SC2086,SC2155 set -euo pipefail @@ -16,14 +16,20 @@ Usage: $0 flush-neighbor-policy $0 flush-prefix-in $0 flush-prefix-out - $0 add-prefix + $0 add-prefix + $0 --batch [ARGS1...] -- [ARGS2...] -- ... Examples: + # Single command execution $0 set-neighbor-policy 1.1.1.1 $0 flush-neighbor-policy 1.1.1.1 $0 flush-prefix-in 1.1.1.1 $0 flush-prefix-out 1.1.1.1 $0 add-prefix in 1.1.1.1 "0.0.0.0/0 0..32","1.1.1.0/24","10.0.0.0/8 16..32" + + # Batch execution (multiple commands in one run) + $0 --batch set-neighbor-policy 1.1.1.1 -- add-prefix in 1.1.1.1 "10.0.0.0/8" + $0 --batch flush-prefix-in 1.1.1.1 -- add-prefix in 1.1.1.1 "192.168.0.0/16" -- flush-prefix-out 1.1.1.1 EOF exit 1 } @@ -154,10 +160,11 @@ add_prefix() { echo "=== Adding prefixes to $prefix_name ===" for entry in "${entries[@]}"; do + # Clean quotes and whitespace entry="${entry%\"}" entry="${entry#\"}" - entry="${entry##*( )}" - entry="${entry%%*( )}" + entry="${entry##( )}" + entry="${entry%%( )}" if [[ $entry =~ ^([^[:space:]]+)[[:space:]]+(.+)$ ]]; then local ip_pref=${BASH_REMATCH[1]} @@ -172,34 +179,96 @@ add_prefix() { echo "=== Done ===" } -main() { - [[ $# -lt 1 ]] && usage - case "$1" in +# Execute a single command +execute_single_command() { + local cmd=$1; shift + + case "$cmd" in set-neighbor-policy) - [[ $# -ne 2 ]] && usage - set_neighbor_policy "$2" + [[ $# -ne 1 ]] && die "set-neighbor-policy requires exactly 1 argument (NEIGHBOR_IP)" + set_neighbor_policy "$1" ;; flush-neighbor-policy) - [[ $# -ne 2 ]] && usage - flush_neighbor_policy "$2" + [[ $# -ne 1 ]] && die "flush-neighbor-policy requires exactly 1 argument (NEIGHBOR_IP)" + flush_neighbor_policy "$1" ;; flush-prefix-in) - [[ $# -ne 2 ]] && usage - flush_prefix_in "$2" + [[ $# -ne 1 ]] && die "flush-prefix-in requires exactly 1 argument (NEIGHBOR_IP)" + flush_prefix_in "$1" ;; flush-prefix-out) - [[ $# -ne 2 ]] && usage - flush_prefix_out "$2" + [[ $# -ne 1 ]] && die "flush-prefix-out requires exactly 1 argument (NEIGHBOR_IP)" + flush_prefix_out "$1" ;; add-prefix) - [[ $# -lt 4 ]] && usage - shift + [[ $# -lt 3 ]] && die "add-prefix requires at least 3 arguments (in|out NEIGHBOR_IP PREFIXES...)" add_prefix "$@" ;; *) - usage + die "Unknown command: $cmd" ;; esac } +# Parse batch commands separated by -- +parse_batch_commands() { + local -a current_cmd=() + local -a all_commands=() +for arg in "$@"; do + if [[ "$arg" == "--" ]]; then + if [[ ${#current_cmd[@]} -gt 0 ]]; then + all_commands+=("$(printf '%s\n' "${current_cmd[@]}")") + current_cmd=() + fi + else + current_cmd+=("$arg") + fi + done + + # Add the last command if exists + if [[ ${#current_cmd[@]} -gt 0 ]]; then + all_commands+=("$(printf '%s\n' "${current_cmd[@]}")") + fi + + # Execute all commands + local cmd_count=1 + for cmd_str in "${all_commands[@]}"; do + echo "" + echo "🔸 Executing batch command #$cmd_count" + echo "-----------------------------------" + + # Convert newline-separated string back to array + local -a cmd_args=() + while IFS= read -r line; do + [[ -n "$line" ]] && cmd_args+=("$line") + done <<< "$cmd_str" + + if [[ ${#cmd_args[@]} -gt 0 ]]; then + execute_single_command "${cmd_args[@]}" + fi + + ((cmd_count++)) + done +} + +main() { + [[ $# -lt 1 ]] && usage + + # Check for batch mode + if [[ "$1" == "--batch" ]]; then + shift + [[ $# -lt 1 ]] && die "Batch mode requires at least one command" + + echo "🚀 Starting batch execution mode" + echo "=================================" + parse_batch_commands "$@" + echo "" + echo "✅ All batch commands completed successfully" + + else + # Single command mode (original behavior) + execute_single_command "$@" + fi +} + main "$@" \ No newline at end of file diff --git a/pkg/apis/kubeovn/v1/gobgp-config.go b/pkg/apis/kubeovn/v1/gobgp-config.go index 48c6cb9f447..81a9860bb93 100644 --- a/pkg/apis/kubeovn/v1/gobgp-config.go +++ b/pkg/apis/kubeovn/v1/gobgp-config.go @@ -41,7 +41,7 @@ func (g *GobgpConfig) Subnet(subnets []string) []string { // } type GobgpConfigSpec struct { - BgpEdgeRouterInfo BgpEdgeRouterInfo `json:"bgpEdgeRouterName"` + BgpEdgeRouterInfo BgpEdgeRouterInfo `json:"bgpEdgeRouterInfo"` Neighbors []Neighbors `json:"neighbors,omitempty"` } diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index 291079733b9..3bbd9da058c 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -2,8 +2,10 @@ package controller import ( "context" + "errors" "fmt" "slices" + "strings" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -125,10 +127,10 @@ func (c *Controller) handleAddGobgpConfig(key string) error { continue } klog.Infof("handle adding gobgp-config %s", key) - // if err = c.addOrDeleteGobgpConfigRule("add", key, pod, gobgpConfig); err != nil { - // klog.Error(err) - // return err - // } + if err = c.execUpdateBgpPolicy(key, pod, nil, gobgpConfig); err != nil { + klog.Error(err) + return err + } } gobgpConfig.Status.Conditions.SetReady("ReconcileSuccess", gobgpConfig.Generation) @@ -175,16 +177,16 @@ func (c *Controller) handleUpdateGobgpConfig(updatedObj *updateVerGobgpConfigObj return err } - // for _, pod := range pods { - // if len(pod.Status.PodIPs) == 0 { - // continue - // } - // klog.Infof("handle adding gobgp-configs %s", key) - // if err = c.updateGobgpConfigRule(key, pod, updatedObj.oldVer, updatedObj.newVer); err != nil { - // klog.Error(err) - // return err - // } - // } + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + klog.Infof("handle adding gobgp-configs %s", key) + if err = c.execUpdateBgpPolicy(key, pod, updatedObj.oldVer, updatedObj.newVer); err != nil { + klog.Error(err) + return err + } + } gobgpConfig.Status.Conditions.SetReady("ReconcileSuccess", gobgpConfig.Generation) if _, err = c.updateGobgpConfigStatus(gobgpConfig); err != nil { @@ -228,10 +230,10 @@ func (c *Controller) handleDelGobgpConfig(key string) error { continue } klog.Infof("handle deleting gobgp-configs %s", key) - // if err = c.addOrDeleteGobgpConfigRule("del", key, pod, gobgpConfig); err != nil { - // klog.Error(err) - // return err - // } + if err = c.execUpdateBgpPolicy(key, pod, gobgpConfig, nil); err != nil { + klog.Error(err) + return err + } } gobgpConfig = cachedGobgpConfig.DeepCopy() @@ -243,100 +245,89 @@ func (c *Controller) handleDelGobgpConfig(key string) error { } } - gobgpConfig.Status.Conditions.SetReady("ReconcileSuccess", gobgpConfig.Generation) - if _, err = c.updateGobgpConfigStatus(gobgpConfig); err != nil { - return err - } klog.Infof("finished reconciling gobgp-config %s", key) return nil } -// func (c *Controller) updateGobgpConfigRule(key string, pod *corev1.Pod, oldGobgpConfig, newGobgpConfig *kubeovnv1.GobgpConfig) error { -// if pod.Name == "" { -// err := fmt.Errorf("failed to get pod name %s", pod.Name) -// klog.Error(err) -// return err -// } -// // klog.Infof("update gobgp-config %s %s %s", key, oldGobgpConfig, newGobgpConfig) - -// // var oldSubnetArray []string -// // var newSubnetArray []string - -// // for _, neighbor := range oldGobgpConfig.Spec.Neighbors { -// // toAdvertisePrefixes := neighbor.ToAdvertise.Allowed.Prefixes -// // toReceivePrefixes := neighbor.ToReceive.Allowed.Prefixes - -// // klog.Infof("cleaning gobgp-config %s for subnet %s", key, prefix) -// // } - -// // for _, prefix := range newGobgpConfig.Spec.ToAdvertise.Allowed.Prefixes { -// // subnet, err := c.subnetsLister.Get(prefix) -// // if err != nil { -// // err = fmt.Errorf("failed to get subnet %s: %w", prefix, err) -// // klog.Error(err) -// // return err -// // } -// // if subnet.Spec.CIDRBlock != "" { -// // newSubnetArray = append(newSubnetArray, subnet.Spec.CIDRBlock) -// // } -// // klog.Infof("cleaning gobgp-config %s for subnet %s", key, subnet.Name) -// // } - -// // if err := c.execUpdateBgpRoute(pod, oldSubnetArray, newSubnetArray); err != nil { -// // klog.Error(err) -// // return err -// // } - -// return nil -// } - -// func (c *Controller) addOrDeleteGobgpConfigRule(op, key string, pod *corev1.Pod, gobgpConfig *kubeovnv1.GobgpConfig) error { -// neighbors := gobgpConfig.Spec.Neighbors -// klog.Infof("add delete gobgp-config %s", key) - -// subnetCidrArray := []string{} -// for _, neighbor := range neighbors { -// // address := neighbor.Address -// toAdvertise := neighbor.ToAdvertise.Allowed -// toReceive := neighbor.ToReceive.Allowed -// if toAdvertise.Mode == "all" { -// // toAdvertisePrefixes := toAdvertise.Prefixes -// // toReceivePrefixes := toReceive.Prefixes -// // Exec gobgpConfig command to advertise prefixes -// } else { -// // toAdvertisePrefixes := toAdvertise.Prefixes -// // toReceivePrefixes := toReceive.Prefixes -// // Exec gobgpConfig command to advertise prefixes -// } - -// if toReceive.Mode == "all" { -// // toAdvertisePrefixes := toAdvertise.Prefixes -// // toReceivePrefixes := toReceive.Prefixes -// // Exec gobgpConfig command to advertise prefixes -// } else { -// // toAdvertisePrefixes := toAdvertise.Prefixes -// // toReceivePrefixes := toReceive.Prefixes -// // Exec gobgpConfig command to advertise prefixes -// } -// } - -// if op == "add" { -// if err := c.execUpdateBgpRoute(pod, nil, subnetCidrArray); err != nil { -// klog.Error(err) -// return err -// } -// } else { -// if err := c.execUpdateBgpRoute(pod, subnetCidrArray, nil); err != nil { -// klog.Error(err) -// return err -// } -// } - -// return nil -// } +func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpConfig, newGobgpConfig *kubeovnv1.GobgpConfig) error { + klog.Infof("execUpdateBgpPolicy %s", key) + + if pod.Name == "" { + err := fmt.Errorf("failed to get pod name %s", pod.Name) + klog.Error(err) + return err + } + klog.Infof("execUpdateBgpPolicy %s", key) + + cmdArs := []string{} + if oldGobgpConfig != nil { + klog.Infof("execUpdateBgpPolicy %s", key) + + for _, neighbor := range oldGobgpConfig.Spec.Neighbors { + nbrIP := neighbor.Address + if len(nbrIP) == 0 { + klog.Warningf("neighbor address is empty for gobgp-config %s", key) + continue + } + // erase neighbor. + cmdArs = append(cmdArs, "--", "flush-neighbor-policy", nbrIP) + // cmdArs = append(cmdArs, "--", "flush-prefix-out", nbrIP) + // cmdArs = append(cmdArs, "--", "flush-prefix-in", nbrIP) + // rcvMode := neighbor.ToReceive.Allowed.Mode + // rcvPrefixes := neighbor.ToReceive.Allowed.Prefixes + // if rcvMode == "all" { + // rcvPrefixes = []string{"0.0.0.0/0 0..32"} + // } + // cmdArs = append(cmdArs, append([]string{"--", "flush-prefix-in"}, rcvPrefixes...)...) + } + } + if newGobgpConfig != nil { + for _, neighbor := range newGobgpConfig.Spec.Neighbors { + nbrIP := neighbor.Address + if len(nbrIP) == 0 { + klog.Warningf("neighbor address is empty for gobgp-config %s", key) + continue + } + cmdArs = append(cmdArs, "--", "set-neighbor-policy", nbrIP) + + // toAdvertise + advMode := neighbor.ToAdvertise.Allowed.Mode + advPrefixes := neighbor.ToAdvertise.Allowed.Prefixes + quoted := make([]string, len(advPrefixes)) + + if advMode == "all" { + advPrefixes = []string{"0.0.0.0/0 0..32"} + } + for i, p := range advPrefixes { + quoted[i] = fmt.Sprintf("\"%s\"", p) + } + cmdArs = append(cmdArs, "--", "add-prefix", "out", nbrIP, strings.Join(quoted, ",")) + + // toReceive + recvMode := neighbor.ToReceive.Allowed.Mode + recvPrefixes := neighbor.ToReceive.Allowed.Prefixes + if recvMode == "all" { + recvPrefixes = []string{"0.0.0.0/0 0..32"} + } + quoted = make([]string, len(recvPrefixes)) + for i, p := range recvPrefixes { + quoted[i] = fmt.Sprintf("\"%s\"", p) + } + cmdArs = append(cmdArs, "--", "add-prefix", "in", nbrIP, strings.Join(quoted, ",")) + } + } + + // cmdArs = append(cmdArs, "list_announced_route") + if err := c.execCmd(pod, cmdArs); err != nil { + klog.Error(err) + return err + } + return nil +} func (c *Controller) validateGobgpConfig(gobgpConfig *kubeovnv1.GobgpConfig) ([]*corev1.Pod, error) { + klog.Infof("gobgpConfignamespace: %s name: %s", gobgpConfig.Spec.BgpEdgeRouterInfo.Namespace, gobgpConfig.Spec.BgpEdgeRouterInfo.Name) ber, err := c.bgpEdgeRouterLister.BgpEdgeRouters(gobgpConfig.Spec.BgpEdgeRouterInfo.Namespace).Get(gobgpConfig.Spec.BgpEdgeRouterInfo.Name) if err != nil { if k8serrors.IsNotFound(err) { @@ -423,6 +414,34 @@ func (c *Controller) updateGobgpConfigStatus(gobgpConfig *kubeovnv1.GobgpConfig) return updatedGobgpConfig, nil } +func (c *Controller) execCmd(pod *corev1.Pod, cmdArs []string) error { + cmd := fmt.Sprintf("bash /kube-ovn/update-bgp-policy.sh --batch %s", strings.Join(cmdArs, " ")) + + klog.Infof("exec command : %s", cmd) + stdOutput, errOutput, err := util.ExecuteCommandInContainer(c.config.KubeClient, c.config.KubeRestConfig, pod.Namespace, pod.Name, "bgp-router-speaker", []string{"/bin/bash", "-c", cmd}...) + if err != nil { + if len(errOutput) > 0 { + klog.Errorf("failed to ExecuteCommandInContainer, errOutput: %v", errOutput) + } + if len(stdOutput) > 0 { + klog.Infof("failed to ExecuteCommandInContainer, stdOutput: %v", stdOutput) + } + klog.Error(err) + return err + } + + if len(stdOutput) > 0 { + klog.Infof("ExecuteCommandInContainer stdOutput: %v", stdOutput) + } + + if len(errOutput) > 0 { + klog.Errorf("failed to ExecuteCommandInContainer errOutput: %v", errOutput) + return errors.New(errOutput) + } + + return nil +} + func containsNeighbor(neighbors []string, address string) bool { return slices.Contains(neighbors, address) } From f835541dd1788efc5e236c3a06bd26d54440d8b3 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Tue, 12 Aug 2025 01:07:35 -0700 Subject: [PATCH 29/66] [feat] add default action in script --- dist/images/update-bgp-policy.sh | 38 +++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index ff3dc598605..df4db997e50 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -2,7 +2,7 @@ # update-bgp-policy.sh - Hybrid Version # shellcheck disable=SC2086,SC2155 -set -euo pipefail +set -u GOBGP_BIN=${GOBGP_BIN:-$(command -v gobgp || true)} [[ -z "$GOBGP_BIN" ]] && { echo "ERROR: gobgp binary not found" >&2; exit 1; } @@ -17,6 +17,7 @@ Usage: $0 flush-prefix-in $0 flush-prefix-out $0 add-prefix + $0 set-default-action $0 --batch [ARGS1...] -- [ARGS2...] -- ... Examples: @@ -179,6 +180,28 @@ add_prefix() { echo "=== Done ===" } +validate_action() { + local action=$1 + if [[ "$action" != "accept" && "$action" != "reject" ]]; then + die "Invalid action: $action. Must be 'accept' or 'reject'" + fi +} + +set_default_action() { + local action=$1; validate_action "$action" + + echo "=== Setting default action to $action ===" + + echo "-> Applying default policy to global import" + exec_cmd $GOBGP_BIN global policy import add default $action + + echo "-> Applying default policy to global export" + exec_cmd $GOBGP_BIN global policy export add default $action + + echo "=== Default action set to $action successfully ===" +} + + # Execute a single command execute_single_command() { local cmd=$1; shift @@ -204,6 +227,10 @@ execute_single_command() { [[ $# -lt 3 ]] && die "add-prefix requires at least 3 arguments (in|out NEIGHBOR_IP PREFIXES...)" add_prefix "$@" ;; + set-default-action) + [[ $# -ne 1 ]] && die "set-default-action requires exactly 1 argument (accept|reject)" + set_default_action "$1" + ;; *) die "Unknown command: $cmd" ;; @@ -214,7 +241,8 @@ execute_single_command() { parse_batch_commands() { local -a current_cmd=() local -a all_commands=() -for arg in "$@"; do + + for arg in "$@"; do if [[ "$arg" == "--" ]]; then if [[ ${#current_cmd[@]} -gt 0 ]]; then all_commands+=("$(printf '%s\n' "${current_cmd[@]}")") @@ -234,7 +262,7 @@ for arg in "$@"; do local cmd_count=1 for cmd_str in "${all_commands[@]}"; do echo "" - echo "🔸 Executing batch command #$cmd_count" + echo "Executing batch command #$cmd_count" echo "-----------------------------------" # Convert newline-separated string back to array @@ -259,11 +287,11 @@ main() { shift [[ $# -lt 1 ]] && die "Batch mode requires at least one command" - echo "🚀 Starting batch execution mode" + echo "Starting batch execution mode" echo "=================================" parse_batch_commands "$@" echo "" - echo "✅ All batch commands completed successfully" + echo "All batch commands completed successfully" else # Single command mode (original behavior) From 08317f9b26d74be2443900d9ddf58b1a45af4ca4 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Tue, 12 Aug 2025 01:26:22 -0700 Subject: [PATCH 30/66] [fix] delete default prefix in bgp config script --- dist/images/update-bgp-policy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index df4db997e50..bbf9683a923 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -57,8 +57,8 @@ set_neighbor_policy() { echo "=== Setting policy for neighbor $nbr_ip ===" echo "-> Creating prefix-lists" - exec_cmd $GOBGP_BIN policy prefix add $prefix_in 0.0.0.0/0 0..32 - exec_cmd $GOBGP_BIN policy prefix add $prefix_out 0.0.0.0/0 0..32 + # exec_cmd $GOBGP_BIN policy prefix add $prefix_in 0.0.0.0/0 0..32 + # exec_cmd $GOBGP_BIN policy prefix add $prefix_out 0.0.0.0/0 0..32 echo "-> Defining neighbor" exec_cmd $GOBGP_BIN policy neighbor add $nbr_name $nbr_ip From 530c433d68043be69ab6a446ebad0065052c4653 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Tue, 12 Aug 2025 23:40:27 -0700 Subject: [PATCH 31/66] modify kube-ovn-crd, gobgp-config crd's neighbors and toReceive mode is all, it cannot be set prefixes, and reduce listing gobgp-config crd's info --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index 76605f4320b..bef165b9148 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3714,18 +3714,6 @@ spec: - name: neighbor-address type: string jsonPath: .spec.neighbors[*].address - - name: toReceive-mode - type: string - jsonPath: .spec.neighbors[*].toReceive.allowed.mode - - name: toReceive-prefixes - type: string - jsonPath: .spec.neighbors[*].toReceive.allowed.prefixes - - name: toAdvertise-mode - type: string - jsonPath: .spec.neighbors[*].toAdvertise.allowed.mode - - name: toAdvertise-prefixes - type: string - jsonPath: .spec.neighbors[*].toAdvertise.allowed.prefixes - name: READY type: boolean jsonPath: .status.ready @@ -3739,7 +3727,7 @@ spec: spec: type: object required: - - bgpEdgeRouter + - bgpEdgeRouterInfo - neighbors properties: bgpEdgeRouterInfo: @@ -3751,6 +3739,9 @@ spec: type: string neighbors: type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - address items: type: object required: @@ -3782,6 +3773,8 @@ spec: x-kubernetes-validations: - rule: "self.mode != 'filtered' || (has(self.prefixes) && size(self.prefixes) > 0)" message: "prefixes must be set and non-empty when mode is 'filtered'" + - rule: "self.mode != 'all' || !has(self.prefixes) || size(self.prefixes) == 0" + message: "If mode is 'all', cannot set prefixes" toReceive: type: object required: @@ -3803,6 +3796,8 @@ spec: x-kubernetes-validations: - rule: "self.mode != 'filtered' || (has(self.prefixes) && size(self.prefixes) > 0)" message: "prefixes must be set and non-empty when mode is 'filtered'" + - rule: "self.mode != 'all' || !has(self.prefixes) || size(self.prefixes) == 0" + message: "If mode is 'all', cannot set prefixes" status: type: object properties: From 3236e3a6acfb77c0186712eac45e5f3ced24a4e5 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Tue, 12 Aug 2025 23:50:11 -0700 Subject: [PATCH 32/66] when user make gobgp policy, adding 0.0.0.0/0 default rule is removed. --- dist/images/update-bgp-policy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index df4db997e50..03b7970fe95 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -57,8 +57,8 @@ set_neighbor_policy() { echo "=== Setting policy for neighbor $nbr_ip ===" echo "-> Creating prefix-lists" - exec_cmd $GOBGP_BIN policy prefix add $prefix_in 0.0.0.0/0 0..32 - exec_cmd $GOBGP_BIN policy prefix add $prefix_out 0.0.0.0/0 0..32 + # exec_cmd $GOBGP_BIN policy prefix add $prefix_in 0.0.0.0/0 0..32 + # exec_cmd $GOBGP_BIN policy prefix add $prefix_out 0.0.0.0/0 0..32 echo "-> Defining neighbor" exec_cmd $GOBGP_BIN policy neighbor add $nbr_name $nbr_ip From d8ddfc4513c2ffdfa637e7b1c61fb8b9c571cbce Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Wed, 13 Aug 2025 02:39:06 -0700 Subject: [PATCH 33/66] [fix] update bgp policy script safely fix finalizer error in add handler change crd for gobgp config TODO set default policy check update handler run automatially after finishing add handler --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 2 +- dist/images/update-bgp-policy.sh | 98 +++++++++++++++++---- pkg/controller/gobgp_config.go | 32 ++++--- 3 files changed, 104 insertions(+), 28 deletions(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index 76605f4320b..096568412e8 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3739,7 +3739,7 @@ spec: spec: type: object required: - - bgpEdgeRouter + - bgpEdgeRouterInfo - neighbors properties: bgpEdgeRouterInfo: diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index bbf9683a923..11a65237354 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# update-bgp-policy.sh - Hybrid Version +# update-bgp-policy.sh - Hybrid Version with Smart Error Resilience # shellcheck disable=SC2086,SC2155 set -u @@ -45,6 +45,45 @@ validate_ip() { exec_cmd() { "$@" || die "failed: $*"; } +# Global array to track failed commands for retry +declare -a FAILED_COMMANDS=() + +# Error-resilient command execution - continues even on failure and tracks failures +exec_cmd_safe() { + if "$@"; then + echo "Success: $*" + return 0 + else + echo "Warning: Failed to execute '$*' (continuing...)" >&2 + # Store the failed command for potential retry + FAILED_COMMANDS+=("$*") + return 1 + fi +} + +# Retry command execution - tries twice +exec_cmd_retry() { + local max_attempts=2 + local attempt=1 + + while [[ $attempt -le $max_attempts ]]; do + if "$@"; then + echo "Success: $* (attempt $attempt)" + return 0 + else + echo "Attempt $attempt failed: $*" >&2 + ((attempt++)) + if [[ $attempt -le $max_attempts ]]; then + echo "Retrying..." >&2 + sleep 1 + fi + fi + done + + echo "All attempts failed for: $*" >&2 + return 1 +} + set_neighbor_policy() { local nbr_ip=$1; validate_ip "$nbr_ip" local prefix_in="prefix-${nbr_ip}-in" @@ -57,8 +96,8 @@ set_neighbor_policy() { echo "=== Setting policy for neighbor $nbr_ip ===" echo "-> Creating prefix-lists" - # exec_cmd $GOBGP_BIN policy prefix add $prefix_in 0.0.0.0/0 0..32 - # exec_cmd $GOBGP_BIN policy prefix add $prefix_out 0.0.0.0/0 0..32 + exec_cmd $GOBGP_BIN policy prefix add $prefix_in #0.0.0.0/0 0..32 + exec_cmd $GOBGP_BIN policy prefix add $prefix_out #0.0.0.0/0 0..32 echo "-> Defining neighbor" exec_cmd $GOBGP_BIN policy neighbor add $nbr_name $nbr_ip @@ -96,27 +135,56 @@ flush_neighbor_policy() { local policy_in="policy-${nbr_ip}-in" local policy_out="policy-${nbr_ip}-out" - echo "=== Flushing policy for neighbor $nbr_ip ===" + # Clear the failed commands array + FAILED_COMMANDS=() + + echo "=== Flushing policy for neighbor $nbr_ip (Smart Error-Resilient Mode) ===" + + # Phase 1: Remove from global policies (safe mode) echo "-> Removing from global policies" - exec_cmd $GOBGP_BIN global policy import del $policy_in - exec_cmd $GOBGP_BIN global policy export del $policy_out + exec_cmd_safe $GOBGP_BIN global policy import del $policy_in + exec_cmd_safe $GOBGP_BIN global policy export del $policy_out + # Phase 2: Remove policies (safe mode) echo "-> Removing policies" - exec_cmd $GOBGP_BIN policy del $policy_in - exec_cmd $GOBGP_BIN policy del $policy_out + exec_cmd_safe $GOBGP_BIN policy del $policy_in + exec_cmd_safe $GOBGP_BIN policy del $policy_out + # Phase 3: Remove statements (safe mode) echo "-> Removing statements" - exec_cmd $GOBGP_BIN policy statement del $stmt_in - exec_cmd $GOBGP_BIN policy statement del $stmt_out + exec_cmd_safe $GOBGP_BIN policy statement del $stmt_in + exec_cmd_safe $GOBGP_BIN policy statement del $stmt_out + # Phase 4: Remove neighbor definition (safe mode) echo "-> Removing neighbor definition" - exec_cmd $GOBGP_BIN policy neighbor del $nbr_name + exec_cmd_safe $GOBGP_BIN policy neighbor del $nbr_name + # Phase 5: Remove prefix-lists (safe mode) echo "-> Removing prefix-lists" - exec_cmd $GOBGP_BIN policy prefix del $prefix_in - exec_cmd $GOBGP_BIN policy prefix del $prefix_out + exec_cmd_safe $GOBGP_BIN policy prefix del $prefix_in + exec_cmd_safe $GOBGP_BIN policy prefix del $prefix_out - echo "=== Policy flushed successfully for $nbr_ip ===" + # Check if any commands failed and need retry + if [[ ${#FAILED_COMMANDS[@]} -eq 0 ]]; then + echo "" + echo "=== Policy flush completed successfully for $nbr_ip ===" + echo "All commands executed successfully - no retry needed" + else + echo "" + echo "=== Retry Phase: Retrying ${#FAILED_COMMANDS[@]} failed command(s) ===" + + local retry_count=0 + for cmd in "${FAILED_COMMANDS[@]}"; do + ((retry_count++)) + echo "-> Retry $retry_count/${#FAILED_COMMANDS[@]}: $cmd" + # Convert string back to array for execution + eval "exec_cmd_retry $cmd" + done + + echo "" + echo "=== Policy flush completed for $nbr_ip (with selective retry) ===" + echo "Retried ${#FAILED_COMMANDS[@]} failed command(s)" + fi } flush_prefix_in() { @@ -201,7 +269,6 @@ set_default_action() { echo "=== Default action set to $action successfully ===" } - # Execute a single command execute_single_command() { local cmd=$1; shift @@ -297,6 +364,7 @@ main() { # Single command mode (original behavior) execute_single_command "$@" fi + echo "Update bgp policy completed successfully" } main "$@" \ No newline at end of file diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index 3bbd9da058c..0e31fad51d2 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -95,14 +95,13 @@ func (c *Controller) handleAddGobgpConfig(key string) error { } klog.V(3).Infof("debug gobgp-config %s", cachedGobgpConfig.Name) - - if _, err := c.initGobgpConfigStatus(cachedGobgpConfig); err != nil { + gobgpConfig := cachedGobgpConfig.DeepCopy() + if gobgpConfig, err = c.initGobgpConfigStatus(gobgpConfig); err != nil { klog.Error(err) return err } - klog.Infof("reconciling gobgp-configuration %s", key) - gobgpConfig := cachedGobgpConfig.DeepCopy() + klog.Infof("reconciling gobgp-configuration %s for add", key) if controllerutil.AddFinalizer(gobgpConfig, util.KubeOVNControllerFinalizer) { updatedGobgpConfig, err := c.config.KubeOvnClient.KubeovnV1().GobgpConfigs(gobgpConfig.Namespace). @@ -126,7 +125,7 @@ func (c *Controller) handleAddGobgpConfig(key string) error { if len(pod.Status.PodIPs) == 0 { continue } - klog.Infof("handle adding gobgp-config %s", key) + klog.Infof("handle adding gobgp-config to pod %s", pod.Name) if err = c.execUpdateBgpPolicy(key, pod, nil, gobgpConfig); err != nil { klog.Error(err) return err @@ -168,7 +167,7 @@ func (c *Controller) handleUpdateGobgpConfig(updatedObj *updateVerGobgpConfigObj return nil } - klog.Infof("reconciling gobgp-configs %s", key) + klog.Infof("reconciling gobgp-configs %s for update", key) gobgpConfig := cachedGobgpConfig.DeepCopy() pods, err := c.validateGobgpConfig(gobgpConfig) @@ -181,7 +180,7 @@ func (c *Controller) handleUpdateGobgpConfig(updatedObj *updateVerGobgpConfigObj if len(pod.Status.PodIPs) == 0 { continue } - klog.Infof("handle adding gobgp-configs %s", key) + klog.Infof("handle adding gobgp-configs to pod %s", pod.Name) if err = c.execUpdateBgpPolicy(key, pod, updatedObj.oldVer, updatedObj.newVer); err != nil { klog.Error(err) return err @@ -284,6 +283,7 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo } if newGobgpConfig != nil { for _, neighbor := range newGobgpConfig.Spec.Neighbors { + klog.Infof("new bgp config neighbor %v", neighbor) nbrIP := neighbor.Address if len(nbrIP) == 0 { klog.Warningf("neighbor address is empty for gobgp-config %s", key) @@ -293,12 +293,14 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo // toAdvertise advMode := neighbor.ToAdvertise.Allowed.Mode - advPrefixes := neighbor.ToAdvertise.Allowed.Prefixes - quoted := make([]string, len(advPrefixes)) - + var advPrefixes []string if advMode == "all" { advPrefixes = []string{"0.0.0.0/0 0..32"} + } else { + advPrefixes = neighbor.ToAdvertise.Allowed.Prefixes } + quoted := make([]string, len(advPrefixes)) + for i, p := range advPrefixes { quoted[i] = fmt.Sprintf("\"%s\"", p) } @@ -306,9 +308,11 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo // toReceive recvMode := neighbor.ToReceive.Allowed.Mode - recvPrefixes := neighbor.ToReceive.Allowed.Prefixes + var recvPrefixes []string if recvMode == "all" { recvPrefixes = []string{"0.0.0.0/0 0..32"} + } else { + recvPrefixes = neighbor.ToReceive.Allowed.Prefixes } quoted = make([]string, len(recvPrefixes)) for i, p := range recvPrefixes { @@ -430,11 +434,15 @@ func (c *Controller) execCmd(pod *corev1.Pod, cmdArs []string) error { return err } + cmdSuccess := false if len(stdOutput) > 0 { klog.Infof("ExecuteCommandInContainer stdOutput: %v", stdOutput) + if strings.Contains(stdOutput, "Update bgp policy completed successfully") { + cmdSuccess = true + } } - if len(errOutput) > 0 { + if len(errOutput) > 0 && !cmdSuccess { klog.Errorf("failed to ExecuteCommandInContainer errOutput: %v", errOutput) return errors.New(errOutput) } From 7081a662b6cfe41b1cf1bc2051e1322859234086 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Wed, 13 Aug 2025 23:25:28 -0700 Subject: [PATCH 34/66] [fix] gobgp config enque update logic --- pkg/controller/gobgp_config.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index 0e31fad51d2..fda29194c2f 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "reflect" "slices" "strings" @@ -42,13 +43,19 @@ func (c *Controller) enqueueUpdateGobgpConfig(oldObj, newObj any) { oldGobgpConfig := oldObj.(*kubeovnv1.GobgpConfig) newGobgpConfig := newObj.(*kubeovnv1.GobgpConfig) + updateConfigVer := &updateVerGobgpConfigObject{ key: key, oldVer: oldGobgpConfig, newVer: newGobgpConfig, } - c.updateGobgpConfigQueue.Add(updateConfigVer) + if !reflect.DeepEqual(oldGobgpConfig.Spec, newGobgpConfig.Spec) { + klog.Infof("enqueue update gobgp-config %s", key) + c.updateGobgpConfigQueue.Add(updateConfigVer) + } + + // c.updateGobgpConfigQueue.Add(updateConfigVer) } func (c *Controller) enqueueDeleteGobgpConfig(obj any) { @@ -168,6 +175,8 @@ func (c *Controller) handleUpdateGobgpConfig(updatedObj *updateVerGobgpConfigObj } klog.Infof("reconciling gobgp-configs %s for update", key) + klog.Infof("debug gobgp-config old version : %v", updatedObj.oldVer) + klog.Infof("debug gobgp-config new version : %v", updatedObj.newVer) gobgpConfig := cachedGobgpConfig.DeepCopy() pods, err := c.validateGobgpConfig(gobgpConfig) From 36486e4b6c322cfe4d6f90b2b67c6e4bd5771848 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Thu, 14 Aug 2025 00:20:26 -0700 Subject: [PATCH 35/66] [feat] update-bgp-policy script soft reset for applying policy right away gobgp config delete logic and add default action --- dist/images/update-bgp-policy.sh | 12 ++++++++++-- pkg/controller/gobgp_config.go | 13 +++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index 11a65237354..50cbf5b9da4 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -164,6 +164,10 @@ flush_neighbor_policy() { exec_cmd_safe $GOBGP_BIN policy prefix del $prefix_in exec_cmd_safe $GOBGP_BIN policy prefix del $prefix_out + # Phase 6: Apply policy to neighbor (safe mode) + echo "-> Soft reset neighbor policy" + exec_cmd_safe $GOBGP_BIN neighbor $nbr_ip softreset + # Check if any commands failed and need retry if [[ ${#FAILED_COMMANDS[@]} -eq 0 ]]; then echo "" @@ -198,7 +202,7 @@ flush_prefix_in() { echo "-> Deleting: $iprange $mask" exec_cmd $GOBGP_BIN policy prefix del $prefix_name $iprange $mask done - + exec_cmd $GOBGP_BIN neighbor $nbr_ip softresetin echo "=== All entries removed from $prefix_name ===" } @@ -213,7 +217,7 @@ flush_prefix_out() { echo "-> Deleting: $iprange $mask" exec_cmd $GOBGP_BIN policy prefix del $prefix_name $iprange $mask done - + exec_cmd $GOBGP_BIN neighbor $nbr_ip softresetout echo "=== All entries removed from $prefix_name ===" } @@ -245,6 +249,7 @@ add_prefix() { exec_cmd $GOBGP_BIN policy prefix add $prefix_name $entry fi done + exec_cmd $GOBGP_BIN neighbor $nbr_ip softreset$dir echo "=== Done ===" } @@ -266,6 +271,9 @@ set_default_action() { echo "-> Applying default policy to global export" exec_cmd $GOBGP_BIN global policy export add default $action + echo "-> Soft reset neighbor policy" + exec_cmd $GOBGP_BIN neighbor $nbr_ip softreset + echo "=== Default action set to $action successfully ===" } diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index fda29194c2f..17eb38d6835 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -50,6 +50,11 @@ func (c *Controller) enqueueUpdateGobgpConfig(oldObj, newObj any) { newVer: newGobgpConfig, } + if !newGobgpConfig.DeletionTimestamp.IsZero() { + c.deleteGobgpConfigQueue.Add(key) + return + } + if !reflect.DeepEqual(oldGobgpConfig.Spec, newGobgpConfig.Spec) { klog.Infof("enqueue update gobgp-config %s", key) c.updateGobgpConfigQueue.Add(updateConfigVer) @@ -289,6 +294,10 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo // } // cmdArs = append(cmdArs, append([]string{"--", "flush-prefix-in"}, rcvPrefixes...)...) } + } else { + // if oldGobgpConfig is nil, it means this is the first time to update the bgp policy + // so we need to set default action to reject + cmdArs = append(cmdArs, "--", "set-default-action", "reject") } if newGobgpConfig != nil { for _, neighbor := range newGobgpConfig.Spec.Neighbors { @@ -329,6 +338,10 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo } cmdArs = append(cmdArs, "--", "add-prefix", "in", nbrIP, strings.Join(quoted, ",")) } + } else { + // if newGobgpConfig is nil, it means the bgp policy is deleted + // so we need to set default action to accept + cmdArs = append(cmdArs, "--", "set-default-action", "accept") } // cmdArs = append(cmdArs, "list_announced_route") From bf8ad3d9c1aa7b69ba4bd97818184000cdd78561 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Tue, 12 Aug 2025 01:26:22 -0700 Subject: [PATCH 36/66] [fix] delete default prefix in bgp config script --- dist/images/update-bgp-policy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index 03b7970fe95..df4db997e50 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -57,8 +57,8 @@ set_neighbor_policy() { echo "=== Setting policy for neighbor $nbr_ip ===" echo "-> Creating prefix-lists" - # exec_cmd $GOBGP_BIN policy prefix add $prefix_in 0.0.0.0/0 0..32 - # exec_cmd $GOBGP_BIN policy prefix add $prefix_out 0.0.0.0/0 0..32 + exec_cmd $GOBGP_BIN policy prefix add $prefix_in 0.0.0.0/0 0..32 + exec_cmd $GOBGP_BIN policy prefix add $prefix_out 0.0.0.0/0 0..32 echo "-> Defining neighbor" exec_cmd $GOBGP_BIN policy neighbor add $nbr_name $nbr_ip From ca81b30b47776efeb4b92f166a6dac4f2e7f5d2d Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Wed, 13 Aug 2025 02:39:06 -0700 Subject: [PATCH 37/66] [fix] update bgp policy script safely fix finalizer error in add handler change crd for gobgp config TODO set default policy check update handler run automatially after finishing add handler --- dist/images/update-bgp-policy.sh | 98 +++++++++++++++++++++++++++----- pkg/controller/gobgp_config.go | 32 +++++++---- 2 files changed, 103 insertions(+), 27 deletions(-) diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index df4db997e50..11a65237354 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# update-bgp-policy.sh - Hybrid Version +# update-bgp-policy.sh - Hybrid Version with Smart Error Resilience # shellcheck disable=SC2086,SC2155 set -u @@ -45,6 +45,45 @@ validate_ip() { exec_cmd() { "$@" || die "failed: $*"; } +# Global array to track failed commands for retry +declare -a FAILED_COMMANDS=() + +# Error-resilient command execution - continues even on failure and tracks failures +exec_cmd_safe() { + if "$@"; then + echo "Success: $*" + return 0 + else + echo "Warning: Failed to execute '$*' (continuing...)" >&2 + # Store the failed command for potential retry + FAILED_COMMANDS+=("$*") + return 1 + fi +} + +# Retry command execution - tries twice +exec_cmd_retry() { + local max_attempts=2 + local attempt=1 + + while [[ $attempt -le $max_attempts ]]; do + if "$@"; then + echo "Success: $* (attempt $attempt)" + return 0 + else + echo "Attempt $attempt failed: $*" >&2 + ((attempt++)) + if [[ $attempt -le $max_attempts ]]; then + echo "Retrying..." >&2 + sleep 1 + fi + fi + done + + echo "All attempts failed for: $*" >&2 + return 1 +} + set_neighbor_policy() { local nbr_ip=$1; validate_ip "$nbr_ip" local prefix_in="prefix-${nbr_ip}-in" @@ -57,8 +96,8 @@ set_neighbor_policy() { echo "=== Setting policy for neighbor $nbr_ip ===" echo "-> Creating prefix-lists" - exec_cmd $GOBGP_BIN policy prefix add $prefix_in 0.0.0.0/0 0..32 - exec_cmd $GOBGP_BIN policy prefix add $prefix_out 0.0.0.0/0 0..32 + exec_cmd $GOBGP_BIN policy prefix add $prefix_in #0.0.0.0/0 0..32 + exec_cmd $GOBGP_BIN policy prefix add $prefix_out #0.0.0.0/0 0..32 echo "-> Defining neighbor" exec_cmd $GOBGP_BIN policy neighbor add $nbr_name $nbr_ip @@ -96,27 +135,56 @@ flush_neighbor_policy() { local policy_in="policy-${nbr_ip}-in" local policy_out="policy-${nbr_ip}-out" - echo "=== Flushing policy for neighbor $nbr_ip ===" + # Clear the failed commands array + FAILED_COMMANDS=() + + echo "=== Flushing policy for neighbor $nbr_ip (Smart Error-Resilient Mode) ===" + + # Phase 1: Remove from global policies (safe mode) echo "-> Removing from global policies" - exec_cmd $GOBGP_BIN global policy import del $policy_in - exec_cmd $GOBGP_BIN global policy export del $policy_out + exec_cmd_safe $GOBGP_BIN global policy import del $policy_in + exec_cmd_safe $GOBGP_BIN global policy export del $policy_out + # Phase 2: Remove policies (safe mode) echo "-> Removing policies" - exec_cmd $GOBGP_BIN policy del $policy_in - exec_cmd $GOBGP_BIN policy del $policy_out + exec_cmd_safe $GOBGP_BIN policy del $policy_in + exec_cmd_safe $GOBGP_BIN policy del $policy_out + # Phase 3: Remove statements (safe mode) echo "-> Removing statements" - exec_cmd $GOBGP_BIN policy statement del $stmt_in - exec_cmd $GOBGP_BIN policy statement del $stmt_out + exec_cmd_safe $GOBGP_BIN policy statement del $stmt_in + exec_cmd_safe $GOBGP_BIN policy statement del $stmt_out + # Phase 4: Remove neighbor definition (safe mode) echo "-> Removing neighbor definition" - exec_cmd $GOBGP_BIN policy neighbor del $nbr_name + exec_cmd_safe $GOBGP_BIN policy neighbor del $nbr_name + # Phase 5: Remove prefix-lists (safe mode) echo "-> Removing prefix-lists" - exec_cmd $GOBGP_BIN policy prefix del $prefix_in - exec_cmd $GOBGP_BIN policy prefix del $prefix_out + exec_cmd_safe $GOBGP_BIN policy prefix del $prefix_in + exec_cmd_safe $GOBGP_BIN policy prefix del $prefix_out - echo "=== Policy flushed successfully for $nbr_ip ===" + # Check if any commands failed and need retry + if [[ ${#FAILED_COMMANDS[@]} -eq 0 ]]; then + echo "" + echo "=== Policy flush completed successfully for $nbr_ip ===" + echo "All commands executed successfully - no retry needed" + else + echo "" + echo "=== Retry Phase: Retrying ${#FAILED_COMMANDS[@]} failed command(s) ===" + + local retry_count=0 + for cmd in "${FAILED_COMMANDS[@]}"; do + ((retry_count++)) + echo "-> Retry $retry_count/${#FAILED_COMMANDS[@]}: $cmd" + # Convert string back to array for execution + eval "exec_cmd_retry $cmd" + done + + echo "" + echo "=== Policy flush completed for $nbr_ip (with selective retry) ===" + echo "Retried ${#FAILED_COMMANDS[@]} failed command(s)" + fi } flush_prefix_in() { @@ -201,7 +269,6 @@ set_default_action() { echo "=== Default action set to $action successfully ===" } - # Execute a single command execute_single_command() { local cmd=$1; shift @@ -297,6 +364,7 @@ main() { # Single command mode (original behavior) execute_single_command "$@" fi + echo "Update bgp policy completed successfully" } main "$@" \ No newline at end of file diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index 3bbd9da058c..0e31fad51d2 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -95,14 +95,13 @@ func (c *Controller) handleAddGobgpConfig(key string) error { } klog.V(3).Infof("debug gobgp-config %s", cachedGobgpConfig.Name) - - if _, err := c.initGobgpConfigStatus(cachedGobgpConfig); err != nil { + gobgpConfig := cachedGobgpConfig.DeepCopy() + if gobgpConfig, err = c.initGobgpConfigStatus(gobgpConfig); err != nil { klog.Error(err) return err } - klog.Infof("reconciling gobgp-configuration %s", key) - gobgpConfig := cachedGobgpConfig.DeepCopy() + klog.Infof("reconciling gobgp-configuration %s for add", key) if controllerutil.AddFinalizer(gobgpConfig, util.KubeOVNControllerFinalizer) { updatedGobgpConfig, err := c.config.KubeOvnClient.KubeovnV1().GobgpConfigs(gobgpConfig.Namespace). @@ -126,7 +125,7 @@ func (c *Controller) handleAddGobgpConfig(key string) error { if len(pod.Status.PodIPs) == 0 { continue } - klog.Infof("handle adding gobgp-config %s", key) + klog.Infof("handle adding gobgp-config to pod %s", pod.Name) if err = c.execUpdateBgpPolicy(key, pod, nil, gobgpConfig); err != nil { klog.Error(err) return err @@ -168,7 +167,7 @@ func (c *Controller) handleUpdateGobgpConfig(updatedObj *updateVerGobgpConfigObj return nil } - klog.Infof("reconciling gobgp-configs %s", key) + klog.Infof("reconciling gobgp-configs %s for update", key) gobgpConfig := cachedGobgpConfig.DeepCopy() pods, err := c.validateGobgpConfig(gobgpConfig) @@ -181,7 +180,7 @@ func (c *Controller) handleUpdateGobgpConfig(updatedObj *updateVerGobgpConfigObj if len(pod.Status.PodIPs) == 0 { continue } - klog.Infof("handle adding gobgp-configs %s", key) + klog.Infof("handle adding gobgp-configs to pod %s", pod.Name) if err = c.execUpdateBgpPolicy(key, pod, updatedObj.oldVer, updatedObj.newVer); err != nil { klog.Error(err) return err @@ -284,6 +283,7 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo } if newGobgpConfig != nil { for _, neighbor := range newGobgpConfig.Spec.Neighbors { + klog.Infof("new bgp config neighbor %v", neighbor) nbrIP := neighbor.Address if len(nbrIP) == 0 { klog.Warningf("neighbor address is empty for gobgp-config %s", key) @@ -293,12 +293,14 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo // toAdvertise advMode := neighbor.ToAdvertise.Allowed.Mode - advPrefixes := neighbor.ToAdvertise.Allowed.Prefixes - quoted := make([]string, len(advPrefixes)) - + var advPrefixes []string if advMode == "all" { advPrefixes = []string{"0.0.0.0/0 0..32"} + } else { + advPrefixes = neighbor.ToAdvertise.Allowed.Prefixes } + quoted := make([]string, len(advPrefixes)) + for i, p := range advPrefixes { quoted[i] = fmt.Sprintf("\"%s\"", p) } @@ -306,9 +308,11 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo // toReceive recvMode := neighbor.ToReceive.Allowed.Mode - recvPrefixes := neighbor.ToReceive.Allowed.Prefixes + var recvPrefixes []string if recvMode == "all" { recvPrefixes = []string{"0.0.0.0/0 0..32"} + } else { + recvPrefixes = neighbor.ToReceive.Allowed.Prefixes } quoted = make([]string, len(recvPrefixes)) for i, p := range recvPrefixes { @@ -430,11 +434,15 @@ func (c *Controller) execCmd(pod *corev1.Pod, cmdArs []string) error { return err } + cmdSuccess := false if len(stdOutput) > 0 { klog.Infof("ExecuteCommandInContainer stdOutput: %v", stdOutput) + if strings.Contains(stdOutput, "Update bgp policy completed successfully") { + cmdSuccess = true + } } - if len(errOutput) > 0 { + if len(errOutput) > 0 && !cmdSuccess { klog.Errorf("failed to ExecuteCommandInContainer errOutput: %v", errOutput) return errors.New(errOutput) } From 018a0704b44cfaf8f2d0da4704cfc514d24aab23 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Wed, 13 Aug 2025 23:25:28 -0700 Subject: [PATCH 38/66] [fix] gobgp config enque update logic --- pkg/controller/gobgp_config.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index 0e31fad51d2..fda29194c2f 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "reflect" "slices" "strings" @@ -42,13 +43,19 @@ func (c *Controller) enqueueUpdateGobgpConfig(oldObj, newObj any) { oldGobgpConfig := oldObj.(*kubeovnv1.GobgpConfig) newGobgpConfig := newObj.(*kubeovnv1.GobgpConfig) + updateConfigVer := &updateVerGobgpConfigObject{ key: key, oldVer: oldGobgpConfig, newVer: newGobgpConfig, } - c.updateGobgpConfigQueue.Add(updateConfigVer) + if !reflect.DeepEqual(oldGobgpConfig.Spec, newGobgpConfig.Spec) { + klog.Infof("enqueue update gobgp-config %s", key) + c.updateGobgpConfigQueue.Add(updateConfigVer) + } + + // c.updateGobgpConfigQueue.Add(updateConfigVer) } func (c *Controller) enqueueDeleteGobgpConfig(obj any) { @@ -168,6 +175,8 @@ func (c *Controller) handleUpdateGobgpConfig(updatedObj *updateVerGobgpConfigObj } klog.Infof("reconciling gobgp-configs %s for update", key) + klog.Infof("debug gobgp-config old version : %v", updatedObj.oldVer) + klog.Infof("debug gobgp-config new version : %v", updatedObj.newVer) gobgpConfig := cachedGobgpConfig.DeepCopy() pods, err := c.validateGobgpConfig(gobgpConfig) From 8f8971c94b010b9661bf36c96dc343d6b5228ad9 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Thu, 14 Aug 2025 00:20:26 -0700 Subject: [PATCH 39/66] [feat] update-bgp-policy script soft reset for applying policy right away gobgp config delete logic and add default action --- dist/images/update-bgp-policy.sh | 12 ++++++++++-- pkg/controller/gobgp_config.go | 13 +++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index 11a65237354..50cbf5b9da4 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -164,6 +164,10 @@ flush_neighbor_policy() { exec_cmd_safe $GOBGP_BIN policy prefix del $prefix_in exec_cmd_safe $GOBGP_BIN policy prefix del $prefix_out + # Phase 6: Apply policy to neighbor (safe mode) + echo "-> Soft reset neighbor policy" + exec_cmd_safe $GOBGP_BIN neighbor $nbr_ip softreset + # Check if any commands failed and need retry if [[ ${#FAILED_COMMANDS[@]} -eq 0 ]]; then echo "" @@ -198,7 +202,7 @@ flush_prefix_in() { echo "-> Deleting: $iprange $mask" exec_cmd $GOBGP_BIN policy prefix del $prefix_name $iprange $mask done - + exec_cmd $GOBGP_BIN neighbor $nbr_ip softresetin echo "=== All entries removed from $prefix_name ===" } @@ -213,7 +217,7 @@ flush_prefix_out() { echo "-> Deleting: $iprange $mask" exec_cmd $GOBGP_BIN policy prefix del $prefix_name $iprange $mask done - + exec_cmd $GOBGP_BIN neighbor $nbr_ip softresetout echo "=== All entries removed from $prefix_name ===" } @@ -245,6 +249,7 @@ add_prefix() { exec_cmd $GOBGP_BIN policy prefix add $prefix_name $entry fi done + exec_cmd $GOBGP_BIN neighbor $nbr_ip softreset$dir echo "=== Done ===" } @@ -266,6 +271,9 @@ set_default_action() { echo "-> Applying default policy to global export" exec_cmd $GOBGP_BIN global policy export add default $action + echo "-> Soft reset neighbor policy" + exec_cmd $GOBGP_BIN neighbor $nbr_ip softreset + echo "=== Default action set to $action successfully ===" } diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index fda29194c2f..17eb38d6835 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -50,6 +50,11 @@ func (c *Controller) enqueueUpdateGobgpConfig(oldObj, newObj any) { newVer: newGobgpConfig, } + if !newGobgpConfig.DeletionTimestamp.IsZero() { + c.deleteGobgpConfigQueue.Add(key) + return + } + if !reflect.DeepEqual(oldGobgpConfig.Spec, newGobgpConfig.Spec) { klog.Infof("enqueue update gobgp-config %s", key) c.updateGobgpConfigQueue.Add(updateConfigVer) @@ -289,6 +294,10 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo // } // cmdArs = append(cmdArs, append([]string{"--", "flush-prefix-in"}, rcvPrefixes...)...) } + } else { + // if oldGobgpConfig is nil, it means this is the first time to update the bgp policy + // so we need to set default action to reject + cmdArs = append(cmdArs, "--", "set-default-action", "reject") } if newGobgpConfig != nil { for _, neighbor := range newGobgpConfig.Spec.Neighbors { @@ -329,6 +338,10 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo } cmdArs = append(cmdArs, "--", "add-prefix", "in", nbrIP, strings.Join(quoted, ",")) } + } else { + // if newGobgpConfig is nil, it means the bgp policy is deleted + // so we need to set default action to accept + cmdArs = append(cmdArs, "--", "set-default-action", "accept") } // cmdArs = append(cmdArs, "list_announced_route") From ba01158e1753819661f8ca413bc000d5fba01277 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Sun, 17 Aug 2025 19:28:42 -0700 Subject: [PATCH 40/66] set_default_action nbr_ip parameter error is resovled. --- dist/images/update-bgp-policy.sh | 4 +- pkg/controller/controller.go | 1 + pkg/controller/gobgp_config.go | 66 +++++++++++++++++++++++++++----- 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index 50cbf5b9da4..a5db5e72d37 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -271,8 +271,8 @@ set_default_action() { echo "-> Applying default policy to global export" exec_cmd $GOBGP_BIN global policy export add default $action - echo "-> Soft reset neighbor policy" - exec_cmd $GOBGP_BIN neighbor $nbr_ip softreset + # echo "-> Soft reset neighbor policy" + # exec_cmd $GOBGP_BIN neighbor $nbr_ip softreset echo "=== Default action set to $action successfully ===" } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 6d05280ea8b..214d2e95fff 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -1298,6 +1298,7 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(runWorker("update bgp edge router advertisement", c.updateBgpEdgeRouterAdvertisementQueue, c.handleUpdateBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) go wait.Until(runWorker("delete bgp edge router advertisement", c.deleteBgpEdgeRouterAdvertisementQueue, c.handleDelBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) go wait.Until(c.resyncBgpRules, 60*time.Second, ctx.Done()) + go wait.Until(c.resyncBgpPolicyRules, 60*time.Second, ctx.Done()) go wait.Until(runWorker("add bgp edge router advertisement", c.addGobgpConfigQueue, c.handleAddGobgpConfig), time.Second, ctx.Done()) go wait.Until(runWorker("update bgp edge router advertisement", c.updateGobgpConfigQueue, c.handleUpdateGobgpConfig), time.Second, ctx.Done()) go wait.Until(runWorker("delete bgp edge router advertisement", c.deleteGobgpConfigQueue, c.handleDelGobgpConfig), time.Second, ctx.Done()) diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index 17eb38d6835..a17accebdb4 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -11,6 +11,7 @@ import ( corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" @@ -271,7 +272,6 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo klog.Error(err) return err } - klog.Infof("execUpdateBgpPolicy %s", key) cmdArs := []string{} if oldGobgpConfig != nil { @@ -285,14 +285,6 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo } // erase neighbor. cmdArs = append(cmdArs, "--", "flush-neighbor-policy", nbrIP) - // cmdArs = append(cmdArs, "--", "flush-prefix-out", nbrIP) - // cmdArs = append(cmdArs, "--", "flush-prefix-in", nbrIP) - // rcvMode := neighbor.ToReceive.Allowed.Mode - // rcvPrefixes := neighbor.ToReceive.Allowed.Prefixes - // if rcvMode == "all" { - // rcvPrefixes = []string{"0.0.0.0/0 0..32"} - // } - // cmdArs = append(cmdArs, append([]string{"--", "flush-prefix-in"}, rcvPrefixes...)...) } } else { // if oldGobgpConfig is nil, it means this is the first time to update the bgp policy @@ -344,11 +336,11 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo cmdArs = append(cmdArs, "--", "set-default-action", "accept") } - // cmdArs = append(cmdArs, "list_announced_route") if err := c.execCmd(pod, cmdArs); err != nil { klog.Error(err) return err } + return nil } @@ -475,3 +467,57 @@ func (c *Controller) execCmd(pod *corev1.Pod, cmdArs []string) error { func containsNeighbor(neighbors []string, address string) bool { return slices.Contains(neighbors, address) } + +func (c *Controller) resyncBgpPolicyRules() { + klog.Info("resync bgp edge router") + // resync all bgp edge routers + gobgpConfigs, err := c.gobgpConfigLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list bgp edge routers: %v", err) + return + } + + for _, gobgpConfig := range gobgpConfigs { + // Check router.Spec.BGP.AdvertisedRoutes same with pods bgp advertised routes + if err := c.syncGobgpPolicy(gobgpConfig); err != nil { + klog.Errorf("failed to sync advertised routes for bgp edge router %s: %v", gobgpConfig.Name, err) + continue + } + klog.Infof("resync bgp edge router %s", gobgpConfig.Name) + } +} + +func (c *Controller) syncGobgpPolicy(gobgpConfig *kubeovnv1.GobgpConfig) error { + key := cache.MetaObjectToName(gobgpConfig).String() + + c.gobgpConfigKeyMutex.LockKey(key) + defer func() { _ = c.gobgpConfigKeyMutex.UnlockKey(key) }() + + if !gobgpConfig.DeletionTimestamp.IsZero() { + c.deleteGobgpConfigQueue.Add(key) + return nil + } + klog.Infof("reconciling bgp-edge-router %s", key) + // Deep copy because we might mutate Status below. + cachedGobgpConfig := gobgpConfig.DeepCopy() + + pods, err := c.validateGobgpConfig(cachedGobgpConfig) + if err != nil || pods == nil { + klog.Error(err) + return err + } + + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + err = c.execUpdateBgpPolicy(key, pod, cachedGobgpConfig, cachedGobgpConfig) + if err != nil { + return err + } + klog.Infof("router pod %s/%s policy: %v", pod.Namespace, pod.Name, gobgpConfig) + } + + klog.Infof("finished sync bgp-edge-router %s advertised routes", key) + return nil +} From e9f6e5b28a1776697fed7c33e7327109f84205e3 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Mon, 18 Aug 2025 00:19:34 -0700 Subject: [PATCH 41/66] For e2e test Haven't tested yet --- dist/images/install.sh | 694 ++++++++++++++++++++++++++ e2e.mk | 10 + pkg/controller/gc.go | 49 ++ test/e2e/edge-router/e2e_test.go | 559 +++++++++++++++++++++ test/e2e/framework/bgp-edge-router.go | 208 ++++++++ 5 files changed, 1520 insertions(+) create mode 100644 test/e2e/edge-router/e2e_test.go create mode 100644 test/e2e/framework/bgp-edge-router.go diff --git a/dist/images/install.sh b/dist/images/install.sh index 676f4236abd..76f90ef8450 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -3359,6 +3359,694 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgp-edge-routers.kubeovn.io +spec: + group: kubeovn.io + names: + plural: bgp-edge-routers + singular: bgp-edge-router + shortNames: + - bgp-er + - ber + kind: BgpEdgeRouter + listKind: BgpEdgeRouterList + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vpc + name: VPC + type: string + - jsonPath: .spec.replicas + name: REPLICAS + type: integer + - jsonPath: .spec.bfd.enabled + name: BFD ENABLED + type: boolean + - jsonPath: .spec.externalSubnet + name: EXTERNAL SUBNET + type: string + - jsonPath: .status.phase + name: PHASE + type: string + - jsonPath: .status.ready + name: READY + type: boolean + - jsonPath: .status.internalIPs + name: INTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.externalIPs + name: EXTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.workload.nodes + name: WORKING NODES + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + scale: + # specReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Spec.Replicas. + specReplicasPath: .spec.replicas + # statusReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Replicas. + statusReplicasPath: .status.replicas + # labelSelectorPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Selector. + labelSelectorPath: .status.labelSelector + schema: + openAPIV3Schema: + type: object + properties: + status: + properties: + replicas: + type: integer + format: int32 + labelSelector: + type: string + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + internalIPs: + items: + type: string + type: array + externalIPs: + items: + type: string + type: array + phase: + type: string + default: Pending + enum: + - Pending + - Processing + - Completed + ready: + type: boolean + default: false + workload: + type: object + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + nodes: + type: array + items: + type: string + required: + - conditions + - phase + type: object + spec: + type: object + required: + - externalSubnet + x-kubernetes-validations: + - rule: "!has(self.internalIPs) || size(self.internalIPs) == 0 || size(self.internalIPs) >= self.replicas" + message: 'Size of Internal IPs MUST be equal to or greater than Replicas' + fieldPath: ".internalIPs" + - rule: "!has(self.externalIPs) || size(self.externalIPs) == 0 || size(self.externalIPs) >= self.replicas" + message: 'Size of External IPs MUST be equal to or greater than Replicas' + fieldPath: ".externalIPs" + - rule: "size(self.policies) != 0 || size(self.selectors) != 0" + message: 'Each BGP Edeg Router MUST have at least one policy or selector' + properties: + replicas: + type: integer + format: int32 + default: 1 + minimum: 0 + maximum: 10 + prefix: + type: string + anyOf: + - pattern: ^$ + - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*[-\.]?$ + x-kubernetes-validations: + - rule: "self == oldSelf" + message: "This field is immutable." + vpc: + type: string + internalSubnet: + type: string + externalSubnet: + type: string + internalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + externalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + image: + type: string + bfd: + type: object + properties: + enabled: + type: boolean + default: false + minRX: + type: integer + format: int32 + default: 1000 + minTX: + type: integer + format: int32 + default: 1000 + multiplier: + type: integer + format: int32 + default: 3 + selectors: + type: array + items: + type: object + properties: + namespaceSelector: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + x-kubernetes-validations: + - rule: "size(self.matchLabels) != 0 || size(self.matchExpressions) != 0" + message: 'Each namespace selector MUST have at least one matchLabels or matchExpressions' + podSelector: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + x-kubernetes-validations: + - rule: "size(self.matchLabels) != 0 || size(self.matchExpressions) != 0" + message: 'Each pod selector MUST have at least one matchLabels or matchExpressions' + policies: + type: array + items: + type: object + properties: + snat: + type: boolean + default: false + ipBlocks: + type: array + x-kubernetes-list-type: set + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + - format: cidr + subnets: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + x-kubernetes-validations: + - rule: "size(self.ipBlocks) != 0 || size(self.subnets) != 0" + message: 'Each policy MUST have at least one ipBlock or subnet' + trafficPolicy: + type: string + enum: + - Local + - Cluster + default: Cluster + nodeSelector: + type: array + items: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator + bgp: + type: object + properties: + edgeRouterMode: + type: boolean + default: false + enabled: + type: boolean + default: false + image: + type: string + asn: + type: integer + format: int32 + minimum: 0 + remoteAsn: + type: integer + format: int32 + minimum: 0 + neighbors: + type: array + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + holdTime: + type: string + pattern: ^[0-9]+[smhd]$ + routerId: + type: string + # The routerId can be an IPv4 or IPv6 address, but we are not enforcing it here + # routerId could be nullable, so that we can use environment value to set it + # refer pkg/speaker/config.go#L186 + anyOf: + - format: ipv4 + - format: ipv6 + - pattern: '^$' + password: + type: string + enableGracefulRestart: + type: boolean + default: false + extraArgs: + type: array + items: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgp-edge-router-advertisements.kubeovn.io +spec: + group: kubeovn.io + names: + plural: bgp-edge-router-advertisements + singular: bgp-edge-router-advertisement + shortNames: + - ber-ad + kind: BgpEdgeRouterAdvertisement + listKind: BgpEdgeRouterAdvertisementList + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.subnet + name: SUBNET + type: string + - jsonPath: .spec.bgpEdgeRouter + name: BGP-EDGE-ROUTER + type: string + - jsonPath: .status.ready + name: READY + type: boolean + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + subnet: + type: array + minItems: 1 + x-kubernetes-list-type: set + items: + type: string + bgpEdgeRouter: + type: string + required: + - subnet + - bgpEdgeRouter + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + ready: + type: boolean + default: false + required: + - conditions + type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gobgp-configs.kubeovn.io +spec: + group: kubeovn.io + names: + plural: gobgp-configs + singular: gobgp-config + shortNames: + - bgp-config + kind: GobgpConfig + listKind: GobgpConfigList + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: bgp-edge-router + type: string + jsonPath: .spec.bgpEdgeRouterInfo.name + - name: neighbor-address + type: string + jsonPath: .spec.neighbors[*].address + - name: toReceive-mode + type: string + jsonPath: .spec.neighbors[*].toReceive.allowed.mode + - name: toReceive-prefixes + type: string + jsonPath: .spec.neighbors[*].toReceive.allowed.prefixes + - name: toAdvertise-mode + type: string + jsonPath: .spec.neighbors[*].toAdvertise.allowed.mode + - name: toAdvertise-prefixes + type: string + jsonPath: .spec.neighbors[*].toAdvertise.allowed.prefixes + - name: READY + type: boolean + jsonPath: .status.ready + - name: AGE + type: date + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + required: + - bgpEdgeRouterInfo + - neighbors + properties: + bgpEdgeRouterInfo: + type: object + properties: + name: + type: string + namespace: + type: string + neighbors: + type: array + items: + type: object + required: + - address + - toAdvertise + - toReceive + properties: + address: + type: string + pattern: '^(([0-9]{1,3}\.){3}[0-9]{1,3}|[0-9a-fA-F:]+)$' + toAdvertise: + type: object + required: + - allowed + properties: + allowed: + type: object + required: + - mode + properties: + mode: + type: string + prefixes: + type: array + x-kubernetes-list-type: set + items: + type: string + pattern: '^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/\d{1,2})?|([0-9a-fA-F:]+(\/\d{1,3})?))$' + x-kubernetes-validations: + - rule: "self.mode != 'filtered' || (has(self.prefixes) && size(self.prefixes) > 0)" + message: "prefixes must be set and non-empty when mode is 'filtered'" + toReceive: + type: object + required: + - allowed + properties: + allowed: + type: object + required: + - mode + properties: + mode: + type: string + prefixes: + type: array + x-kubernetes-list-type: set + items: + type: string + pattern: '^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/\d{1,2})?|([0-9a-fA-F:]+(\/\d{1,3})?))$' + x-kubernetes-validations: + - rule: "self.mode != 'filtered' || (has(self.prefixes) && size(self.prefixes) > 0)" + message: "prefixes must be set and non-empty when mode is 'filtered'" + status: + type: object + properties: + ready: + type: boolean + default: false + conditions: + type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - type + items: + type: object + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + properties: + type: + type: string + maxLength: 316 + pattern: '^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$' + status: + type: string + enum: + - "True" + - "False" + - "Unknown" + reason: + type: string + minLength: 1 + maxLength: 1024 + pattern: '^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$' + message: + type: string + maxLength: 32768 + lastTransitionTime: + type: string + format: date-time + lastUpdateTime: + type: string + format: date-time + observedGeneration: + type: integer + format: int64 + minimum: 0 EOF cat < ovn-ovs-sa.yaml @@ -3471,6 +4159,12 @@ rules: - vpc-dnses/status - qos-policies - qos-policies/status + - bgp-edge-routers + - bgp-edge-routers/status + - bgp-edge-router-advertisements + - bgp-edge-router-advertisements/status + - gobgp-configs + - gobgp-configs/status verbs: - "*" - apiGroups: diff --git a/e2e.mk b/e2e.mk index f5ea1819559..92797ee3e4f 100644 --- a/e2e.mk +++ b/e2e.mk @@ -91,6 +91,7 @@ e2e-build: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/connectivity ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/metallb ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ipsec-cert-mgr + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/edge-router .PHONY: k8s-conformance-e2e k8s-conformance-e2e: @@ -278,3 +279,12 @@ kube-ovn-underlay-metallb-e2e: E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ --focus=CNI:Kube-OVN ./test/e2e/metallb/metallb.test -- $(TEST_BIN_ARGS) + +.PHONY: kube-ovn-edge-router-e2e +kube-ovn-edge-router-e2e: + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/edge-router + E2E_BRANCH=$(E2E_BRANCH) \ + E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ + E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ + ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ + --focus=CNI:Kube-OVN ./test/e2e/edge-router/edge-router.test -- $(TEST_BIN_ARGS) diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go index c2d7e84c891..f3893022621 100644 --- a/pkg/controller/gc.go +++ b/pkg/controller/gc.go @@ -50,6 +50,7 @@ func (c *Controller) gc() error { c.gcVip, c.gcLbSvcPods, c.gcVPCDNS, + c.gcEdgeRouter, } for _, gcFunc := range gcFunctions { if err := gcFunc(); err != nil { @@ -60,6 +61,54 @@ func (c *Controller) gc() error { return nil } +func (c *Controller) gcEdgeRouter() error { + klog.Infof("start to gc edge router") + + edgeRouterAdvertisement, errAdv := c.bgpEdgeRouterAdvertisementLister.List(labels.Everything()) + if errAdv != nil { + klog.Errorf("failed to list edge router advertisements, %v", errAdv) + return errAdv + } + for _, advertisement := range edgeRouterAdvertisement { + if err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouterAdvertisements(advertisement.Namespace).Delete(context.Background(), advertisement.Name, metav1.DeleteOptions{}); err != nil { + if !k8serrors.IsNotFound(err) { + klog.Errorf("failed to delete edge router advertisement %s, %v", advertisement.Name, err) + return err + } + } + } + + bgpConfigs, errConf := c.gobgpConfigLister.List(labels.Everything()) + if errConf != nil { + klog.Errorf("failed to list gobgp configs, %v", errConf) + return errConf + } + for _, config := range bgpConfigs { + if err := c.config.KubeOvnClient.KubeovnV1().GobgpConfigs(config.Namespace).Delete(context.Background(), config.Name, metav1.DeleteOptions{}); err != nil { + if !k8serrors.IsNotFound(err) { + klog.Errorf("failed to delete gobgp config %s, %v", config.Name, err) + return err + } + } + } + + edgeRouters, errER := c.bgpEdgeRouterLister.List(labels.Everything()) + if errER != nil { + klog.Errorf("failed to list edge routers, %v", errER) + return errER + } + for _, edgeRouter := range edgeRouters { + if err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouters(edgeRouter.Namespace).Delete(context.Background(), edgeRouter.Name, metav1.DeleteOptions{}); err != nil { + if !k8serrors.IsNotFound(err) { + klog.Errorf("failed to delete edge router %s, %v", edgeRouter.Name, err) + return err + } + } + } + + return nil +} + func (c *Controller) gcLogicalRouterPort() error { klog.Infof("start to gc logical router port") vpcs, err := c.vpcsLister.List(labels.Everything()) diff --git a/test/e2e/edge-router/e2e_test.go b/test/e2e/edge-router/e2e_test.go new file mode 100644 index 00000000000..6a38517a007 --- /dev/null +++ b/test/e2e/edge-router/e2e_test.go @@ -0,0 +1,559 @@ +package multus + +import ( + "context" + "flag" + "fmt" + "maps" + "math/rand/v2" + "net" + "reflect" + + // "slices" + "strconv" + "strings" + "testing" + + // "time" + + dockernetwork "github.com/docker/docker/api/types/network" + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/component-base/logs" + "k8s.io/klog/v2" + commontest "k8s.io/kubernetes/test/e2e/common" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/test/e2e/framework" + "github.com/kubeovn/kube-ovn/test/e2e/framework/docker" + + // "github.com/kubeovn/kube-ovn/test/e2e/framework/iproute" + "github.com/kubeovn/kube-ovn/test/e2e/framework/kind" +) + +func init() { + klog.SetOutput(ginkgo.GinkgoWriter) + + // Register flags. + config.CopyFlags(config.Flags, flag.CommandLine) + k8sframework.RegisterCommonFlags(flag.CommandLine) + k8sframework.RegisterClusterFlags(flag.CommandLine) +} + +func TestE2E(t *testing.T) { + k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) + + logs.InitLogs() + defer logs.FlushLogs() + klog.EnableContextualLogging(true) + + gomega.RegisterFailHandler(k8sframework.Fail) + + // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins + suiteConfig, reporterConfig := k8sframework.CreateGinkgoConfig() + klog.Infof("Starting e2e run %q on Ginkgo node %d", k8sframework.RunID, suiteConfig.ParallelProcess) + ginkgo.RunSpecs(t, "Kube-OVN e2e suite", suiteConfig, reporterConfig) +} + +const ( + kindNetwork = "kind" + + controlPlaneLabel = "node-role.kubernetes.io/control-plane" +) + +var clusterName string + +var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { + // Reference common test to make the import valid. + commontest.CurrentSuite = commontest.E2E + + cs, err := k8sframework.LoadClientset() + framework.ExpectNoError(err) + + ginkgo.By("Getting k8s nodes") + k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) + framework.ExpectNoError(err) + + var ok bool + if clusterName, ok = kind.IsKindProvided(k8sNodes.Items[0].Spec.ProviderID); !ok { + ginkgo.Fail("vpc-egress-gateway spec only runs on kind clusters") + } + + return []byte(clusterName) +}, func(data []byte) { + clusterName = string(data) +}) + +var _ = framework.Describe("[group:ber]", func() { + f := framework.NewDefaultFramework("ber") + + var vpcClient *framework.VpcClient + var subnetClient *framework.SubnetClient + var nadClient *framework.NetworkAttachmentDefinitionClient + var nadName, externalSubnetName, namespaceName string + var schedulableNodes []corev1.Node + + var replicas int32 + ginkgo.BeforeEach(func() { + namespaceName = f.Namespace.Name + nadName = "nad-" + framework.RandomSuffix() + externalSubnetName = "ext-" + framework.RandomSuffix() + vpcClient = f.VpcClient() + subnetClient = f.SubnetClient() + nadClient = f.NetworkAttachmentDefinitionClient() + + nodeList, err := e2enode.GetReadyNodesIncludingTainted(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + framework.ExpectNotEmpty(nodeList.Items) + + nodeList, err = e2enode.GetReadySchedulableNodes(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + framework.ExpectNotEmpty(nodeList.Items) + schedulableNodes = nodeList.Items + + replicas = min(int32(len(schedulableNodes)), 3) + }) + + // framework.ConformanceIt("should be able to create edge-router with both underlay and overlay subnet", func() { + // provider := fmt.Sprintf("%s.%s.%s", nadName, namespaceName, util.OvnProvider) + + // ginkgo.By("Creating network attachment definition " + nadName) + // nad := framework.MakeOVNNetworkAttachmentDefinition(nadName, namespaceName, provider, nil) + // ginkgo.DeferCleanup(func() { + // ginkgo.By("Deleting network attachment definition " + nadName) + // nadClient.Delete(nadName) + // }) + // nad = nadClient.Create(nad) + // framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + // dockerNetworkName := "net-" + framework.RandomSuffix() + // ginkgo.By("Creating docker network " + dockerNetworkName) + // dockerNetwork, err := docker.NetworkCreate(dockerNetworkName, true, true) + // framework.ExpectNoError(err, "creating docker network "+dockerNetworkName) + // ginkgo.DeferCleanup(func() { + // ginkgo.By("Deleting docker network " + dockerNetworkName) + // err = docker.NetworkRemove(dockerNetworkName) + // framework.ExpectNoError(err, "removing docker network "+dockerNetworkName) + // }) + + // ginkgo.By("Getting kind nodes") + // kindNodes, err := kind.ListNodes(clusterName, "") + // framework.ExpectNoError(err, "getting nodes in kind cluster") + // framework.ExpectNotEmpty(nodes) + + // ginkgo.By("Connecting nodes to the docker network") + // err = kind.NetworkConnect(dockerNetwork.ID, kindNodes) + // framework.ExpectNoError(err, "connecting nodes to network "+dockerNetworkName) + // ginkgo.DeferCleanup(func() { + // err = kind.NetworkDisconnect(dockerNetwork.ID, kindNodes) + // framework.ExpectNoError(err, "disconnecting nodes from network "+dockerNetworkName) + // }) + + // ginkgo.By("Getting node links that belong to the docker network") + // kindNodes, err = kind.ListNodes(clusterName, "") + // framework.ExpectNoError(err, "getting nodes in kind cluster") + // linkMap := make(map[string]*iproute.Link, len(nodes)) + // for _, node := range kindNodes { + // links, err := node.ListLinks() + // framework.ExpectNoError(err, "failed to list links on node %s: %v", node.Name(), err) + + // for _, link := range links { + // if link.Address == node.NetworkSettings.Networks[dockerNetworkName].MacAddress { + // linkMap[node.Name()] = &link + // break + // } + // } + // framework.ExpectHaveKey(linkMap, node.Name()) + // } + + // providerNetworkName := "pn-" + framework.RandomSuffix() + // ginkgo.By("Creating provider network " + providerNetworkName) + // providerNetworkClient := f.ProviderNetworkClient() + // ginkgo.DeferCleanup(func() { + // ginkgo.By("Deleting provider network " + providerNetworkName) + // providerNetworkClient.DeleteSync(providerNetworkName) + // }) + // var defaultInterface string + // customInterfaces := make(map[string][]string, 0) + // for node, link := range linkMap { + // if defaultInterface == "" { + // defaultInterface = link.IfName + // } else if link.IfName != defaultInterface { + // customInterfaces[link.IfName] = append(customInterfaces[link.IfName], node) + // } + // } + // pn := framework.MakeProviderNetwork(providerNetworkName, false, defaultInterface, customInterfaces, nil) + // _ = providerNetworkClient.CreateSync(pn) + + // vlanName := "vlan-" + framework.RandomSuffix() + // ginkgo.By("Creating vlan " + vlanName) + // vlanClient := f.VlanClient() + // vlan := framework.MakeVlan(vlanName, providerNetworkName, 0) + // _ = vlanClient.Create(vlan) + // ginkgo.DeferCleanup(func() { + // ginkgo.By("Deleting vlan " + vlanName) + // vlanClient.Delete(vlanName) + // }) + + // ginkgo.By("Getting docker network " + dockerNetworkName) + // network, err := docker.NetworkInspect(dockerNetworkName) + // framework.ExpectNoError(err, "getting docker network "+dockerNetworkName) + + // externalSubnet := generateSubnetFromDockerNetwork(externalSubnetName, network, f.HasIPv4(), f.HasIPv6()) + // externalSubnet.Spec.Provider = provider + // externalSubnet.Spec.Vlan = vlanName + + // ginkgo.By("Creating underlay subnet " + externalSubnetName) + // ginkgo.DeferCleanup(func() { + // ginkgo.By("Deleting external subnet " + externalSubnetName) + // subnetClient.DeleteSync(externalSubnetName) + // }) + // _ = subnetClient.CreateSync(externalSubnet) + + // vpcName := util.DefaultVpc + // cidr := framework.RandomCIDR(f.ClusterIPFamily) + // bfdIP := framework.RandomIPs(cidr, ";", 1) + // ginkgo.By("Enabling BFD Port with IP " + bfdIP + " for VPC " + vpcName) + // vpc := vpcClient.Get(vpcName) + // patchedVpc := vpc.DeepCopy() + // patchedVpc.Spec.BFDPort = &apiv1.BFDPort{ + // Enabled: true, + // IP: bfdIP, + // NodeSelector: &metav1.LabelSelector{ + // MatchExpressions: []metav1.LabelSelectorRequirement{{ + // Key: controlPlaneLabel, + // Operator: metav1.LabelSelectorOpExists, + // }}, + // }, + // } + // updatedVpc := vpcClient.PatchSync(vpc, patchedVpc, 10*time.Second) + // ginkgo.DeferCleanup(func() { + // ginkgo.By("Disabling BFD Port for VPC " + vpcName) + // patchedVpc := updatedVpc.DeepCopy() + // patchedVpc.Spec.BFDPort = nil + // updatedVpc := vpcClient.PatchSync(updatedVpc, patchedVpc, 10*time.Second) + // framework.ExpectEmpty(updatedVpc.Status.BFDPort.Name) + // framework.ExpectEmpty(updatedVpc.Status.BFDPort.Nodes) + // }) + + // framework.ExpectNotEmpty(updatedVpc.Status.BFDPort.Name) + // for _, node := range nodes { + // if slices.Contains(updatedVpc.Status.BFDPort.Nodes, node.Name) { + // framework.ExpectHaveKey(node.Labels, controlPlaneLabel) + // } else { + // framework.ExpectNotHaveKey(node.Labels, controlPlaneLabel) + // } + // } + + // // TODO: check ovn LRP + + // berTest(f, true, provider, nadName, vpcName, vpc.Status.DefaultLogicalSwitch, externalSubnetName, replicas) + // }) + + framework.ConformanceIt("should be able to create edge-router with macvlan", func() { + provider := fmt.Sprintf("%s.%s", nadName, namespaceName) + + ginkgo.By("Creating network attachment definition " + nadName) + nad := framework.MakeMacvlanNetworkAttachmentDefinition(nadName, namespaceName, "eth0", "bridge", provider, nil) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting network attachment definition " + nadName) + nadClient.Delete(nadName) + }) + nad = nadClient.Create(nad) + framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + vpcName := "vpc-" + framework.RandomSuffix() + ginkgo.By("Creating vpc " + vpcName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting vpc " + vpcName) + vpcClient.DeleteSync(vpcName) + }) + vpc := &apiv1.Vpc{ObjectMeta: metav1.ObjectMeta{Name: vpcName}} + vpc = vpcClient.CreateSync(vpc) + framework.ExpectEmpty(vpc.Status.BFDPort.Name) + framework.ExpectEmpty(vpc.Status.BFDPort.IP) + framework.ExpectEmpty(vpc.Status.BFDPort.Nodes) + + internalSubnetName := "int-" + framework.RandomSuffix() + ginkgo.By("Creating internal subnet " + internalSubnetName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting internal subnet " + internalSubnetName) + subnetClient.DeleteSync(internalSubnetName) + }) + cidr := framework.RandomCIDR(f.ClusterIPFamily) + internalSubnet := framework.MakeSubnet(internalSubnetName, "", cidr, "", vpcName, nad.Name, nil, nil, nil) + _ = subnetClient.CreateSync(internalSubnet) + + ginkgo.By("Getting docker network " + kindNetwork) + network, err := docker.NetworkInspect(kindNetwork) + framework.ExpectNoError(err, "getting docker network "+kindNetwork) + + externalSubnet := generateSubnetFromDockerNetwork(externalSubnetName, network, f.HasIPv4(), f.HasIPv6()) + externalSubnet.Spec.Provider = provider + + ginkgo.By("Creating macvlan subnet " + externalSubnetName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting external subnet " + externalSubnetName) + subnetClient.DeleteSync(externalSubnetName) + }) + _ = subnetClient.CreateSync(externalSubnet) + + berTest(f, false, provider, nadName, vpcName, internalSubnetName, externalSubnetName, replicas) + }) +}) + +func generateSubnetFromDockerNetwork(subnetName string, network *dockernetwork.Inspect, ipv4, ipv6 bool) *apiv1.Subnet { + ginkgo.GinkgoHelper() + + ginkgo.By("Generating subnet configuration from docker network " + network.Name) + var cidrV4, cidrV6, gatewayV4, gatewayV6 string + for _, config := range network.IPAM.Config { + switch util.CheckProtocol(config.Subnet) { + case apiv1.ProtocolIPv4: + if ipv4 { + cidrV4 = config.Subnet + gatewayV4 = config.Gateway + } + case apiv1.ProtocolIPv6: + if ipv6 { + cidrV6 = config.Subnet + if gatewayV6 = config.Gateway; gatewayV6 == "" { + var err error + gatewayV6, err = util.FirstIP(cidrV6) + framework.ExpectNoError(err) + } + } + } + } + + cidr := make([]string, 0, 2) + gateway := make([]string, 0, 2) + if ipv4 { + cidr = append(cidr, cidrV4) + gateway = append(gateway, gatewayV4) + } + if ipv6 { + cidr = append(cidr, cidrV6) + gateway = append(gateway, gatewayV6) + } + + excludeIPs := make([]string, 0, len(network.Containers)*2) + for _, container := range network.Containers { + if container.IPv4Address != "" && ipv4 { + excludeIPs = append(excludeIPs, strings.Split(container.IPv4Address, "/")[0]) + } + if container.IPv6Address != "" && ipv6 { + excludeIPs = append(excludeIPs, strings.Split(container.IPv6Address, "/")[0]) + } + } + + return framework.MakeSubnet(subnetName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", "", excludeIPs, nil, nil) +} + +func checkEgressAccess(f *framework.Framework, namespaceName, svrPodName, image, svrPort string, svrIPs, extIPs []string, intIPs map[string][]string, subnetName, nodeName string, snat bool) { + ginkgo.GinkgoHelper() + + podName := "pod-" + framework.RandomSuffix() + ginkgo.By("Creating client pod " + podName + " within subnet " + subnetName) + labels := map[string]string{"snat": strconv.FormatBool(snat)} + annotations := map[string]string{util.LogicalSwitchAnnotation: subnetName} + pod := framework.MakePrivilegedPod(namespaceName, podName, labels, annotations, image, []string{"sleep", "infinity"}, nil) + pod.Spec.NodeName = nodeName + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting pod " + podName) + f.PodClient().DeleteSync(podName) + }) + pod = f.PodClient().CreateSync(pod) + + if !snat { + // skip egress route check if SNAT is enabled + // traceroute does not work for pods selected by the selectors + var hops []string + if nodeName == "" { + for ips := range maps.Values(intIPs) { + hops = append(hops, ips...) + } + } else { + hops = intIPs[nodeName] + } + framework.CheckPodEgressRoutes(pod.Namespace, pod.Name, f.HasIPv4(), f.HasIPv6(), 2, hops) + } + + if !snat { + podIPv4, podIPv6 := util.SplitIpsByProtocol(util.PodIPs(*pod)) + hopsIPv4, hopsIPv6 := util.SplitIpsByProtocol(extIPs) + addEcmpRoutes(namespaceName, svrPodName, podIPv4, hopsIPv4) + addEcmpRoutes(namespaceName, svrPodName, podIPv6, hopsIPv6) + } + + expectedClientIPs := extIPs + if !snat { + expectedClientIPs = util.PodIPs(*pod) + } + for _, svrIP := range svrIPs { + protocol := strings.ToLower(util.CheckProtocol(svrIP)) + ginkgo.By("Checking connection from " + pod.Name + " to " + svrIP + " via " + protocol) + cmd := fmt.Sprintf("curl -q -s --connect-timeout 2 --max-time 2 %s/clientip", net.JoinHostPort(svrIP, svrPort)) + ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, cmd, pod.Namespace, pod.Name)) + output := e2epodoutput.RunHostCmdOrDie(pod.Namespace, pod.Name, cmd) + clientIP, _, err := net.SplitHostPort(strings.TrimSpace(output)) + framework.ExpectNoError(err) + framework.ExpectContainElement(expectedClientIPs, clientIP) + } +} + +func addEcmpRoutes(namespaceName, podName string, destinations, nextHops []string) { + ginkgo.GinkgoHelper() + + if len(destinations) == 0 || len(nextHops) == 0 { + return + } + + var args string + if len(nextHops) == 1 { + args = " via " + nextHops[0] + } else { + for _, ip := range nextHops { + args += fmt.Sprintf(" nexthop via %s dev net1 weight 1", ip) + } + } + for _, dst := range destinations { + cmd := fmt.Sprintf("ip route add %s%s", dst, args) + output, err := e2epodoutput.RunHostCmd(namespaceName, podName, cmd) + framework.ExpectNoError(err, output) + } +} + +func berTest(f *framework.Framework, bfd bool, provider, nadName, vpcName, internalSubnetName, externalSubnetName string, replicas int32) { + ginkgo.GinkgoHelper() + + namespaceName := f.Namespace.Name + forwardSubnetName := "forward-" + framework.RandomSuffix() + subnetClient := f.SubnetClient() + berClient := f.BgpEdgeRouterClient() + deployClient := f.DeploymentClient() + podClient := f.PodClient() + + var forwardSubnet *apiv1.Subnet + ginkgo.By("Creating subnet " + forwardSubnetName) + cidr := framework.RandomCIDR(f.ClusterIPFamily) + subnet := framework.MakeSubnet(forwardSubnetName, "", cidr, "", vpcName, "", nil, nil, nil) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting subnet " + forwardSubnetName) + subnetClient.DeleteSync(forwardSubnetName) + }) + _ = subnetClient.CreateSync(subnet) + forwardSubnet = subnet + + berName := "ber-" + framework.RandomSuffix() + ginkgo.By("Creating bgp edge router " + berName) + ber := framework.MakeBgpEdgeRouter(namespaceName, berName, vpcName, replicas, internalSubnetName, externalSubnetName) + if rand.Int32N(2) == 0 { + ber.Spec.Prefix = fmt.Sprintf("e2e-%s-", framework.RandomSuffix()) + } + ber.Spec.BFD.Enabled = bfd + ber.Spec.Policies = []apiv1.BgpEdgeRouterPolicy{{ + SNAT: false, + IPBlocks: strings.Split(forwardSubnet.Spec.CIDRBlock, ","), + }} + if vpcName == util.DefaultVpc { + ber.Spec.VPC = "" // test whether the ber works without specifying VPC + ber.Spec.TrafficPolicy = apiv1.TrafficPolicyLocal + } + if util.IsOvnProvider(provider) { + ber.Spec.Selectors = []apiv1.BgpEdgeRouterSelector{{ + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + corev1.LabelMetadataName: namespaceName, + }, + }, + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "snat": strconv.FormatBool(true), + }, + }, + }} + } else { + ber.Spec.Policies = append(ber.Spec.Policies, apiv1.BgpEdgeRouterPolicy{ + SNAT: true, + Subnets: []string{forwardSubnetName}, + }) + } + + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting bgp edge router " + berName) + berClient.DeleteSync(berName) + }) + ber = berClient.CreateSync(ber) + + ginkgo.By("Validating bgp edge router status") + framework.ExpectTrue(ber.Status.Ready) + framework.ExpectEqual(ber.Status.Phase, apiv1.PhaseCompleted) + framework.ExpectHaveLen(ber.Status.InternalIPs, int(replicas)) + framework.ExpectHaveLen(ber.Status.ExternalIPs, int(replicas)) + + ginkgo.By("Validating bgp edge router workload") + framework.ExpectEqual(ber.Status.Workload.Name, ber.Spec.Prefix+ber.Name) + deploy := deployClient.Get(ber.Status.Workload.Name) + framework.ExpectEqual(deploy.Status.Replicas, replicas) + framework.ExpectEqual(deploy.Status.ReadyReplicas, replicas) + gvk := appsv1.SchemeGroupVersion.WithKind(reflect.TypeOf(deploy).Elem().Name()) + framework.ExpectEqual(ber.Status.Workload.APIVersion, gvk.GroupVersion().String()) + framework.ExpectEqual(ber.Status.Workload.Kind, gvk.Kind) + framework.ExpectHaveLen(ber.Status.Workload.Nodes, int(replicas)) + workloadPods, err := deployClient.GetPods(deploy) + framework.ExpectNoError(err) + framework.ExpectHaveLen(workloadPods.Items, int(replicas)) + podNodes := make([]string, 0, len(workloadPods.Items)) + intIPs := make(map[string][]string, len(workloadPods.Items)) + for _, pod := range workloadPods.Items { + framework.ExpectNotContainElement(podNodes, pod.Spec.NodeName) + podNodes = append(podNodes, pod.Spec.NodeName) + intIPs[pod.Spec.NodeName] = util.PodIPs(pod) + } + framework.ExpectConsistOf(ber.Status.Workload.Nodes, podNodes) + + svrPodName := "svr-" + framework.RandomSuffix() + ginkgo.By("Creating netexec server pod " + svrPodName) + routes := util.NewPodRoutes() + dstV4, dstV6 := util.SplitStringIP(forwardSubnet.Spec.CIDRBlock) + gwV4, gwV6 := util.SplitStringIP(ber.Status.ExternalIPs[0]) + routes.Add(provider, dstV4, gwV4) + routes.Add(provider, dstV6, gwV6) + annotations, err := routes.ToAnnotations() + framework.ExpectNoError(err) + attachmentNetworkName := fmt.Sprintf("%s/%s", namespaceName, nadName) + annotations[nadv1.NetworkAttachmentAnnot] = attachmentNetworkName + port := strconv.Itoa(8000 + rand.IntN(1000)) + args := []string{"netexec", "--http-port", port} + svrPod := framework.MakePrivilegedPod(namespaceName, svrPodName, nil, annotations, framework.AgnhostImage, nil, args) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting pod " + svrPodName) + podClient.DeleteSync(svrPodName) + }) + svrPod = podClient.CreateSync(svrPod) + svrIPs, err := util.PodAttachmentIPs(svrPod, attachmentNetworkName) + framework.ExpectNoError(err) + + image := workloadPods.Items[0].Spec.Containers[0].Image + extIPs := make([]string, 0, len(ber.Status.ExternalIPs)*2) + for _, ips := range ber.Status.ExternalIPs { + extIPs = append(extIPs, strings.Split(ips, ",")...) + } + + var nodeName string + if ber.Spec.TrafficPolicy == apiv1.TrafficPolicyLocal { + nodeName = ber.Status.Workload.Nodes[0] + } + checkEgressAccess(f, namespaceName, svrPodName, image, port, svrIPs, extIPs, intIPs, forwardSubnetName, nodeName, false) +} diff --git a/test/e2e/framework/bgp-edge-router.go b/test/e2e/framework/bgp-edge-router.go new file mode 100644 index 00000000000..c8b2cf30984 --- /dev/null +++ b/test/e2e/framework/bgp-edge-router.go @@ -0,0 +1,208 @@ +package framework + +import ( + "context" + "errors" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + v1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// BgpEdgeRouterClient is a struct for bgp edge router client. +type BgpEdgeRouterClient struct { + f *Framework + namespace string + v1.BgpEdgeRouterInterface +} + +func NewBgpEdgeRouterClient(cs clientset.Interface, namespapce string) *BgpEdgeRouterClient { + return &BgpEdgeRouterClient{ + namespace: namespapce, + BgpEdgeRouterInterface: cs.KubeovnV1().BgpEdgeRouters(namespapce), + } +} + +func (f *Framework) BgpEdgeRouterClient() *BgpEdgeRouterClient { + return &BgpEdgeRouterClient{ + f: f, + namespace: f.Namespace.Name, + BgpEdgeRouterInterface: f.KubeOVNClientSet.KubeovnV1().BgpEdgeRouters(f.Namespace.Name), + } +} + +func (f *Framework) BgpEdgeRouterClientNS(namespapce string) *BgpEdgeRouterClient { + return &BgpEdgeRouterClient{ + f: f, + namespace: namespapce, + BgpEdgeRouterInterface: f.KubeOVNClientSet.KubeovnV1().BgpEdgeRouters(namespapce), + } +} + +func (c *BgpEdgeRouterClient) Get(name string) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + router, err := c.BgpEdgeRouterInterface.Get(context.TODO(), name, metav1.GetOptions{}) + ExpectNoError(err) + return router +} + +// Create creates a new bgp-edge-router according to the framework specifications +func (c *BgpEdgeRouterClient) Create(router *apiv1.BgpEdgeRouter) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + g, err := c.BgpEdgeRouterInterface.Create(context.TODO(), router, metav1.CreateOptions{}) + ExpectNoError(err, "Error creating bgp-edge-router") + return g.DeepCopy() +} + +// CreateSync creates a new bgp-edge-router according to the framework specifications, and waits for it to be ready. +func (c *BgpEdgeRouterClient) CreateSync(router *apiv1.BgpEdgeRouter) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + _ = c.Create(router) + return c.WaitUntil(router.Name, func(g *apiv1.BgpEdgeRouter) (bool, error) { + return g.Ready(), nil + }, "Ready", 2*time.Second, timeout) +} + +// Patch patches the router +func (c *BgpEdgeRouterClient) Patch(original, modified *apiv1.BgpEdgeRouter) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + + patch, err := util.GenerateMergePatchPayload(original, modified) + ExpectNoError(err) + + var patchedRouter *apiv1.BgpEdgeRouter + err = wait.PollUntilContextTimeout(context.Background(), 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + g, err := c.BgpEdgeRouterInterface.Patch(ctx, original.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "") + if err != nil { + return handleWaitingAPIError(err, false, "patch bgp-edge-router %s/%s", original.Namespace, original.Name) + } + patchedRouter = g + return true, nil + }) + if err == nil { + return patchedRouter.DeepCopy() + } + + if errors.Is(err, context.DeadlineExceeded) { + Failf("timed out while retrying to patch bgp-edge-router %s/%s", original.Namespace, original.Name) + } + Failf("error occurred while retrying to patch bgp-edge-router %s/%s: %v", original.Namespace, original.Name, err) + + return nil +} + +// PatchSync patches the router and waits the router to meet the condition +func (c *BgpEdgeRouterClient) PatchSync(original, modified *apiv1.BgpEdgeRouter) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + _ = c.Patch(original, modified) + return c.WaitUntil(original.Name, func(g *apiv1.BgpEdgeRouter) (bool, error) { + return g.Ready(), nil + }, "Ready", 2*time.Second, timeout) +} + +// Delete deletes a bgp-edge-router if the bgp-edge-router exists +func (c *BgpEdgeRouterClient) Delete(name string) { + ginkgo.GinkgoHelper() + err := c.BgpEdgeRouterInterface.Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + Failf("Failed to delete bgp-edge-router %s/%s: %v", c.namespace, name, err) + } +} + +// DeleteSync deletes the bgp-edge-router and waits for the bgp-edge-router to disappear for `timeout`. +// If the bgp-edge-router doesn't disappear before the timeout, it will fail the test. +func (c *BgpEdgeRouterClient) DeleteSync(name string) { + ginkgo.GinkgoHelper() + c.Delete(name) + gomega.Expect(c.WaitToDisappear(name, 2*time.Second, timeout)).To(gomega.Succeed(), "wait for bgp-edge-router %s/%s to disappear", c.namespace, name) +} + +// WaitUntil waits the given timeout duration for the specified condition to be met. +func (c *BgpEdgeRouterClient) WaitUntil(name string, cond func(g *apiv1.BgpEdgeRouter) (bool, error), condDesc string, interval, timeout time.Duration) *apiv1.BgpEdgeRouter { + var router *apiv1.BgpEdgeRouter + err := wait.PollUntilContextTimeout(context.Background(), interval, timeout, false, func(_ context.Context) (bool, error) { + Logf("Waiting for bgp-edge-router %s/%s to meet condition %q", c.namespace, name, condDesc) + router = c.Get(name).DeepCopy() + met, err := cond(router) + if err != nil { + return false, fmt.Errorf("failed to check condition for bgp-edge-router %s/%s: %w", c.namespace, name, err) + } + if met { + Logf("bgp-edge-router %s/%s met condition %q", c.namespace, name, condDesc) + } else { + Logf("bgp-edge-router %s/%s not met condition %q", c.namespace, name, condDesc) + } + return met, nil + }) + if err == nil { + return router + } + + if errors.Is(err, context.DeadlineExceeded) { + Failf("timed out while waiting for bgp-edge-router %s/%s to meet condition %q", c.namespace, name, condDesc) + } + Failf("error occurred while waiting for bgp-edge-router %s/%s to meet condition %q: %v", c.namespace, name, condDesc, err) + + return nil +} + +// WaitToDisappear waits the given timeout duration for the specified bgp-edge-router to disappear. +func (c *BgpEdgeRouterClient) WaitToDisappear(name string, _, timeout time.Duration) error { + err := framework.Gomega().Eventually(context.Background(), framework.HandleRetry(func(ctx context.Context) (*apiv1.BgpEdgeRouter, error) { + svc, err := c.BgpEdgeRouterInterface.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, nil + } + return svc, err + })).WithTimeout(timeout).Should(gomega.BeNil()) + if err != nil { + return fmt.Errorf("expected vpc-egress-gateway %s/%s to not be found: %w", c.namespace, name, err) + } + return nil +} + +func MakeBgpEdgeRouter(namespace, name, vpc string, replicas int32, internalSubnet, externalSubnet string) *apiv1.BgpEdgeRouter { + return &apiv1.BgpEdgeRouter{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: apiv1.BgpEdgeRouterSpec{ + Replicas: replicas, + VPC: vpc, + InternalSubnet: internalSubnet, + ExternalSubnet: externalSubnet, + BFD: apiv1.BgpEdgeRouterBFDConfig{ + Enabled: true, + MinRX: 300, + MinTX: 300, + Multiplier: 3, + }, + Policies: []apiv1.BgpEdgeRouterPolicy{ + { + SNAT: false, + Subnets: []string{ + internalSubnet, + }, + }, + }, + BGP: apiv1.BgpEdgeRouterBGPConfig{ + Enabled: true, + ASN: 65000, + EnableGracefulRestart: true, + }, + }, + } +} From a0d8fc148615911ea894adae4e8a8910de48398a Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Mon, 18 Aug 2025 00:55:08 -0700 Subject: [PATCH 42/66] [fix] add reset logic at set default action in update bgp policy --- dist/images/update-bgp-policy.sh | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index a5db5e72d37..de93a41e691 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -271,8 +271,27 @@ set_default_action() { echo "-> Applying default policy to global export" exec_cmd $GOBGP_BIN global policy export add default $action - # echo "-> Soft reset neighbor policy" - # exec_cmd $GOBGP_BIN neighbor $nbr_ip softreset + echo "-> Soft reset neighbor policy" + # Get all neighbor IPs and perform soft reset for each + local neighbors=() + while IFS= read -r line; do + # Extract IP address from gobgp neighbor output (assuming first column is IP) + local neighbor_ip=$(echo "$line" | awk '{print $1}') + # Skip header lines and empty lines, validate IP format + if [[ "$neighbor_ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then + neighbors+=("$neighbor_ip") + fi + done < <($GOBGP_BIN neighbor 2>/dev/null | tail -n +2) + + if [[ ${#neighbors[@]} -eq 0 ]]; then + echo " No neighbors found to reset" + else + echo " Found ${#neighbors[@]} neighbor(s) to reset:" + for neighbor_ip in "${neighbors[@]}"; do + echo " -> Soft resetting neighbor: $neighbor_ip" + exec_cmd_safe $GOBGP_BIN neighbor "$neighbor_ip" softreset + done + fi echo "=== Default action set to $action successfully ===" } From 78bdab813fc4481fef6af64614b4cea541cce1b2 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 18 Aug 2025 00:56:00 -0700 Subject: [PATCH 43/66] kube-ovn-crd's routeServerClient mode added. --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index bef165b9148..3584df2e497 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3542,6 +3542,9 @@ spec: edgeRouterMode: type: boolean default: false + routeServerClient: + type: boolean + default: false enabled: type: boolean default: false From a497b209d34647446430049de743fc2761d918c3 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 18 Aug 2025 00:57:02 -0700 Subject: [PATCH 44/66] bgp-edge-advertisement update logic modified. --- pkg/controller/bgp_edge_router_advertisement.go | 11 ++++++++++- pkg/controller/gobgp_config.go | 6 ++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/controller/bgp_edge_router_advertisement.go b/pkg/controller/bgp_edge_router_advertisement.go index 8e6d15aa2a1..399c7fcc169 100644 --- a/pkg/controller/bgp_edge_router_advertisement.go +++ b/pkg/controller/bgp_edge_router_advertisement.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "reflect" "regexp" "slices" "sort" @@ -51,7 +52,15 @@ func (c *Controller) enqueueUpdateBgpEdgeRouterAdvertisement(oldObj, newObj any) newVer: newRouter, } - c.updateBgpEdgeRouterAdvertisementQueue.Add(updateVer) + if !newRouter.DeletionTimestamp.IsZero() { + c.deleteBgpEdgeRouterAdvertisementQueue.Add(key) + return + } + + if !reflect.DeepEqual(oldRouter.Spec, newRouter.Spec) { + klog.Infof("enqueue update bgp-edge-router-advertisement %s", key) + c.updateBgpEdgeRouterAdvertisementQueue.Add(updateVer) + } } func (c *Controller) enqueueDeleteBgpEdgeRouterAdvertisement(obj any) { diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index a17accebdb4..e8dc68d95f3 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -60,12 +60,11 @@ func (c *Controller) enqueueUpdateGobgpConfig(oldObj, newObj any) { klog.Infof("enqueue update gobgp-config %s", key) c.updateGobgpConfigQueue.Add(updateConfigVer) } - - // c.updateGobgpConfigQueue.Add(updateConfigVer) } func (c *Controller) enqueueDeleteGobgpConfig(obj any) { var gobgpConfig *kubeovnv1.GobgpConfig + switch t := obj.(type) { case *kubeovnv1.GobgpConfig: gobgpConfig = t @@ -74,11 +73,14 @@ func (c *Controller) enqueueDeleteGobgpConfig(obj any) { gobgpConfig = v } } + if gobgpConfig == nil { klog.Warning("enqueueDeleteGobgpConfig: object is not GobgpConfig") return } + key := cache.MetaObjectToName(obj.(*kubeovnv1.GobgpConfig)).String() + klog.V(3).Infof("enqueue delete gobgp-config %s", key) c.deleteGobgpConfigQueue.Add(key) } From 752fd9bbb9eaafdb672f5dd4329011b8b86021b1 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 18 Aug 2025 01:14:02 -0700 Subject: [PATCH 45/66] ClusterRole modified. --- charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml | 6 ++++++ charts/kube-ovn/templates/ovn-CR.yaml | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml b/charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml index e1f0b05fd66..82f34b1a461 100644 --- a/charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml +++ b/charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml @@ -50,6 +50,12 @@ rules: - vpc-dnses/status - qos-policies - qos-policies/status + - bgp-edge-routers + - bgp-edge-routers/status + - bgp-edge-router-advertisements + - bgp-edge-router-advertisements/status + - gobgp-configs + - gobgp-configs/status verbs: - "*" - apiGroups: diff --git a/charts/kube-ovn/templates/ovn-CR.yaml b/charts/kube-ovn/templates/ovn-CR.yaml index c5f54a4ecee..061cea683f4 100644 --- a/charts/kube-ovn/templates/ovn-CR.yaml +++ b/charts/kube-ovn/templates/ovn-CR.yaml @@ -52,6 +52,10 @@ rules: - qos-policies/status - bgp-edge-routers - bgp-edge-routers/status + - bgp-edge-router-advertisements + - bgp-edge-router-advertisements/status + - gobgp-configs + - gobgp-configs/status verbs: - "*" - apiGroups: From 64d965b5f226f587579d570cfd2ef0dfb941d322 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Tue, 12 Aug 2025 23:40:27 -0700 Subject: [PATCH 46/66] modify kube-ovn-crd, gobgp-config crd's neighbors and toReceive mode is all, it cannot be set prefixes, and reduce listing gobgp-config crd's info --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index 096568412e8..bef165b9148 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3714,18 +3714,6 @@ spec: - name: neighbor-address type: string jsonPath: .spec.neighbors[*].address - - name: toReceive-mode - type: string - jsonPath: .spec.neighbors[*].toReceive.allowed.mode - - name: toReceive-prefixes - type: string - jsonPath: .spec.neighbors[*].toReceive.allowed.prefixes - - name: toAdvertise-mode - type: string - jsonPath: .spec.neighbors[*].toAdvertise.allowed.mode - - name: toAdvertise-prefixes - type: string - jsonPath: .spec.neighbors[*].toAdvertise.allowed.prefixes - name: READY type: boolean jsonPath: .status.ready @@ -3751,6 +3739,9 @@ spec: type: string neighbors: type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - address items: type: object required: @@ -3782,6 +3773,8 @@ spec: x-kubernetes-validations: - rule: "self.mode != 'filtered' || (has(self.prefixes) && size(self.prefixes) > 0)" message: "prefixes must be set and non-empty when mode is 'filtered'" + - rule: "self.mode != 'all' || !has(self.prefixes) || size(self.prefixes) == 0" + message: "If mode is 'all', cannot set prefixes" toReceive: type: object required: @@ -3803,6 +3796,8 @@ spec: x-kubernetes-validations: - rule: "self.mode != 'filtered' || (has(self.prefixes) && size(self.prefixes) > 0)" message: "prefixes must be set and non-empty when mode is 'filtered'" + - rule: "self.mode != 'all' || !has(self.prefixes) || size(self.prefixes) == 0" + message: "If mode is 'all', cannot set prefixes" status: type: object properties: From 5e335ae6e3cb0fcd10c7338be028e8d5386658e4 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Wed, 13 Aug 2025 02:39:06 -0700 Subject: [PATCH 47/66] [fix] update bgp policy script safely fix finalizer error in add handler change crd for gobgp config TODO set default policy check update handler run automatially after finishing add handler --- dist/images/update-bgp-policy.sh | 4 ---- pkg/controller/gobgp_config.go | 2 -- 2 files changed, 6 deletions(-) diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index 50cbf5b9da4..e4eef57b607 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -164,10 +164,6 @@ flush_neighbor_policy() { exec_cmd_safe $GOBGP_BIN policy prefix del $prefix_in exec_cmd_safe $GOBGP_BIN policy prefix del $prefix_out - # Phase 6: Apply policy to neighbor (safe mode) - echo "-> Soft reset neighbor policy" - exec_cmd_safe $GOBGP_BIN neighbor $nbr_ip softreset - # Check if any commands failed and need retry if [[ ${#FAILED_COMMANDS[@]} -eq 0 ]]; then echo "" diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index 17eb38d6835..e87d135a0c8 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -180,8 +180,6 @@ func (c *Controller) handleUpdateGobgpConfig(updatedObj *updateVerGobgpConfigObj } klog.Infof("reconciling gobgp-configs %s for update", key) - klog.Infof("debug gobgp-config old version : %v", updatedObj.oldVer) - klog.Infof("debug gobgp-config new version : %v", updatedObj.newVer) gobgpConfig := cachedGobgpConfig.DeepCopy() pods, err := c.validateGobgpConfig(gobgpConfig) From ad3bcfd18e2e69d4ccca855784d63070c62a9abe Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Sun, 17 Aug 2025 19:28:42 -0700 Subject: [PATCH 48/66] set_default_action nbr_ip parameter error is resovled. --- dist/images/update-bgp-policy.sh | 4 +- pkg/controller/controller.go | 1 + pkg/controller/gobgp_config.go | 66 +++++++++++++++++++++++++++----- 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index e4eef57b607..7b91952c87b 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -267,8 +267,8 @@ set_default_action() { echo "-> Applying default policy to global export" exec_cmd $GOBGP_BIN global policy export add default $action - echo "-> Soft reset neighbor policy" - exec_cmd $GOBGP_BIN neighbor $nbr_ip softreset + # echo "-> Soft reset neighbor policy" + # exec_cmd $GOBGP_BIN neighbor $nbr_ip softreset echo "=== Default action set to $action successfully ===" } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 6d05280ea8b..214d2e95fff 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -1298,6 +1298,7 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(runWorker("update bgp edge router advertisement", c.updateBgpEdgeRouterAdvertisementQueue, c.handleUpdateBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) go wait.Until(runWorker("delete bgp edge router advertisement", c.deleteBgpEdgeRouterAdvertisementQueue, c.handleDelBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) go wait.Until(c.resyncBgpRules, 60*time.Second, ctx.Done()) + go wait.Until(c.resyncBgpPolicyRules, 60*time.Second, ctx.Done()) go wait.Until(runWorker("add bgp edge router advertisement", c.addGobgpConfigQueue, c.handleAddGobgpConfig), time.Second, ctx.Done()) go wait.Until(runWorker("update bgp edge router advertisement", c.updateGobgpConfigQueue, c.handleUpdateGobgpConfig), time.Second, ctx.Done()) go wait.Until(runWorker("delete bgp edge router advertisement", c.deleteGobgpConfigQueue, c.handleDelGobgpConfig), time.Second, ctx.Done()) diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index e87d135a0c8..2a1ee3e1557 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -11,6 +11,7 @@ import ( corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" @@ -269,7 +270,6 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo klog.Error(err) return err } - klog.Infof("execUpdateBgpPolicy %s", key) cmdArs := []string{} if oldGobgpConfig != nil { @@ -283,14 +283,6 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo } // erase neighbor. cmdArs = append(cmdArs, "--", "flush-neighbor-policy", nbrIP) - // cmdArs = append(cmdArs, "--", "flush-prefix-out", nbrIP) - // cmdArs = append(cmdArs, "--", "flush-prefix-in", nbrIP) - // rcvMode := neighbor.ToReceive.Allowed.Mode - // rcvPrefixes := neighbor.ToReceive.Allowed.Prefixes - // if rcvMode == "all" { - // rcvPrefixes = []string{"0.0.0.0/0 0..32"} - // } - // cmdArs = append(cmdArs, append([]string{"--", "flush-prefix-in"}, rcvPrefixes...)...) } } else { // if oldGobgpConfig is nil, it means this is the first time to update the bgp policy @@ -342,11 +334,11 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo cmdArs = append(cmdArs, "--", "set-default-action", "accept") } - // cmdArs = append(cmdArs, "list_announced_route") if err := c.execCmd(pod, cmdArs); err != nil { klog.Error(err) return err } + return nil } @@ -473,3 +465,57 @@ func (c *Controller) execCmd(pod *corev1.Pod, cmdArs []string) error { func containsNeighbor(neighbors []string, address string) bool { return slices.Contains(neighbors, address) } + +func (c *Controller) resyncBgpPolicyRules() { + klog.Info("resync bgp edge router") + // resync all bgp edge routers + gobgpConfigs, err := c.gobgpConfigLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list bgp edge routers: %v", err) + return + } + + for _, gobgpConfig := range gobgpConfigs { + // Check router.Spec.BGP.AdvertisedRoutes same with pods bgp advertised routes + if err := c.syncGobgpPolicy(gobgpConfig); err != nil { + klog.Errorf("failed to sync advertised routes for bgp edge router %s: %v", gobgpConfig.Name, err) + continue + } + klog.Infof("resync bgp edge router %s", gobgpConfig.Name) + } +} + +func (c *Controller) syncGobgpPolicy(gobgpConfig *kubeovnv1.GobgpConfig) error { + key := cache.MetaObjectToName(gobgpConfig).String() + + c.gobgpConfigKeyMutex.LockKey(key) + defer func() { _ = c.gobgpConfigKeyMutex.UnlockKey(key) }() + + if !gobgpConfig.DeletionTimestamp.IsZero() { + c.deleteGobgpConfigQueue.Add(key) + return nil + } + klog.Infof("reconciling bgp-edge-router %s", key) + // Deep copy because we might mutate Status below. + cachedGobgpConfig := gobgpConfig.DeepCopy() + + pods, err := c.validateGobgpConfig(cachedGobgpConfig) + if err != nil || pods == nil { + klog.Error(err) + return err + } + + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + err = c.execUpdateBgpPolicy(key, pod, cachedGobgpConfig, cachedGobgpConfig) + if err != nil { + return err + } + klog.Infof("router pod %s/%s policy: %v", pod.Namespace, pod.Name, gobgpConfig) + } + + klog.Infof("finished sync bgp-edge-router %s advertised routes", key) + return nil +} From 00617ef7c43db248cf7b8b1baa13318b18d3ec8e Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Mon, 18 Aug 2025 00:55:08 -0700 Subject: [PATCH 49/66] [fix] add reset logic at set default action in update bgp policy --- dist/images/update-bgp-policy.sh | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index 7b91952c87b..3582c53f8db 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -267,8 +267,27 @@ set_default_action() { echo "-> Applying default policy to global export" exec_cmd $GOBGP_BIN global policy export add default $action - # echo "-> Soft reset neighbor policy" - # exec_cmd $GOBGP_BIN neighbor $nbr_ip softreset + echo "-> Soft reset neighbor policy" + # Get all neighbor IPs and perform soft reset for each + local neighbors=() + while IFS= read -r line; do + # Extract IP address from gobgp neighbor output (assuming first column is IP) + local neighbor_ip=$(echo "$line" | awk '{print $1}') + # Skip header lines and empty lines, validate IP format + if [[ "$neighbor_ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then + neighbors+=("$neighbor_ip") + fi + done < <($GOBGP_BIN neighbor 2>/dev/null | tail -n +2) + + if [[ ${#neighbors[@]} -eq 0 ]]; then + echo " No neighbors found to reset" + else + echo " Found ${#neighbors[@]} neighbor(s) to reset:" + for neighbor_ip in "${neighbors[@]}"; do + echo " -> Soft resetting neighbor: $neighbor_ip" + exec_cmd_safe $GOBGP_BIN neighbor "$neighbor_ip" softreset + done + fi echo "=== Default action set to $action successfully ===" } From bdf5f449d181d945b33eeb451ebbc7cd403ccf71 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 18 Aug 2025 00:56:00 -0700 Subject: [PATCH 50/66] kube-ovn-crd's routeServerClient mode added. --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index bef165b9148..3584df2e497 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3542,6 +3542,9 @@ spec: edgeRouterMode: type: boolean default: false + routeServerClient: + type: boolean + default: false enabled: type: boolean default: false From b86367436f5fa885a0677c3543f4b9f9dc9a126e Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 18 Aug 2025 00:57:02 -0700 Subject: [PATCH 51/66] bgp-edge-advertisement update logic modified. --- pkg/controller/bgp_edge_router_advertisement.go | 11 ++++++++++- pkg/controller/gobgp_config.go | 6 ++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/controller/bgp_edge_router_advertisement.go b/pkg/controller/bgp_edge_router_advertisement.go index 8e6d15aa2a1..399c7fcc169 100644 --- a/pkg/controller/bgp_edge_router_advertisement.go +++ b/pkg/controller/bgp_edge_router_advertisement.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "reflect" "regexp" "slices" "sort" @@ -51,7 +52,15 @@ func (c *Controller) enqueueUpdateBgpEdgeRouterAdvertisement(oldObj, newObj any) newVer: newRouter, } - c.updateBgpEdgeRouterAdvertisementQueue.Add(updateVer) + if !newRouter.DeletionTimestamp.IsZero() { + c.deleteBgpEdgeRouterAdvertisementQueue.Add(key) + return + } + + if !reflect.DeepEqual(oldRouter.Spec, newRouter.Spec) { + klog.Infof("enqueue update bgp-edge-router-advertisement %s", key) + c.updateBgpEdgeRouterAdvertisementQueue.Add(updateVer) + } } func (c *Controller) enqueueDeleteBgpEdgeRouterAdvertisement(obj any) { diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index 2a1ee3e1557..f90a04438cd 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -60,12 +60,11 @@ func (c *Controller) enqueueUpdateGobgpConfig(oldObj, newObj any) { klog.Infof("enqueue update gobgp-config %s", key) c.updateGobgpConfigQueue.Add(updateConfigVer) } - - // c.updateGobgpConfigQueue.Add(updateConfigVer) } func (c *Controller) enqueueDeleteGobgpConfig(obj any) { var gobgpConfig *kubeovnv1.GobgpConfig + switch t := obj.(type) { case *kubeovnv1.GobgpConfig: gobgpConfig = t @@ -74,11 +73,14 @@ func (c *Controller) enqueueDeleteGobgpConfig(obj any) { gobgpConfig = v } } + if gobgpConfig == nil { klog.Warning("enqueueDeleteGobgpConfig: object is not GobgpConfig") return } + key := cache.MetaObjectToName(obj.(*kubeovnv1.GobgpConfig)).String() + klog.V(3).Infof("enqueue delete gobgp-config %s", key) c.deleteGobgpConfigQueue.Add(key) } From a5002365f41aad5c5715249f2e56f49007c27bca Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 18 Aug 2025 01:14:02 -0700 Subject: [PATCH 52/66] ClusterRole modified. --- charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml | 6 ++++++ charts/kube-ovn/templates/ovn-CR.yaml | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml b/charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml index e1f0b05fd66..82f34b1a461 100644 --- a/charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml +++ b/charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml @@ -50,6 +50,12 @@ rules: - vpc-dnses/status - qos-policies - qos-policies/status + - bgp-edge-routers + - bgp-edge-routers/status + - bgp-edge-router-advertisements + - bgp-edge-router-advertisements/status + - gobgp-configs + - gobgp-configs/status verbs: - "*" - apiGroups: diff --git a/charts/kube-ovn/templates/ovn-CR.yaml b/charts/kube-ovn/templates/ovn-CR.yaml index c5f54a4ecee..061cea683f4 100644 --- a/charts/kube-ovn/templates/ovn-CR.yaml +++ b/charts/kube-ovn/templates/ovn-CR.yaml @@ -52,6 +52,10 @@ rules: - qos-policies/status - bgp-edge-routers - bgp-edge-routers/status + - bgp-edge-router-advertisements + - bgp-edge-router-advertisements/status + - gobgp-configs + - gobgp-configs/status verbs: - "*" - apiGroups: From 8abc6e122918e785ff64d6d324ce0433ff748455 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Mon, 18 Aug 2025 21:14:58 -0700 Subject: [PATCH 53/66] When configs are not synced, gobgp sync logic is modfied. --- dist/images/update-bgp-policy.sh | 13 ++ pkg/controller/gobgp_config.go | 204 ++++++++++++++++++++++++++++++- 2 files changed, 215 insertions(+), 2 deletions(-) diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh index de93a41e691..696b4fedf6a 100644 --- a/dist/images/update-bgp-policy.sh +++ b/dist/images/update-bgp-policy.sh @@ -325,6 +325,10 @@ execute_single_command() { [[ $# -ne 1 ]] && die "set-default-action requires exactly 1 argument (accept|reject)" set_default_action "$1" ;; + list-global-policy) + [[ $# -ne 0 ]] && die "list-global-policy requires exactly no argument" + list_global_policy + ;; *) die "Unknown command: $cmd" ;; @@ -373,6 +377,15 @@ parse_batch_commands() { done } +list_global_policy() { + echo "=== Global Policy ===" + $GOBGP_BIN global policy + echo "" + echo "=== Policy Prefix ===" + $GOBGP_BIN policy prefix +} + + main() { [[ $# -lt 1 ]] && usage diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index e8dc68d95f3..582e0a86f04 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -293,6 +293,7 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo // so we need to set default action to reject cmdArs = append(cmdArs, "--", "set-default-action", "reject") } + if newGobgpConfig != nil { for _, neighbor := range newGobgpConfig.Spec.Neighbors { klog.Infof("new bgp config neighbor %v", neighbor) @@ -434,6 +435,38 @@ func (c *Controller) updateGobgpConfigStatus(gobgpConfig *kubeovnv1.GobgpConfig) return updatedGobgpConfig, nil } +func (c *Controller) execGetCmd(pod *corev1.Pod, cmdArs []string) (string, error) { + cmd := fmt.Sprintf("bash /kube-ovn/update-bgp-policy.sh %s", strings.Join(cmdArs, " ")) + + klog.Infof("exec command : %s", cmd) + stdOutput, errOutput, err := util.ExecuteCommandInContainer(c.config.KubeClient, c.config.KubeRestConfig, pod.Namespace, pod.Name, "bgp-router-speaker", []string{"/bin/bash", "-c", cmd}...) + if err != nil { + if len(errOutput) > 0 { + klog.Errorf("failed to ExecuteCommandInContainer, errOutput: %v", errOutput) + } + if len(stdOutput) > 0 { + klog.Infof("failed to ExecuteCommandInContainer, stdOutput: %v", stdOutput) + } + klog.Error(err) + return "", err + } + + cmdSuccess := false + if len(stdOutput) > 0 { + klog.Infof("ExecuteCommandInContainer stdOutput: %v", stdOutput) + if strings.Contains(stdOutput, "Update bgp policy completed successfully") { + cmdSuccess = true + } + } + + if len(errOutput) > 0 && !cmdSuccess { + klog.Errorf("failed to ExecuteCommandInContainer errOutput: %v", errOutput) + return "", errors.New(errOutput) + } + + return stdOutput, nil +} + func (c *Controller) execCmd(pod *corev1.Pod, cmdArs []string) error { cmd := fmt.Sprintf("bash /kube-ovn/update-bgp-policy.sh --batch %s", strings.Join(cmdArs, " ")) @@ -472,6 +505,7 @@ func containsNeighbor(neighbors []string, address string) bool { func (c *Controller) resyncBgpPolicyRules() { klog.Info("resync bgp edge router") + // resync all bgp edge routers gobgpConfigs, err := c.gobgpConfigLister.List(labels.Everything()) if err != nil { @@ -480,7 +514,6 @@ func (c *Controller) resyncBgpPolicyRules() { } for _, gobgpConfig := range gobgpConfigs { - // Check router.Spec.BGP.AdvertisedRoutes same with pods bgp advertised routes if err := c.syncGobgpPolicy(gobgpConfig); err != nil { klog.Errorf("failed to sync advertised routes for bgp edge router %s: %v", gobgpConfig.Name, err) continue @@ -513,13 +546,180 @@ func (c *Controller) syncGobgpPolicy(gobgpConfig *kubeovnv1.GobgpConfig) error { if len(pod.Status.PodIPs) == 0 { continue } - err = c.execUpdateBgpPolicy(key, pod, cachedGobgpConfig, cachedGobgpConfig) + + // execGetBgpPolicy + // if + cmdArs := []string{"list-global-policy"} + output, err := c.execGetCmd(pod, cmdArs) if err != nil { + klog.Error(err) return err } + + validate, err := c.validateSyncGobgpConfig(output, cachedGobgpConfig) + if !validate { + err = c.execUpdateBgpPolicy(key, pod, cachedGobgpConfig, cachedGobgpConfig) + if err != nil { + return err + } + } else if err != nil { + return err + } + klog.Infof("router pod %s/%s policy: %v", pod.Namespace, pod.Name, gobgpConfig) } klog.Infof("finished sync bgp-edge-router %s advertised routes", key) return nil } + +func (c *Controller) validateSyncGobgpConfig(output string, gobgpConfig *kubeovnv1.GobgpConfig) (bool, error) { + klog.Infof("output: %s", output) + // Parse the output to verify that all neighbors, statements, and prefixes in gobgpConfig.Spec.Neighbors exist in the output. + for _, neighbor := range gobgpConfig.Spec.Neighbors { + nbrIP := neighbor.Address + if nbrIP == "" { + klog.Warningf("neighbor address is empty for gobgpConfig %s", gobgpConfig.Name) + return false, errors.New("neighbor address is empty") + } + + // Check Import policy statement and prefix + var inPrefixes, outPrefixes []string + inPrefixName := fmt.Sprintf("prefix-%s-in", nbrIP) + outPrefixName := fmt.Sprintf("prefix-%s-out", nbrIP) + + if neighbor.ToReceive.Allowed.Mode == "all" || neighbor.ToReceive.Allowed.Mode == "filtered" { + importPolicy := fmt.Sprintf("policy-%s-in", nbrIP) + importStmt := fmt.Sprintf("stmt-%s-in", nbrIP) + importPrefix := fmt.Sprintf("prefix-%s-in", nbrIP) + if !strings.Contains(output, importPolicy) || + !strings.Contains(output, importStmt) || + !strings.Contains(output, importPrefix) { + klog.Warningf("missing import policy/statement/prefix for neighbor %s", nbrIP) + return false, nil + } + } + + if neighbor.ToAdvertise.Allowed.Mode == "all" || neighbor.ToAdvertise.Allowed.Mode == "filtered" { + // Check Export policy statement and prefix + exportPolicy := fmt.Sprintf("policy-%s-out", nbrIP) + exportStmt := fmt.Sprintf("stmt-%s-out", nbrIP) + exportPrefix := fmt.Sprintf("prefix-%s-out", nbrIP) + if !strings.Contains(output, exportPolicy) || + !strings.Contains(output, exportStmt) || + !strings.Contains(output, exportPrefix) { + klog.Warningf("missing export policy/statement/prefix for neighbor %s", nbrIP) + return false, nil + } + } + + // Parse global policy prefix lines for this neighbor + // Parse only the lines after "=== Global Policy Prefix ===" + + lines := strings.Split(output, "\n") + startIdx := 0 + for i, line := range lines { + if strings.Contains(line, "=== Policy Prefix ===") { + startIdx = i + 2 + break + } + } + + var inDir bool + if strings.HasPrefix(lines[startIdx], inPrefixName) { + inDir = true + } else { + inDir = false + } + + for _, line := range lines[startIdx:] { + line = strings.TrimSpace(line) + // klog.Infof("line: %v, startIndex: %d", line, startIdx) + + if strings.Contains(line, "Update bgp policy completed successfully") { + break + } + + if strings.HasPrefix(line, inPrefixName) || inDir { + // klog.Infof("line: %v", line) + prefixPart := strings.TrimPrefix(line, inPrefixName+" ") + // klog.Infof("prefixPart: %v", prefixPart) + inPrefixes = append(inPrefixes, prefixPart) + if strings.HasPrefix(lines[startIdx+1], outPrefixName) { + inDir = false + } + } else if strings.HasPrefix(line, outPrefixName) || !inDir { + klog.Infof("line: %v", line) + prefixPart := strings.TrimPrefix(line, outPrefixName+" ") + klog.Infof("prefixPart: %v", prefixPart) + outPrefixes = append(outPrefixes, prefixPart) + if strings.HasPrefix(lines[startIdx+1], inPrefixName) { + inDir = true + } + } + } + klog.Warningf("inPrefixes: %v, outPrefixes: %v", inPrefixes, outPrefixes) + + // Check advertised prefixes (out) + if neighbor.ToAdvertise.Allowed.Mode == "filtered" { + for _, p := range neighbor.ToAdvertise.Allowed.Prefixes { + found := false + for _, out := range outPrefixes { + if strings.Contains(out, p) { + found = true + break + } + } + if !found { + klog.Warningf("missing advertised prefix %s for neighbor %s", p, nbrIP) + return false, nil + } + } + } else { + // Mode "all": should contain 0.0.0.0/0 0..32 + found := false + for _, out := range outPrefixes { + if strings.Contains(out, "0.0.0.0/0") && strings.Contains(out, "0..32") { + found = true + break + } + } + if !found { + klog.Warningf("missing advertised prefix 0.0.0.0/0 0..32 for neighbor %s", nbrIP) + return false, nil + } + } + + // Check received prefixes (in) + if neighbor.ToReceive.Allowed.Mode == "filtered" { + for _, p := range neighbor.ToReceive.Allowed.Prefixes { + found := false + for _, in := range inPrefixes { + if strings.Contains(in, p) { + found = true + break + } + } + if !found { + klog.Warningf("missing received prefix %s for neighbor %s", p, nbrIP) + return false, nil + } + } + } else { + // Mode "all": should contain 0.0.0.0/0 0..32 + found := false + for _, in := range inPrefixes { + if strings.Contains(in, "0.0.0.0/0") && strings.Contains(in, "0..32") { + found = true + break + } + } + if !found { + klog.Warningf("missing received prefix 0.0.0.0/0 0..32 for neighbor %s", nbrIP) + return false, nil + } + } + } + klog.Infof("Sync is not needed.") + return true, nil +} From 7d60b35d5d5a941e53593195982e2b57d5fe2553 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Tue, 19 Aug 2025 01:19:39 -0700 Subject: [PATCH 54/66] gobgp-config log is deleted. --- pkg/controller/gobgp_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index 582e0a86f04..c5b98b409bb 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -566,7 +566,7 @@ func (c *Controller) syncGobgpPolicy(gobgpConfig *kubeovnv1.GobgpConfig) error { return err } - klog.Infof("router pod %s/%s policy: %v", pod.Namespace, pod.Name, gobgpConfig) + // klog.Infof("router pod %s/%s policy: %v", pod.Namespace, pod.Name, gobgpConfig) } klog.Infof("finished sync bgp-edge-router %s advertised routes", key) From 4d85b635cd8e1a7f0c944684a8ec4894d45e155b Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Wed, 20 Aug 2025 00:55:04 -0700 Subject: [PATCH 55/66] bgp edge router test code added. --- test/e2e/bgp-edge-router/e2e_test.go | 544 ++++++++++++++++++++++++++ test/e2e/framework/bgp-edge-router.go | 189 +++++++++ test/e2e/framework/util.go | 33 ++ 3 files changed, 766 insertions(+) create mode 100644 test/e2e/bgp-edge-router/e2e_test.go create mode 100644 test/e2e/framework/bgp-edge-router.go diff --git a/test/e2e/bgp-edge-router/e2e_test.go b/test/e2e/bgp-edge-router/e2e_test.go new file mode 100644 index 00000000000..7003c26ee22 --- /dev/null +++ b/test/e2e/bgp-edge-router/e2e_test.go @@ -0,0 +1,544 @@ +package bgpedgerouter + +import ( + "context" + "flag" + "fmt" + "maps" + "math/rand/v2" + "net" + "reflect" + "strconv" + "strings" + "testing" + + dockernetwork "github.com/docker/docker/api/types/network" + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/component-base/logs" + "k8s.io/klog/v2" + commontest "k8s.io/kubernetes/test/e2e/common" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/test/e2e/framework" + "github.com/kubeovn/kube-ovn/test/e2e/framework/docker" + "github.com/kubeovn/kube-ovn/test/e2e/framework/kind" +) + +func init() { + klog.SetOutput(ginkgo.GinkgoWriter) + + // Register flags. + config.CopyFlags(config.Flags, flag.CommandLine) + k8sframework.RegisterCommonFlags(flag.CommandLine) + k8sframework.RegisterClusterFlags(flag.CommandLine) +} + +func TestE2E(t *testing.T) { + k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) + + logs.InitLogs() + defer logs.FlushLogs() + klog.EnableContextualLogging(true) + + gomega.RegisterFailHandler(k8sframework.Fail) + + // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins + suiteConfig, reporterConfig := k8sframework.CreateGinkgoConfig() + klog.Infof("Starting e2e run %q on Ginkgo node %d", k8sframework.RunID, suiteConfig.ParallelProcess) + ginkgo.RunSpecs(t, "Kube-OVN e2e suite", suiteConfig, reporterConfig) +} + +const ( + kindNetwork = "kind" + + controlPlaneLabel = "node-role.kubernetes.io/control-plane" +) + +var clusterName string + +var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { + // Reference common test to make the import valid. + commontest.CurrentSuite = commontest.E2E + + cs, err := k8sframework.LoadClientset() + framework.ExpectNoError(err) + + ginkgo.By("Getting k8s nodes") + k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) + framework.ExpectNoError(err) + + var ok bool + if clusterName, ok = kind.IsKindProvided(k8sNodes.Items[0].Spec.ProviderID); !ok { + ginkgo.Fail("bgp-edge-router spec only runs on kind clusters") + } + + return []byte(clusterName) +}, func(data []byte) { + clusterName = string(data) +}) + +var _ = framework.Describe("[group:ber]", func() { + f := framework.NewDefaultFramework("ber") + + var vpcClient *framework.VpcClient + var subnetClient *framework.SubnetClient + var nadClient *framework.NetworkAttachmentDefinitionClient + var nadName, externalSubnetName, namespaceName string + var schedulableNodes []corev1.Node + var replicas int32 + ginkgo.BeforeEach(func() { + namespaceName = f.Namespace.Name + nadName = "nad-" + framework.RandomSuffix() + externalSubnetName = "ext-" + framework.RandomSuffix() + vpcClient = f.VpcClient() + subnetClient = f.SubnetClient() + nadClient = f.NetworkAttachmentDefinitionClient() + + nodeList, err := e2enode.GetReadyNodesIncludingTainted(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + framework.ExpectNotEmpty(nodeList.Items) + + nodeList, err = e2enode.GetReadySchedulableNodes(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + framework.ExpectNotEmpty(nodeList.Items) + schedulableNodes = nodeList.Items + + replicas = min(int32(len(schedulableNodes)), 3) + }) + + // framework.ConformanceIt("should be able to create bgp-edge-router with underlay subnet", func() { + // provider := fmt.Sprintf("%s.%s.%s", nadName, namespaceName, util.OvnProvider) + + // //Creating nad. + // ginkgo.By("Creating network attachment definition " + nadName) + // nad := framework.MakeMacvlanNetworkAttachmentDefinition(nadName, namespaceName, "eth0", "bridge", provider, nil) + // ginkgo.DeferCleanup(func() { + // ginkgo.By("Deleting network attachment definition " + nadName) + // nadClient.Delete(nadName) + // }) + // nad = nadClient.Create(nad) + // framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + // dockerNetworkName := "net-" + framework.RandomSuffix() + // ginkgo.By("Creating docker network " + dockerNetworkName) + // dockerNetwork, err := docker.NetworkCreate(dockerNetworkName, true, true) + // framework.ExpectNoError(err, "creating docker network "+dockerNetworkName) + // ginkgo.DeferCleanup(func() { + // ginkgo.By("Deleting docker network " + dockerNetworkName) + // err = docker.NetworkRemove(dockerNetworkName) + // framework.ExpectNoError(err, "removing docker network "+dockerNetworkName) + // }) + + // ginkgo.By("Getting kind nodes") + // kindNodes, err := kind.ListNodes(clusterName, "") + // framework.ExpectNoError(err, "getting nodes in kind cluster") + // framework.ExpectNotEmpty(nodes) + + // ginkgo.By("Connecting nodes to the docker network") + // err = kind.NetworkConnect(dockerNetwork.ID, kindNodes) + // framework.ExpectNoError(err, "connecting nodes to network "+dockerNetworkName) + // ginkgo.DeferCleanup(func() { + // err = kind.NetworkDisconnect(dockerNetwork.ID, kindNodes) + // framework.ExpectNoError(err, "disconnecting nodes from network "+dockerNetworkName) + // }) + + // ginkgo.By("Getting node links that belong to the docker network") + // kindNodes, err = kind.ListNodes(clusterName, "") + // framework.ExpectNoError(err, "getting nodes in kind cluster") + // linkMap := make(map[string]*iproute.Link, len(nodes)) + // for _, node := range kindNodes { + // links, err := node.ListLinks() + // framework.ExpectNoError(err, "failed to list links on node %s: %v", node.Name(), err) + + // for _, link := range links { + // if link.Address == node.NetworkSettings.Networks[dockerNetworkName].MacAddress { + // linkMap[node.Name()] = &link + // break + // } + // } + // framework.ExpectHaveKey(linkMap, node.Name()) + // } + + // ginkgo.By("Getting docker network " + dockerNetworkName) + // network, err := docker.NetworkInspect(dockerNetworkName) + // framework.ExpectNoError(err, "getting docker network "+dockerNetworkName) + + // externalSubnet := generateSubnetFromDockerNetwork(externalSubnetName, network, f.HasIPv4(), f.HasIPv6()) + // externalSubnet.Spec.Provider = provider + + // ginkgo.By("Creating underlay subnet " + externalSubnetName) + // ginkgo.DeferCleanup(func() { + // ginkgo.By("Deleting external subnet " + externalSubnetName) + // subnetClient.DeleteSync(externalSubnetName) + // }) + // _ = subnetClient.CreateSync(externalSubnet) + + // vpcName := util.DefaultVpc + // cidr := framework.RandomCIDR(f.ClusterIPFamily) + // bfdIP := framework.RandomIPs(cidr, ";", 1) + // ginkgo.By("Enabling BFD Port with IP " + bfdIP + " for VPC " + vpcName) + // vpc := vpcClient.Get(vpcName) + // patchedVpc := vpc.DeepCopy() + // patchedVpc.Spec.BFDPort = &apiv1.BFDPort{ + // Enabled: true, + // IP: bfdIP, + // NodeSelector: &metav1.LabelSelector{ + // MatchExpressions: []metav1.LabelSelectorRequirement{{ + // Key: controlPlaneLabel, + // Operator: metav1.LabelSelectorOpExists, + // }}, + // }, + // } + // updatedVpc := vpcClient.PatchSync(vpc, patchedVpc, 10*time.Second) + // ginkgo.DeferCleanup(func() { + // ginkgo.By("Disabling BFD Port for VPC " + vpcName) + // patchedVpc := updatedVpc.DeepCopy() + // patchedVpc.Spec.BFDPort = nil + // updatedVpc := vpcClient.PatchSync(updatedVpc, patchedVpc, 10*time.Second) + // framework.ExpectEmpty(updatedVpc.Status.BFDPort.Name) + // framework.ExpectEmpty(updatedVpc.Status.BFDPort.Nodes) + // }) + + // framework.ExpectNotEmpty(updatedVpc.Status.BFDPort.Name) + // for _, node := range nodes { + // if slices.Contains(updatedVpc.Status.BFDPort.Nodes, node.Name) { + // framework.ExpectHaveKey(node.Labels, controlPlaneLabel) + // } else { + // framework.ExpectNotHaveKey(node.Labels, controlPlaneLabel) + // } + // } + + // // TODO: check ovn LRP + + // vegTest(f, true, provider, nadName, vpcName, vpc.Status.DefaultLogicalSwitch, externalSubnetName, replicas) + // }) + + framework.ConformanceIt("should be able to create bgp-edge-router with macvlan", func() { + provider := fmt.Sprintf("%s.%s", nadName, namespaceName) + + ginkgo.By("Creating network attachment definition " + nadName) + nad := framework.MakeMacvlanNetworkAttachmentDefinition(nadName, namespaceName, "eth0", "bridge", provider, nil) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting network attachment definition " + nadName) + nadClient.Delete(nadName) + }) + nad = nadClient.Create(nad) + framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + vpcName := "vpc-" + framework.RandomSuffix() + ginkgo.By("Creating vpc " + vpcName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting vpc " + vpcName) + vpcClient.DeleteSync(vpcName) + }) + vpc := &apiv1.Vpc{ObjectMeta: metav1.ObjectMeta{Name: vpcName}} + vpc = vpcClient.CreateSync(vpc) + framework.ExpectEmpty(vpc.Status.BFDPort.Name) + framework.ExpectEmpty(vpc.Status.BFDPort.IP) + framework.ExpectEmpty(vpc.Status.BFDPort.Nodes) + + internalSubnetName := "int-" + framework.RandomSuffix() + ginkgo.By("Creating internal subnet " + internalSubnetName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting internal subnet " + internalSubnetName) + subnetClient.DeleteSync(internalSubnetName) + }) + cidr := framework.RandomCIDR(f.ClusterIPFamily) + internalGateway := framework.CIDRGatewayIP(cidr) + internalSubnet := framework.MakeSubnet(internalSubnetName, "", cidr, internalGateway, vpcName, provider, nil, nil, nil) + _ = subnetClient.CreateSync(internalSubnet) + + ginkgo.By("Getting docker network " + kindNetwork) + network, err := docker.NetworkInspect(kindNetwork) + framework.ExpectNoError(err, "getting docker network "+kindNetwork) + + externalSubnet := generateSubnetFromDockerNetwork(externalSubnetName, network, f.HasIPv4(), f.HasIPv6()) + externalSubnet.Spec.Provider = provider + + ginkgo.By("Creating macvlan subnet " + externalSubnetName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting external subnet " + externalSubnetName) + subnetClient.DeleteSync(externalSubnetName) + }) + _ = subnetClient.CreateSync(externalSubnet) + + vegTest(f, true, true, provider, nadName, vpcName, internalSubnetName, externalSubnetName, replicas) + }) +}) + +func generateSubnetFromDockerNetwork(subnetName string, network *dockernetwork.Inspect, ipv4, ipv6 bool) *apiv1.Subnet { + ginkgo.GinkgoHelper() + + ginkgo.By("Generating subnet configuration from docker network " + network.Name) + var cidrV4, cidrV6, gatewayV4, gatewayV6 string + for _, config := range network.IPAM.Config { + switch util.CheckProtocol(config.Subnet) { + case apiv1.ProtocolIPv4: + if ipv4 { + cidrV4 = config.Subnet + gatewayV4 = config.Gateway + } + case apiv1.ProtocolIPv6: + if ipv6 { + cidrV6 = config.Subnet + if gatewayV6 = config.Gateway; gatewayV6 == "" { + var err error + gatewayV6, err = util.FirstIP(cidrV6) + framework.ExpectNoError(err) + } + } + } + } + + cidr := make([]string, 0, 2) + gateway := make([]string, 0, 2) + if ipv4 { + cidr = append(cidr, cidrV4) + gateway = append(gateway, gatewayV4) + } + if ipv6 { + cidr = append(cidr, cidrV6) + gateway = append(gateway, gatewayV6) + } + + excludeIPs := make([]string, 0, len(network.Containers)*2) + for _, container := range network.Containers { + if container.IPv4Address != "" && ipv4 { + excludeIPs = append(excludeIPs, strings.Split(container.IPv4Address, "/")[0]) + } + if container.IPv6Address != "" && ipv6 { + excludeIPs = append(excludeIPs, strings.Split(container.IPv6Address, "/")[0]) + } + } + + return framework.MakeSubnet(subnetName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", "", excludeIPs, nil, nil) +} + +func checkBgpEdgeRouterAccess(f *framework.Framework, namespaceName, svrPodName, image, svrPort string, svrIPs, extIPs []string, intIPs map[string][]string, subnetName, nodeName string, snat bool) { + ginkgo.GinkgoHelper() + + podName := "pod-" + framework.RandomSuffix() + ginkgo.By("Creating client pod " + podName + " within subnet " + subnetName) + labels := map[string]string{"snat": strconv.FormatBool(snat)} + annotations := map[string]string{util.LogicalSwitchAnnotation: subnetName} + pod := framework.MakePrivilegedPod(namespaceName, podName, labels, annotations, image, []string{"sleep", "infinity"}, nil) + pod.Spec.NodeName = nodeName + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting pod " + podName) + f.PodClient().DeleteSync(podName) + }) + pod = f.PodClient().CreateSync(pod) + + if !snat { + // skip bgp edge router route check if SNAT is enabled + // traceroute does not work for pods selected by the selectors + var hops []string + if nodeName == "" { + for ips := range maps.Values(intIPs) { + hops = append(hops, ips...) + } + } else { + hops = intIPs[nodeName] + } + framework.CheckPodEgressRoutes(pod.Namespace, pod.Name, f.HasIPv4(), f.HasIPv6(), 2, hops) + } + + if !snat { + podIPv4, podIPv6 := util.SplitIpsByProtocol(util.PodIPs(*pod)) + hopsIPv4, hopsIPv6 := util.SplitIpsByProtocol(extIPs) + addEcmpRoutes(namespaceName, svrPodName, podIPv4, hopsIPv4) + addEcmpRoutes(namespaceName, svrPodName, podIPv6, hopsIPv6) + } + + expectedClientIPs := extIPs + if !snat { + expectedClientIPs = util.PodIPs(*pod) + } + for _, svrIP := range svrIPs { + protocol := strings.ToLower(util.CheckProtocol(svrIP)) + ginkgo.By("Checking connection from " + pod.Name + " to " + svrIP + " via " + protocol) + cmd := fmt.Sprintf("curl -q -s --connect-timeout 2 --max-time 2 %s/clientip", net.JoinHostPort(svrIP, svrPort)) + ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, cmd, pod.Namespace, pod.Name)) + output := e2epodoutput.RunHostCmdOrDie(pod.Namespace, pod.Name, cmd) + clientIP, _, err := net.SplitHostPort(strings.TrimSpace(output)) + framework.ExpectNoError(err) + framework.ExpectContainElement(expectedClientIPs, clientIP) + } +} + +func addEcmpRoutes(namespaceName, podName string, destinations, nextHops []string) { + ginkgo.GinkgoHelper() + + if len(destinations) == 0 || len(nextHops) == 0 { + return + } + + var args string + if len(nextHops) == 1 { + args = " via " + nextHops[0] + } else { + for _, ip := range nextHops { + args += fmt.Sprintf(" nexthop via %s dev net1 weight 1", ip) + } + } + for _, dst := range destinations { + cmd := fmt.Sprintf("ip route add %s%s", dst, args) + output, err := e2epodoutput.RunHostCmd(namespaceName, podName, cmd) + framework.ExpectNoError(err, output) + } +} + +func vegTest(f *framework.Framework, bfd bool, bgp bool, provider, nadName, vpcName, internalSubnetName, externalSubnetName string, replicas int32) { + ginkgo.GinkgoHelper() + + namespaceName := f.Namespace.Name + // snatSubnetName := "snat-" + framework.RandomSuffix() + // forwardSubnetName := "forward-" + framework.RandomSuffix() + // subnetClient := f.SubnetClient() + berClient := f.BgpEdgeRouterClient() + deployClient := f.DeploymentClient() + podClient := f.PodClient() + + // var forwardSubnet *apiv1.Subnet + // for _, subnetName := range []string{snatSubnetName, forwardSubnetName} { + // ginkgo.By("Creating subnet " + subnetName) + // cidr := framework.RandomCIDR(f.ClusterIPFamily) + // subnet := framework.MakeSubnet(subnetName, "", cidr, "", vpcName, "", nil, nil, nil) + // ginkgo.DeferCleanup(func() { + // ginkgo.By("Deleting subnet " + subnetName) + // subnetClient.DeleteSync(subnetName) + // }) + // _ = subnetClient.CreateSync(subnet) + // if subnetName == forwardSubnetName { + // forwardSubnet = subnet + // } + // } + + berName := "ber-" + framework.RandomSuffix() + ginkgo.By("Creating bgp edge router " + berName) + ber := framework.MakeBgpEdgeRouter(namespaceName, berName, vpcName, replicas, internalSubnetName, externalSubnetName) + if rand.Int32N(2) == 0 { + ber.Spec.Prefix = fmt.Sprintf("e2e-%s-", framework.RandomSuffix()) + } + ber.Spec.BFD.Enabled = bfd + ber.Spec.BFD.MinRX = 300 + ber.Spec.BFD.MinTX = 300 + ber.Spec.BFD.Multiplier = 3 + + ber.Spec.Policies = []apiv1.BgpEdgeRouterPolicy{{ + SNAT: false, + Subnets: []string{internalSubnetName}, + // IPBlocks: strings.Split(forwardSubnet.Spec.CIDRBlock, ","), + }} + + ber.Spec.BGP.Enabled = bgp + ber.Spec.BGP.Image = "inyong7635/kube-ovn:v1.15.0" + ber.Spec.BGP.ASN = 65001 + ber.Spec.BGP.RemoteASN = 65002 + ber.Spec.BGP.Neighbors = []string{"10.9.103.190"} + ber.Spec.BGP.RouterID = "10.9.103.189" + ber.Spec.BGP.EnableGracefulRestart = true + + if vpcName == util.DefaultVpc { + ber.Spec.VPC = "" // test whether the ber works without specifying VPC + ber.Spec.TrafficPolicy = apiv1.TrafficPolicyLocal + } + if util.IsOvnProvider(provider) { + ber.Spec.Selectors = []apiv1.BgpEdgeRouterSelector{{ + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + corev1.LabelMetadataName: namespaceName, + }, + }, + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "snat": strconv.FormatBool(true), + }, + }, + }} + } else { + ber.Spec.Policies = append(ber.Spec.Policies, apiv1.BgpEdgeRouterPolicy{ + SNAT: true, + Subnets: []string{internalSubnetName}, + }) + } + + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting bgp edge router " + berName) + berClient.DeleteSync(berName) + }) + ber = berClient.CreateSync(ber) + + ginkgo.By("Validating bgp edge router status") + framework.ExpectTrue(ber.Status.Ready) + framework.ExpectEqual(ber.Status.Phase, apiv1.PhaseCompleted) + framework.ExpectHaveLen(ber.Status.InternalIPs, int(replicas)) + framework.ExpectHaveLen(ber.Status.ExternalIPs, int(replicas)) + + ginkgo.By("Validating bgp edge router workload") + framework.ExpectEqual(ber.Status.Workload.Name, ber.Spec.Prefix+ber.Name) + deploy := deployClient.Get(ber.Status.Workload.Name) + framework.ExpectEqual(deploy.Status.Replicas, replicas) + framework.ExpectEqual(deploy.Status.ReadyReplicas, replicas) + gvk := appsv1.SchemeGroupVersion.WithKind(reflect.TypeOf(deploy).Elem().Name()) + framework.ExpectEqual(ber.Status.Workload.APIVersion, gvk.GroupVersion().String()) + framework.ExpectEqual(ber.Status.Workload.Kind, gvk.Kind) + framework.ExpectHaveLen(ber.Status.Workload.Nodes, int(replicas)) + workloadPods, err := deployClient.GetPods(deploy) + framework.ExpectNoError(err) + framework.ExpectHaveLen(workloadPods.Items, int(replicas)) + podNodes := make([]string, 0, len(workloadPods.Items)) + intIPs := make(map[string][]string, len(workloadPods.Items)) + for _, pod := range workloadPods.Items { + framework.ExpectNotContainElement(podNodes, pod.Spec.NodeName) + podNodes = append(podNodes, pod.Spec.NodeName) + intIPs[pod.Spec.NodeName] = util.PodIPs(pod) + } + framework.ExpectConsistOf(ber.Status.Workload.Nodes, podNodes) + + svrPodName := "svr-" + framework.RandomSuffix() + ginkgo.By("Creating netexec server pod " + svrPodName) + routes := util.NewPodRoutes() + // dstV4, dstV6 := util.SplitStringIP(forwardSubnet.Spec.CIDRBlock) + // gwV4, gwV6 := util.SplitStringIP(ber.Status.ExternalIPs[0]) + // routes.Add(provider, dstV4, gwV4) + // routes.Add(provider, dstV6, gwV6) + annotations, err := routes.ToAnnotations() + framework.ExpectNoError(err) + attachmentNetworkName := fmt.Sprintf("%s/%s", namespaceName, nadName) + annotations[nadv1.NetworkAttachmentAnnot] = attachmentNetworkName + port := strconv.Itoa(8000 + rand.IntN(1000)) + args := []string{"netexec", "--http-port", port} + svrPod := framework.MakePrivilegedPod(namespaceName, svrPodName, nil, annotations, framework.AgnhostImage, nil, args) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting pod " + svrPodName) + podClient.DeleteSync(svrPodName) + }) + svrPod = podClient.CreateSync(svrPod) + svrIPs, err := util.PodAttachmentIPs(svrPod, attachmentNetworkName) + framework.ExpectNoError(err) + + image := workloadPods.Items[0].Spec.Containers[0].Image + extIPs := make([]string, 0, len(ber.Status.ExternalIPs)*2) + for _, ips := range ber.Status.ExternalIPs { + extIPs = append(extIPs, strings.Split(ips, ",")...) + } + + var nodeName string + if ber.Spec.TrafficPolicy == apiv1.TrafficPolicyLocal { + nodeName = ber.Status.Workload.Nodes[0] + } + checkBgpEdgeRouterAccess(f, namespaceName, svrPodName, image, port, svrIPs, extIPs, intIPs, externalSubnetName, nodeName, true) + checkBgpEdgeRouterAccess(f, namespaceName, svrPodName, image, port, svrIPs, extIPs, intIPs, internalSubnetName, nodeName, false) +} diff --git a/test/e2e/framework/bgp-edge-router.go b/test/e2e/framework/bgp-edge-router.go new file mode 100644 index 00000000000..692942429d1 --- /dev/null +++ b/test/e2e/framework/bgp-edge-router.go @@ -0,0 +1,189 @@ +package framework + +import ( + "context" + "errors" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + v1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// BgpEdgeRouterClient is a struct for bgp edge router client. +type BgpEdgeRouterClient struct { + f *Framework + namespace string + v1.BgpEdgeRouterInterface +} + +func NewBgpEdgeRouterClient(cs clientset.Interface, namespapce string) *BgpEdgeRouterClient { + return &BgpEdgeRouterClient{ + namespace: namespapce, + BgpEdgeRouterInterface: cs.KubeovnV1().BgpEdgeRouters(namespapce), + } +} + +func (f *Framework) BgpEdgeRouterClient() *BgpEdgeRouterClient { + return &BgpEdgeRouterClient{ + f: f, + namespace: f.Namespace.Name, + BgpEdgeRouterInterface: f.KubeOVNClientSet.KubeovnV1().BgpEdgeRouters(f.Namespace.Name), + } +} + +func (f *Framework) BgpEdgeRouterClientNS(namespapce string) *BgpEdgeRouterClient { + return &BgpEdgeRouterClient{ + f: f, + namespace: namespapce, + BgpEdgeRouterInterface: f.KubeOVNClientSet.KubeovnV1().BgpEdgeRouters(namespapce), + } +} + +func (c *BgpEdgeRouterClient) Get(name string) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + router, err := c.BgpEdgeRouterInterface.Get(context.TODO(), name, metav1.GetOptions{}) + ExpectNoError(err) + return router +} + +// Create creates a new bgp-edge-router according to the framework specifications +func (c *BgpEdgeRouterClient) Create(router *apiv1.BgpEdgeRouter) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + r, err := c.BgpEdgeRouterInterface.Create(context.TODO(), router, metav1.CreateOptions{}) + ExpectNoError(err, "Error creating bgp-edge-router") + return r.DeepCopy() +} + +// CreateSync creates a new bgp-edge-router according to the framework specifications, and waits for it to be ready. +func (c *BgpEdgeRouterClient) CreateSync(router *apiv1.BgpEdgeRouter) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + _ = c.Create(router) + return c.WaitUntil(router.Name, func(r *apiv1.BgpEdgeRouter) (bool, error) { + return r.Ready(), nil + }, "Ready", 2*time.Second, timeout) +} + +// Patch patches the router +func (c *BgpEdgeRouterClient) Patch(original, modified *apiv1.BgpEdgeRouter) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + + patch, err := util.GenerateMergePatchPayload(original, modified) + ExpectNoError(err) + + var patchedRouter *apiv1.BgpEdgeRouter + err = wait.PollUntilContextTimeout(context.Background(), 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + g, err := c.BgpEdgeRouterInterface.Patch(ctx, original.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "") + if err != nil { + return handleWaitingAPIError(err, false, "patch bgp-edge-router %s/%s", original.Namespace, original.Name) + } + patchedRouter = g + return true, nil + }) + if err == nil { + return patchedRouter.DeepCopy() + } + + if errors.Is(err, context.DeadlineExceeded) { + Failf("timed out while retrying to patch bgp-edge-router %s/%s", original.Namespace, original.Name) + } + Failf("error occurred while retrying to patch bgp-edge-router %s/%s: %v", original.Namespace, original.Name, err) + + return nil +} + +// PatchSync patches the router and waits the router to meet the condition +func (c *BgpEdgeRouterClient) PatchSync(original, modified *apiv1.BgpEdgeRouter) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + _ = c.Patch(original, modified) + return c.WaitUntil(original.Name, func(r *apiv1.BgpEdgeRouter) (bool, error) { + return r.Ready(), nil + }, "Ready", 2*time.Second, timeout) +} + +// Delete deletes a bgp-edge-router if the bgp-edge-router exists +func (c *BgpEdgeRouterClient) Delete(name string) { + ginkgo.GinkgoHelper() + err := c.BgpEdgeRouterInterface.Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + Failf("Failed to delete bgp-edge-router %s/%s: %v", c.namespace, name, err) + } +} + +// DeleteSync deletes the bgp-edge-router and waits for the bgp-edge-router to disappear for `timeout`. +// If the bgp-edge-router doesn't disappear before the timeout, it will fail the test. +func (c *BgpEdgeRouterClient) DeleteSync(name string) { + ginkgo.GinkgoHelper() + c.Delete(name) + gomega.Expect(c.WaitToDisappear(name, 2*time.Second, timeout)).To(gomega.Succeed(), "wait for bgp-edge-router %s/%s to disappear", c.namespace, name) +} + +// WaitUntil waits the given timeout duration for the specified condition to be met. +func (c *BgpEdgeRouterClient) WaitUntil(name string, cond func(r *apiv1.BgpEdgeRouter) (bool, error), condDesc string, interval, timeout time.Duration) *apiv1.BgpEdgeRouter { + var router *apiv1.BgpEdgeRouter + err := wait.PollUntilContextTimeout(context.Background(), interval, timeout, false, func(_ context.Context) (bool, error) { + Logf("Waiting for bgp-edge-router %s/%s to meet condition %q", c.namespace, name, condDesc) + router = c.Get(name).DeepCopy() + met, err := cond(router) + if err != nil { + return false, fmt.Errorf("failed to check condition for bgp-edge-router %s/%s: %w", c.namespace, name, err) + } + if met { + Logf("bgp-edge-router %s/%s met condition %q", c.namespace, name, condDesc) + } else { + Logf("bgp-edge-router %s/%s not met condition %q", c.namespace, name, condDesc) + } + return met, nil + }) + if err == nil { + return router + } + + if errors.Is(err, context.DeadlineExceeded) { + Failf("timed out while waiting for bgp-edge-router %s/%s to meet condition %q", c.namespace, name, condDesc) + } + Failf("error occurred while waiting for bgp-edge-router %s/%s to meet condition %q: %v", c.namespace, name, condDesc, err) + + return nil +} + +// WaitToDisappear waits the given timeout duration for the specified bgp-edge-router to disappear. +func (c *BgpEdgeRouterClient) WaitToDisappear(name string, _, timeout time.Duration) error { + err := framework.Gomega().Eventually(context.Background(), framework.HandleRetry(func(ctx context.Context) (*apiv1.BgpEdgeRouter, error) { + svc, err := c.BgpEdgeRouterInterface.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, nil + } + return svc, err + })).WithTimeout(timeout).Should(gomega.BeNil()) + if err != nil { + return fmt.Errorf("expected bgp-edge-router %s/%s to not be found: %w", c.namespace, name, err) + } + return nil +} + +func MakeBgpEdgeRouter(namespace, name, vpc string, replicas int32, internalSubnet, externalSubnet string) *apiv1.BgpEdgeRouter { + return &apiv1.BgpEdgeRouter{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: apiv1.BgpEdgeRouterSpec{ + Replicas: replicas, + VPC: vpc, + InternalSubnet: internalSubnet, + ExternalSubnet: externalSubnet, + }, + } +} diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index c829cf194b0..3a484828752 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -102,6 +102,39 @@ func RandomCIDR(family string) string { } } +func CIDRGatewayIP(cidr string) string { + ginkgo.GinkgoHelper() + + if cidr == "" { + Failf("cidr is empty") + return "" + } + + _, ipNet, err := net.ParseCIDR(cidr) + ExpectNoError(err) + + ip := ipNet.IP + ones, _ := ipNet.Mask.Size() + if ip.To4() != nil { + // IPv4 + ip = ip.To4() + if ones == 24 { + ip[3] = 1 + } else { + ip[3] = 1 + } + return ip.String() + } else if ip.To16() != nil { + // IPv6 + ip = ip.To16() + ip[15] = 1 + return ip.String() + } else { + Failf("invalid cidr: %s", cidr) + return "" + } +} + func sortIPs(ips []string) { ginkgo.GinkgoHelper() From d24084f412f88d90e45969f7e1dd2f85888e91f0 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Wed, 20 Aug 2025 18:01:30 -0700 Subject: [PATCH 56/66] [fix] edge router e2e test --- dist/images/install.sh | 3 +++ test/e2e/edge-router/e2e_test.go | 2 +- test/e2e/framework/bgp-edge-router.go | 9 +++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/dist/images/install.sh b/dist/images/install.sh index 76f90ef8450..a3c8e08f1fd 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -3737,6 +3737,9 @@ spec: edgeRouterMode: type: boolean default: false + routeServerClient: + type: boolean + default: false enabled: type: boolean default: false diff --git a/test/e2e/edge-router/e2e_test.go b/test/e2e/edge-router/e2e_test.go index 6a38517a007..68c9ad11c5f 100644 --- a/test/e2e/edge-router/e2e_test.go +++ b/test/e2e/edge-router/e2e_test.go @@ -68,7 +68,7 @@ func TestE2E(t *testing.T) { const ( kindNetwork = "kind" - controlPlaneLabel = "node-role.kubernetes.io/control-plane" +// controlPlaneLabel = "node-role.kubernetes.io/control-plane" ) var clusterName string diff --git a/test/e2e/framework/bgp-edge-router.go b/test/e2e/framework/bgp-edge-router.go index c8b2cf30984..b2596a99b31 100644 --- a/test/e2e/framework/bgp-edge-router.go +++ b/test/e2e/framework/bgp-edge-router.go @@ -199,8 +199,13 @@ func MakeBgpEdgeRouter(namespace, name, vpc string, replicas int32, internalSubn }, }, BGP: apiv1.BgpEdgeRouterBGPConfig{ - Enabled: true, - ASN: 65000, + Enabled: true, + ASN: 65000, + EdgeRouterMode: true, + RemoteASN: 65100, + Neighbors: []string{ + "192.168.1.1", + }, EnableGracefulRestart: true, }, }, From 6c867e5ba2a83e4f99dc8f2020503aae594cb3e0 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Thu, 21 Aug 2025 01:56:15 -0700 Subject: [PATCH 57/66] change if-clause to switch --- test/e2e/framework/util.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 3a484828752..e6f534faee9 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -114,22 +114,20 @@ func CIDRGatewayIP(cidr string) string { ExpectNoError(err) ip := ipNet.IP - ones, _ := ipNet.Mask.Size() - if ip.To4() != nil { + // ones, _ := ipNet.Mask.Size() + switch { + case ip.To4() != nil: // IPv4 ip = ip.To4() - if ones == 24 { - ip[3] = 1 - } else { - ip[3] = 1 - } + ip[3] = 1 + return ip.String() - } else if ip.To16() != nil { - // IPv6 + case ip.To16() == nil: + // IPv4-mapped IPv6 ip = ip.To16() ip[15] = 1 return ip.String() - } else { + default: Failf("invalid cidr: %s", cidr) return "" } From bc50e917df0f224519931f714946a0c03543e201 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Thu, 21 Aug 2025 01:57:19 -0700 Subject: [PATCH 58/66] e2e_test.go controlPlaneLabel var commented. --- test/e2e/bgp-edge-router/e2e_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/bgp-edge-router/e2e_test.go b/test/e2e/bgp-edge-router/e2e_test.go index 7003c26ee22..5511dd1f71f 100644 --- a/test/e2e/bgp-edge-router/e2e_test.go +++ b/test/e2e/bgp-edge-router/e2e_test.go @@ -62,7 +62,7 @@ func TestE2E(t *testing.T) { const ( kindNetwork = "kind" - controlPlaneLabel = "node-role.kubernetes.io/control-plane" + // controlPlaneLabel = "node-role.kubernetes.io/control-plane" ) var clusterName string @@ -399,7 +399,7 @@ func addEcmpRoutes(namespaceName, podName string, destinations, nextHops []strin } } -func vegTest(f *framework.Framework, bfd bool, bgp bool, provider, nadName, vpcName, internalSubnetName, externalSubnetName string, replicas int32) { +func vegTest(f *framework.Framework, bfd, bgp bool, provider, nadName, vpcName, internalSubnetName, externalSubnetName string, replicas int32) { ginkgo.GinkgoHelper() namespaceName := f.Namespace.Name From acdc5a1a953876f6c667e24f8216afdbe90d2d5b Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Thu, 21 Aug 2025 19:40:06 -0700 Subject: [PATCH 59/66] [fix] bgp-edge-router e2e test --- pkg/controller/bgp_edge_router.go | 2 +- test/e2e/edge-router/e2e_test.go | 312 ++++++++++++-------------- test/e2e/framework/bgp-edge-router.go | 4 +- 3 files changed, 148 insertions(+), 170 deletions(-) diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go index 99e132121c0..2ad161255ad 100644 --- a/pkg/controller/bgp_edge_router.go +++ b/pkg/controller/bgp_edge_router.go @@ -1143,7 +1143,7 @@ func bgpEdgeRouterContainerBGP(speakerImage, routerName string, speakerParams *k Name: "MULTI_NET_STATUS", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.annotations['k8s.v1.cni.cncf.io/networks-status']", + FieldPath: "metadata.annotations['k8s.v1.cni.cncf.io/network-status']", }, }, }, diff --git a/test/e2e/edge-router/e2e_test.go b/test/e2e/edge-router/e2e_test.go index 68c9ad11c5f..34dcd06d337 100644 --- a/test/e2e/edge-router/e2e_test.go +++ b/test/e2e/edge-router/e2e_test.go @@ -2,6 +2,7 @@ package multus import ( "context" + "encoding/json" "flag" "fmt" "maps" @@ -41,6 +42,51 @@ import ( "github.com/kubeovn/kube-ovn/test/e2e/framework/kind" ) +type GlobalConfig struct { + ASN uint32 `json:"asn"` + RouterID string `json:"router_id"` + ListenPort int `json:"listen_port"` + ListenAddresses []string `json:"listen_addresses"` +} + +type NeighborMessages struct { + Received struct { + Notification uint64 `json:"notification,omitempty"` + Open uint64 `json:"open,omitempty"` + Update uint64 `json:"update,omitempty"` + Keepalive uint64 `json:"keepalive,omitempty"` + Total uint64 `json:"total,omitempty"` + } `json:"received"` + Sent struct { + Open uint64 `json:"open,omitempty"` + Update uint64 `json:"update,omitempty"` + Keepalive uint64 `json:"keepalive,omitempty"` + Total uint64 `json:"total,omitempty"` + } `json:"sent"` +} + +type NeighborConfig struct { + LocalASN uint32 `json:"local_asn"` + NeighborAddress string `json:"neighbor_address"` + PeerASN uint32 `json:"peer_asn"` + Type int `json:"type"` +} + +type NeighborState struct { + LocalASN uint32 `json:"local_asn"` + Messages NeighborMessages `json:"messages"` + NeighborAddress string `json:"neighbor_address"` + PeerASN uint32 `json:"peer_asn"` + Type int `json:"type"` + SessionState int `json:"session_state"` + RouterID string `json:"router_id,omitempty"` +} + +type NeighborEntry struct { + Conf NeighborConfig `json:"conf"` + State NeighborState `json:"state"` +} + func init() { klog.SetOutput(ginkgo.GinkgoWriter) @@ -124,142 +170,6 @@ var _ = framework.Describe("[group:ber]", func() { replicas = min(int32(len(schedulableNodes)), 3) }) - // framework.ConformanceIt("should be able to create edge-router with both underlay and overlay subnet", func() { - // provider := fmt.Sprintf("%s.%s.%s", nadName, namespaceName, util.OvnProvider) - - // ginkgo.By("Creating network attachment definition " + nadName) - // nad := framework.MakeOVNNetworkAttachmentDefinition(nadName, namespaceName, provider, nil) - // ginkgo.DeferCleanup(func() { - // ginkgo.By("Deleting network attachment definition " + nadName) - // nadClient.Delete(nadName) - // }) - // nad = nadClient.Create(nad) - // framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) - - // dockerNetworkName := "net-" + framework.RandomSuffix() - // ginkgo.By("Creating docker network " + dockerNetworkName) - // dockerNetwork, err := docker.NetworkCreate(dockerNetworkName, true, true) - // framework.ExpectNoError(err, "creating docker network "+dockerNetworkName) - // ginkgo.DeferCleanup(func() { - // ginkgo.By("Deleting docker network " + dockerNetworkName) - // err = docker.NetworkRemove(dockerNetworkName) - // framework.ExpectNoError(err, "removing docker network "+dockerNetworkName) - // }) - - // ginkgo.By("Getting kind nodes") - // kindNodes, err := kind.ListNodes(clusterName, "") - // framework.ExpectNoError(err, "getting nodes in kind cluster") - // framework.ExpectNotEmpty(nodes) - - // ginkgo.By("Connecting nodes to the docker network") - // err = kind.NetworkConnect(dockerNetwork.ID, kindNodes) - // framework.ExpectNoError(err, "connecting nodes to network "+dockerNetworkName) - // ginkgo.DeferCleanup(func() { - // err = kind.NetworkDisconnect(dockerNetwork.ID, kindNodes) - // framework.ExpectNoError(err, "disconnecting nodes from network "+dockerNetworkName) - // }) - - // ginkgo.By("Getting node links that belong to the docker network") - // kindNodes, err = kind.ListNodes(clusterName, "") - // framework.ExpectNoError(err, "getting nodes in kind cluster") - // linkMap := make(map[string]*iproute.Link, len(nodes)) - // for _, node := range kindNodes { - // links, err := node.ListLinks() - // framework.ExpectNoError(err, "failed to list links on node %s: %v", node.Name(), err) - - // for _, link := range links { - // if link.Address == node.NetworkSettings.Networks[dockerNetworkName].MacAddress { - // linkMap[node.Name()] = &link - // break - // } - // } - // framework.ExpectHaveKey(linkMap, node.Name()) - // } - - // providerNetworkName := "pn-" + framework.RandomSuffix() - // ginkgo.By("Creating provider network " + providerNetworkName) - // providerNetworkClient := f.ProviderNetworkClient() - // ginkgo.DeferCleanup(func() { - // ginkgo.By("Deleting provider network " + providerNetworkName) - // providerNetworkClient.DeleteSync(providerNetworkName) - // }) - // var defaultInterface string - // customInterfaces := make(map[string][]string, 0) - // for node, link := range linkMap { - // if defaultInterface == "" { - // defaultInterface = link.IfName - // } else if link.IfName != defaultInterface { - // customInterfaces[link.IfName] = append(customInterfaces[link.IfName], node) - // } - // } - // pn := framework.MakeProviderNetwork(providerNetworkName, false, defaultInterface, customInterfaces, nil) - // _ = providerNetworkClient.CreateSync(pn) - - // vlanName := "vlan-" + framework.RandomSuffix() - // ginkgo.By("Creating vlan " + vlanName) - // vlanClient := f.VlanClient() - // vlan := framework.MakeVlan(vlanName, providerNetworkName, 0) - // _ = vlanClient.Create(vlan) - // ginkgo.DeferCleanup(func() { - // ginkgo.By("Deleting vlan " + vlanName) - // vlanClient.Delete(vlanName) - // }) - - // ginkgo.By("Getting docker network " + dockerNetworkName) - // network, err := docker.NetworkInspect(dockerNetworkName) - // framework.ExpectNoError(err, "getting docker network "+dockerNetworkName) - - // externalSubnet := generateSubnetFromDockerNetwork(externalSubnetName, network, f.HasIPv4(), f.HasIPv6()) - // externalSubnet.Spec.Provider = provider - // externalSubnet.Spec.Vlan = vlanName - - // ginkgo.By("Creating underlay subnet " + externalSubnetName) - // ginkgo.DeferCleanup(func() { - // ginkgo.By("Deleting external subnet " + externalSubnetName) - // subnetClient.DeleteSync(externalSubnetName) - // }) - // _ = subnetClient.CreateSync(externalSubnet) - - // vpcName := util.DefaultVpc - // cidr := framework.RandomCIDR(f.ClusterIPFamily) - // bfdIP := framework.RandomIPs(cidr, ";", 1) - // ginkgo.By("Enabling BFD Port with IP " + bfdIP + " for VPC " + vpcName) - // vpc := vpcClient.Get(vpcName) - // patchedVpc := vpc.DeepCopy() - // patchedVpc.Spec.BFDPort = &apiv1.BFDPort{ - // Enabled: true, - // IP: bfdIP, - // NodeSelector: &metav1.LabelSelector{ - // MatchExpressions: []metav1.LabelSelectorRequirement{{ - // Key: controlPlaneLabel, - // Operator: metav1.LabelSelectorOpExists, - // }}, - // }, - // } - // updatedVpc := vpcClient.PatchSync(vpc, patchedVpc, 10*time.Second) - // ginkgo.DeferCleanup(func() { - // ginkgo.By("Disabling BFD Port for VPC " + vpcName) - // patchedVpc := updatedVpc.DeepCopy() - // patchedVpc.Spec.BFDPort = nil - // updatedVpc := vpcClient.PatchSync(updatedVpc, patchedVpc, 10*time.Second) - // framework.ExpectEmpty(updatedVpc.Status.BFDPort.Name) - // framework.ExpectEmpty(updatedVpc.Status.BFDPort.Nodes) - // }) - - // framework.ExpectNotEmpty(updatedVpc.Status.BFDPort.Name) - // for _, node := range nodes { - // if slices.Contains(updatedVpc.Status.BFDPort.Nodes, node.Name) { - // framework.ExpectHaveKey(node.Labels, controlPlaneLabel) - // } else { - // framework.ExpectNotHaveKey(node.Labels, controlPlaneLabel) - // } - // } - - // // TODO: check ovn LRP - - // berTest(f, true, provider, nadName, vpcName, vpc.Status.DefaultLogicalSwitch, externalSubnetName, replicas) - // }) - framework.ConformanceIt("should be able to create edge-router with macvlan", func() { provider := fmt.Sprintf("%s.%s", nadName, namespaceName) @@ -273,16 +183,26 @@ var _ = framework.Describe("[group:ber]", func() { framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) vpcName := "vpc-" + framework.RandomSuffix() - ginkgo.By("Creating vpc " + vpcName) + vpcCidr := framework.RandomCIDR(f.ClusterIPFamily) + bfdIP := framework.RandomIPs(vpcCidr, ";", 1) + ginkgo.By("Creating vpc " + vpcName + ", enabling BFD Port with IP " + bfdIP + " for VPC " + vpcName) ginkgo.DeferCleanup(func() { ginkgo.By("Deleting vpc " + vpcName) vpcClient.DeleteSync(vpcName) }) - vpc := &apiv1.Vpc{ObjectMeta: metav1.ObjectMeta{Name: vpcName}} + vpc := &apiv1.Vpc{ + ObjectMeta: metav1.ObjectMeta{ + Name: vpcName, + }, + Spec: apiv1.VpcSpec{ + BFDPort: &apiv1.BFDPort{ + Enabled: true, + IP: bfdIP, + }, + }, + } vpc = vpcClient.CreateSync(vpc) - framework.ExpectEmpty(vpc.Status.BFDPort.Name) - framework.ExpectEmpty(vpc.Status.BFDPort.IP) - framework.ExpectEmpty(vpc.Status.BFDPort.Nodes) + framework.ExpectNotEmpty(vpc.Status.BFDPort.Name) internalSubnetName := "int-" + framework.RandomSuffix() ginkgo.By("Creating internal subnet " + internalSubnetName) @@ -291,7 +211,7 @@ var _ = framework.Describe("[group:ber]", func() { subnetClient.DeleteSync(internalSubnetName) }) cidr := framework.RandomCIDR(f.ClusterIPFamily) - internalSubnet := framework.MakeSubnet(internalSubnetName, "", cidr, "", vpcName, nad.Name, nil, nil, nil) + internalSubnet := framework.MakeSubnet(internalSubnetName, "", cidr, "", vpcName, "", nil, nil, nil) _ = subnetClient.CreateSync(internalSubnet) ginkgo.By("Getting docker network " + kindNetwork) @@ -308,7 +228,7 @@ var _ = framework.Describe("[group:ber]", func() { }) _ = subnetClient.CreateSync(externalSubnet) - berTest(f, false, provider, nadName, vpcName, internalSubnetName, externalSubnetName, replicas) + berTest(f, true, provider, nadName, vpcName, internalSubnetName, externalSubnetName, replicas) }) }) @@ -434,6 +354,76 @@ func addEcmpRoutes(namespaceName, podName string, destinations, nextHops []strin } } +func parseGobgpOutput(output string) (*GlobalConfig, []NeighborEntry, error) { + lines := strings.Split(strings.TrimSpace(output), "\n") + + var globalJSON, neighborJSON string + var foundGlobal bool + + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + // global config starts with { + if strings.HasPrefix(line, "{") && !foundGlobal { + globalJSON = line + foundGlobal = true + } else if strings.HasPrefix(line, "[") { + // neighbor config starts with [ + neighborJSON = line + } + } + + if globalJSON == "" || neighborJSON == "" { + return nil, nil, fmt.Errorf("failed to extract JSON parts from output") + } + + // GlobalConfig + var global GlobalConfig + if err := json.Unmarshal([]byte(globalJSON), &global); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal global config: %v", err) + } + + // NeighborEntry + var neighbors []NeighborEntry + if err := json.Unmarshal([]byte(neighborJSON), &neighbors); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal neighbor config: %v", err) + } + + return &global, neighbors, nil +} + +func checkBgpInitSetting(ber *apiv1.BgpEdgeRouter, output string) { + ginkgo.GinkgoHelper() + + global, neighbors, err := parseGobgpOutput(output) + framework.ExpectNoError(err, "parsing gobgp output") + + remoteAsn := ber.Spec.BGP.RemoteASN + + framework.ExpectEqual(global.ASN, ber.Spec.BGP.ASN, "global ASN mismatch") + if ber.Spec.BGP.RouterID != "" { + framework.ExpectEqual(global.RouterID, ber.Spec.BGP.RouterID, "global Router ID mismatch") + } + // TODO check when ber.Spec.BGP.RouterID is empty + + for _, berNeighborAddr := range ber.Spec.BGP.Neighbors { + matchFound := false + for _, neighbor := range neighbors { + if neighbor.Conf.NeighborAddress == berNeighborAddr { + matchFound = true + framework.ExpectEqual(neighbor.Conf.PeerASN, remoteAsn, "neighbor %s peer ASN %d mismatch", berNeighborAddr, remoteAsn) + break + } + } + if !matchFound { + framework.Failf("neighbor address %s not found", berNeighborAddr) + } + } +} + func berTest(f *framework.Framework, bfd bool, provider, nadName, vpcName, internalSubnetName, externalSubnetName string, replicas int32) { ginkgo.GinkgoHelper() @@ -457,38 +447,15 @@ func berTest(f *framework.Framework, bfd bool, provider, nadName, vpcName, inter berName := "ber-" + framework.RandomSuffix() ginkgo.By("Creating bgp edge router " + berName) - ber := framework.MakeBgpEdgeRouter(namespaceName, berName, vpcName, replicas, internalSubnetName, externalSubnetName) + ber := framework.MakeBgpEdgeRouter(namespaceName, berName, vpcName, replicas, internalSubnetName, externalSubnetName, forwardSubnetName) if rand.Int32N(2) == 0 { ber.Spec.Prefix = fmt.Sprintf("e2e-%s-", framework.RandomSuffix()) } ber.Spec.BFD.Enabled = bfd - ber.Spec.Policies = []apiv1.BgpEdgeRouterPolicy{{ - SNAT: false, - IPBlocks: strings.Split(forwardSubnet.Spec.CIDRBlock, ","), - }} if vpcName == util.DefaultVpc { ber.Spec.VPC = "" // test whether the ber works without specifying VPC ber.Spec.TrafficPolicy = apiv1.TrafficPolicyLocal } - if util.IsOvnProvider(provider) { - ber.Spec.Selectors = []apiv1.BgpEdgeRouterSelector{{ - NamespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - corev1.LabelMetadataName: namespaceName, - }, - }, - PodSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "snat": strconv.FormatBool(true), - }, - }, - }} - } else { - ber.Spec.Policies = append(ber.Spec.Policies, apiv1.BgpEdgeRouterPolicy{ - SNAT: true, - Subnets: []string{forwardSubnetName}, - }) - } ginkgo.DeferCleanup(func() { ginkgo.By("Deleting bgp edge router " + berName) @@ -520,9 +487,20 @@ func berTest(f *framework.Framework, bfd bool, provider, nadName, vpcName, inter framework.ExpectNotContainElement(podNodes, pod.Spec.NodeName) podNodes = append(podNodes, pod.Spec.NodeName) intIPs[pod.Spec.NodeName] = util.PodIPs(pod) + // exec and list + ginkgo.By("Checking bgp setting " + pod.Name) + cmd := fmt.Sprintf("gobgp global -j && gobgp neighbor -j") + ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, cmd, pod.Namespace, pod.Name)) + output := e2epodoutput.RunHostCmdOrDie(pod.Namespace, pod.Name, cmd) + checkBgpInitSetting(ber, output) } framework.ExpectConsistOf(ber.Status.Workload.Nodes, podNodes) + // TODO + // Add route advertisement + + // Add bgp policy + svrPodName := "svr-" + framework.RandomSuffix() ginkgo.By("Creating netexec server pod " + svrPodName) routes := util.NewPodRoutes() diff --git a/test/e2e/framework/bgp-edge-router.go b/test/e2e/framework/bgp-edge-router.go index b2596a99b31..026d335cc49 100644 --- a/test/e2e/framework/bgp-edge-router.go +++ b/test/e2e/framework/bgp-edge-router.go @@ -173,7 +173,7 @@ func (c *BgpEdgeRouterClient) WaitToDisappear(name string, _, timeout time.Durat return nil } -func MakeBgpEdgeRouter(namespace, name, vpc string, replicas int32, internalSubnet, externalSubnet string) *apiv1.BgpEdgeRouter { +func MakeBgpEdgeRouter(namespace, name, vpc string, replicas int32, internalSubnet, externalSubnet, forwardSubnet string) *apiv1.BgpEdgeRouter { return &apiv1.BgpEdgeRouter{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -194,7 +194,7 @@ func MakeBgpEdgeRouter(namespace, name, vpc string, replicas int32, internalSubn { SNAT: false, Subnets: []string{ - internalSubnet, + forwardSubnet, }, }, }, From 00b3ba25b6493ec6dddf6de3c36ef571475b055b Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Thu, 21 Aug 2025 23:50:34 -0700 Subject: [PATCH 60/66] ber test deleted. --- test/e2e/bgp-edge-router/e2e_test.go | 544 --------------------------- 1 file changed, 544 deletions(-) delete mode 100644 test/e2e/bgp-edge-router/e2e_test.go diff --git a/test/e2e/bgp-edge-router/e2e_test.go b/test/e2e/bgp-edge-router/e2e_test.go deleted file mode 100644 index 5511dd1f71f..00000000000 --- a/test/e2e/bgp-edge-router/e2e_test.go +++ /dev/null @@ -1,544 +0,0 @@ -package bgpedgerouter - -import ( - "context" - "flag" - "fmt" - "maps" - "math/rand/v2" - "net" - "reflect" - "strconv" - "strings" - "testing" - - dockernetwork "github.com/docker/docker/api/types/network" - nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/component-base/logs" - "k8s.io/klog/v2" - commontest "k8s.io/kubernetes/test/e2e/common" - k8sframework "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/framework/config" - e2enode "k8s.io/kubernetes/test/e2e/framework/node" - e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" - - "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" - - apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" - "github.com/kubeovn/kube-ovn/pkg/util" - "github.com/kubeovn/kube-ovn/test/e2e/framework" - "github.com/kubeovn/kube-ovn/test/e2e/framework/docker" - "github.com/kubeovn/kube-ovn/test/e2e/framework/kind" -) - -func init() { - klog.SetOutput(ginkgo.GinkgoWriter) - - // Register flags. - config.CopyFlags(config.Flags, flag.CommandLine) - k8sframework.RegisterCommonFlags(flag.CommandLine) - k8sframework.RegisterClusterFlags(flag.CommandLine) -} - -func TestE2E(t *testing.T) { - k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) - - logs.InitLogs() - defer logs.FlushLogs() - klog.EnableContextualLogging(true) - - gomega.RegisterFailHandler(k8sframework.Fail) - - // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins - suiteConfig, reporterConfig := k8sframework.CreateGinkgoConfig() - klog.Infof("Starting e2e run %q on Ginkgo node %d", k8sframework.RunID, suiteConfig.ParallelProcess) - ginkgo.RunSpecs(t, "Kube-OVN e2e suite", suiteConfig, reporterConfig) -} - -const ( - kindNetwork = "kind" - - // controlPlaneLabel = "node-role.kubernetes.io/control-plane" -) - -var clusterName string - -var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { - // Reference common test to make the import valid. - commontest.CurrentSuite = commontest.E2E - - cs, err := k8sframework.LoadClientset() - framework.ExpectNoError(err) - - ginkgo.By("Getting k8s nodes") - k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) - framework.ExpectNoError(err) - - var ok bool - if clusterName, ok = kind.IsKindProvided(k8sNodes.Items[0].Spec.ProviderID); !ok { - ginkgo.Fail("bgp-edge-router spec only runs on kind clusters") - } - - return []byte(clusterName) -}, func(data []byte) { - clusterName = string(data) -}) - -var _ = framework.Describe("[group:ber]", func() { - f := framework.NewDefaultFramework("ber") - - var vpcClient *framework.VpcClient - var subnetClient *framework.SubnetClient - var nadClient *framework.NetworkAttachmentDefinitionClient - var nadName, externalSubnetName, namespaceName string - var schedulableNodes []corev1.Node - var replicas int32 - ginkgo.BeforeEach(func() { - namespaceName = f.Namespace.Name - nadName = "nad-" + framework.RandomSuffix() - externalSubnetName = "ext-" + framework.RandomSuffix() - vpcClient = f.VpcClient() - subnetClient = f.SubnetClient() - nadClient = f.NetworkAttachmentDefinitionClient() - - nodeList, err := e2enode.GetReadyNodesIncludingTainted(context.Background(), f.ClientSet) - framework.ExpectNoError(err) - framework.ExpectNotEmpty(nodeList.Items) - - nodeList, err = e2enode.GetReadySchedulableNodes(context.Background(), f.ClientSet) - framework.ExpectNoError(err) - framework.ExpectNotEmpty(nodeList.Items) - schedulableNodes = nodeList.Items - - replicas = min(int32(len(schedulableNodes)), 3) - }) - - // framework.ConformanceIt("should be able to create bgp-edge-router with underlay subnet", func() { - // provider := fmt.Sprintf("%s.%s.%s", nadName, namespaceName, util.OvnProvider) - - // //Creating nad. - // ginkgo.By("Creating network attachment definition " + nadName) - // nad := framework.MakeMacvlanNetworkAttachmentDefinition(nadName, namespaceName, "eth0", "bridge", provider, nil) - // ginkgo.DeferCleanup(func() { - // ginkgo.By("Deleting network attachment definition " + nadName) - // nadClient.Delete(nadName) - // }) - // nad = nadClient.Create(nad) - // framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) - - // dockerNetworkName := "net-" + framework.RandomSuffix() - // ginkgo.By("Creating docker network " + dockerNetworkName) - // dockerNetwork, err := docker.NetworkCreate(dockerNetworkName, true, true) - // framework.ExpectNoError(err, "creating docker network "+dockerNetworkName) - // ginkgo.DeferCleanup(func() { - // ginkgo.By("Deleting docker network " + dockerNetworkName) - // err = docker.NetworkRemove(dockerNetworkName) - // framework.ExpectNoError(err, "removing docker network "+dockerNetworkName) - // }) - - // ginkgo.By("Getting kind nodes") - // kindNodes, err := kind.ListNodes(clusterName, "") - // framework.ExpectNoError(err, "getting nodes in kind cluster") - // framework.ExpectNotEmpty(nodes) - - // ginkgo.By("Connecting nodes to the docker network") - // err = kind.NetworkConnect(dockerNetwork.ID, kindNodes) - // framework.ExpectNoError(err, "connecting nodes to network "+dockerNetworkName) - // ginkgo.DeferCleanup(func() { - // err = kind.NetworkDisconnect(dockerNetwork.ID, kindNodes) - // framework.ExpectNoError(err, "disconnecting nodes from network "+dockerNetworkName) - // }) - - // ginkgo.By("Getting node links that belong to the docker network") - // kindNodes, err = kind.ListNodes(clusterName, "") - // framework.ExpectNoError(err, "getting nodes in kind cluster") - // linkMap := make(map[string]*iproute.Link, len(nodes)) - // for _, node := range kindNodes { - // links, err := node.ListLinks() - // framework.ExpectNoError(err, "failed to list links on node %s: %v", node.Name(), err) - - // for _, link := range links { - // if link.Address == node.NetworkSettings.Networks[dockerNetworkName].MacAddress { - // linkMap[node.Name()] = &link - // break - // } - // } - // framework.ExpectHaveKey(linkMap, node.Name()) - // } - - // ginkgo.By("Getting docker network " + dockerNetworkName) - // network, err := docker.NetworkInspect(dockerNetworkName) - // framework.ExpectNoError(err, "getting docker network "+dockerNetworkName) - - // externalSubnet := generateSubnetFromDockerNetwork(externalSubnetName, network, f.HasIPv4(), f.HasIPv6()) - // externalSubnet.Spec.Provider = provider - - // ginkgo.By("Creating underlay subnet " + externalSubnetName) - // ginkgo.DeferCleanup(func() { - // ginkgo.By("Deleting external subnet " + externalSubnetName) - // subnetClient.DeleteSync(externalSubnetName) - // }) - // _ = subnetClient.CreateSync(externalSubnet) - - // vpcName := util.DefaultVpc - // cidr := framework.RandomCIDR(f.ClusterIPFamily) - // bfdIP := framework.RandomIPs(cidr, ";", 1) - // ginkgo.By("Enabling BFD Port with IP " + bfdIP + " for VPC " + vpcName) - // vpc := vpcClient.Get(vpcName) - // patchedVpc := vpc.DeepCopy() - // patchedVpc.Spec.BFDPort = &apiv1.BFDPort{ - // Enabled: true, - // IP: bfdIP, - // NodeSelector: &metav1.LabelSelector{ - // MatchExpressions: []metav1.LabelSelectorRequirement{{ - // Key: controlPlaneLabel, - // Operator: metav1.LabelSelectorOpExists, - // }}, - // }, - // } - // updatedVpc := vpcClient.PatchSync(vpc, patchedVpc, 10*time.Second) - // ginkgo.DeferCleanup(func() { - // ginkgo.By("Disabling BFD Port for VPC " + vpcName) - // patchedVpc := updatedVpc.DeepCopy() - // patchedVpc.Spec.BFDPort = nil - // updatedVpc := vpcClient.PatchSync(updatedVpc, patchedVpc, 10*time.Second) - // framework.ExpectEmpty(updatedVpc.Status.BFDPort.Name) - // framework.ExpectEmpty(updatedVpc.Status.BFDPort.Nodes) - // }) - - // framework.ExpectNotEmpty(updatedVpc.Status.BFDPort.Name) - // for _, node := range nodes { - // if slices.Contains(updatedVpc.Status.BFDPort.Nodes, node.Name) { - // framework.ExpectHaveKey(node.Labels, controlPlaneLabel) - // } else { - // framework.ExpectNotHaveKey(node.Labels, controlPlaneLabel) - // } - // } - - // // TODO: check ovn LRP - - // vegTest(f, true, provider, nadName, vpcName, vpc.Status.DefaultLogicalSwitch, externalSubnetName, replicas) - // }) - - framework.ConformanceIt("should be able to create bgp-edge-router with macvlan", func() { - provider := fmt.Sprintf("%s.%s", nadName, namespaceName) - - ginkgo.By("Creating network attachment definition " + nadName) - nad := framework.MakeMacvlanNetworkAttachmentDefinition(nadName, namespaceName, "eth0", "bridge", provider, nil) - ginkgo.DeferCleanup(func() { - ginkgo.By("Deleting network attachment definition " + nadName) - nadClient.Delete(nadName) - }) - nad = nadClient.Create(nad) - framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) - - vpcName := "vpc-" + framework.RandomSuffix() - ginkgo.By("Creating vpc " + vpcName) - ginkgo.DeferCleanup(func() { - ginkgo.By("Deleting vpc " + vpcName) - vpcClient.DeleteSync(vpcName) - }) - vpc := &apiv1.Vpc{ObjectMeta: metav1.ObjectMeta{Name: vpcName}} - vpc = vpcClient.CreateSync(vpc) - framework.ExpectEmpty(vpc.Status.BFDPort.Name) - framework.ExpectEmpty(vpc.Status.BFDPort.IP) - framework.ExpectEmpty(vpc.Status.BFDPort.Nodes) - - internalSubnetName := "int-" + framework.RandomSuffix() - ginkgo.By("Creating internal subnet " + internalSubnetName) - ginkgo.DeferCleanup(func() { - ginkgo.By("Deleting internal subnet " + internalSubnetName) - subnetClient.DeleteSync(internalSubnetName) - }) - cidr := framework.RandomCIDR(f.ClusterIPFamily) - internalGateway := framework.CIDRGatewayIP(cidr) - internalSubnet := framework.MakeSubnet(internalSubnetName, "", cidr, internalGateway, vpcName, provider, nil, nil, nil) - _ = subnetClient.CreateSync(internalSubnet) - - ginkgo.By("Getting docker network " + kindNetwork) - network, err := docker.NetworkInspect(kindNetwork) - framework.ExpectNoError(err, "getting docker network "+kindNetwork) - - externalSubnet := generateSubnetFromDockerNetwork(externalSubnetName, network, f.HasIPv4(), f.HasIPv6()) - externalSubnet.Spec.Provider = provider - - ginkgo.By("Creating macvlan subnet " + externalSubnetName) - ginkgo.DeferCleanup(func() { - ginkgo.By("Deleting external subnet " + externalSubnetName) - subnetClient.DeleteSync(externalSubnetName) - }) - _ = subnetClient.CreateSync(externalSubnet) - - vegTest(f, true, true, provider, nadName, vpcName, internalSubnetName, externalSubnetName, replicas) - }) -}) - -func generateSubnetFromDockerNetwork(subnetName string, network *dockernetwork.Inspect, ipv4, ipv6 bool) *apiv1.Subnet { - ginkgo.GinkgoHelper() - - ginkgo.By("Generating subnet configuration from docker network " + network.Name) - var cidrV4, cidrV6, gatewayV4, gatewayV6 string - for _, config := range network.IPAM.Config { - switch util.CheckProtocol(config.Subnet) { - case apiv1.ProtocolIPv4: - if ipv4 { - cidrV4 = config.Subnet - gatewayV4 = config.Gateway - } - case apiv1.ProtocolIPv6: - if ipv6 { - cidrV6 = config.Subnet - if gatewayV6 = config.Gateway; gatewayV6 == "" { - var err error - gatewayV6, err = util.FirstIP(cidrV6) - framework.ExpectNoError(err) - } - } - } - } - - cidr := make([]string, 0, 2) - gateway := make([]string, 0, 2) - if ipv4 { - cidr = append(cidr, cidrV4) - gateway = append(gateway, gatewayV4) - } - if ipv6 { - cidr = append(cidr, cidrV6) - gateway = append(gateway, gatewayV6) - } - - excludeIPs := make([]string, 0, len(network.Containers)*2) - for _, container := range network.Containers { - if container.IPv4Address != "" && ipv4 { - excludeIPs = append(excludeIPs, strings.Split(container.IPv4Address, "/")[0]) - } - if container.IPv6Address != "" && ipv6 { - excludeIPs = append(excludeIPs, strings.Split(container.IPv6Address, "/")[0]) - } - } - - return framework.MakeSubnet(subnetName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", "", excludeIPs, nil, nil) -} - -func checkBgpEdgeRouterAccess(f *framework.Framework, namespaceName, svrPodName, image, svrPort string, svrIPs, extIPs []string, intIPs map[string][]string, subnetName, nodeName string, snat bool) { - ginkgo.GinkgoHelper() - - podName := "pod-" + framework.RandomSuffix() - ginkgo.By("Creating client pod " + podName + " within subnet " + subnetName) - labels := map[string]string{"snat": strconv.FormatBool(snat)} - annotations := map[string]string{util.LogicalSwitchAnnotation: subnetName} - pod := framework.MakePrivilegedPod(namespaceName, podName, labels, annotations, image, []string{"sleep", "infinity"}, nil) - pod.Spec.NodeName = nodeName - ginkgo.DeferCleanup(func() { - ginkgo.By("Deleting pod " + podName) - f.PodClient().DeleteSync(podName) - }) - pod = f.PodClient().CreateSync(pod) - - if !snat { - // skip bgp edge router route check if SNAT is enabled - // traceroute does not work for pods selected by the selectors - var hops []string - if nodeName == "" { - for ips := range maps.Values(intIPs) { - hops = append(hops, ips...) - } - } else { - hops = intIPs[nodeName] - } - framework.CheckPodEgressRoutes(pod.Namespace, pod.Name, f.HasIPv4(), f.HasIPv6(), 2, hops) - } - - if !snat { - podIPv4, podIPv6 := util.SplitIpsByProtocol(util.PodIPs(*pod)) - hopsIPv4, hopsIPv6 := util.SplitIpsByProtocol(extIPs) - addEcmpRoutes(namespaceName, svrPodName, podIPv4, hopsIPv4) - addEcmpRoutes(namespaceName, svrPodName, podIPv6, hopsIPv6) - } - - expectedClientIPs := extIPs - if !snat { - expectedClientIPs = util.PodIPs(*pod) - } - for _, svrIP := range svrIPs { - protocol := strings.ToLower(util.CheckProtocol(svrIP)) - ginkgo.By("Checking connection from " + pod.Name + " to " + svrIP + " via " + protocol) - cmd := fmt.Sprintf("curl -q -s --connect-timeout 2 --max-time 2 %s/clientip", net.JoinHostPort(svrIP, svrPort)) - ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, cmd, pod.Namespace, pod.Name)) - output := e2epodoutput.RunHostCmdOrDie(pod.Namespace, pod.Name, cmd) - clientIP, _, err := net.SplitHostPort(strings.TrimSpace(output)) - framework.ExpectNoError(err) - framework.ExpectContainElement(expectedClientIPs, clientIP) - } -} - -func addEcmpRoutes(namespaceName, podName string, destinations, nextHops []string) { - ginkgo.GinkgoHelper() - - if len(destinations) == 0 || len(nextHops) == 0 { - return - } - - var args string - if len(nextHops) == 1 { - args = " via " + nextHops[0] - } else { - for _, ip := range nextHops { - args += fmt.Sprintf(" nexthop via %s dev net1 weight 1", ip) - } - } - for _, dst := range destinations { - cmd := fmt.Sprintf("ip route add %s%s", dst, args) - output, err := e2epodoutput.RunHostCmd(namespaceName, podName, cmd) - framework.ExpectNoError(err, output) - } -} - -func vegTest(f *framework.Framework, bfd, bgp bool, provider, nadName, vpcName, internalSubnetName, externalSubnetName string, replicas int32) { - ginkgo.GinkgoHelper() - - namespaceName := f.Namespace.Name - // snatSubnetName := "snat-" + framework.RandomSuffix() - // forwardSubnetName := "forward-" + framework.RandomSuffix() - // subnetClient := f.SubnetClient() - berClient := f.BgpEdgeRouterClient() - deployClient := f.DeploymentClient() - podClient := f.PodClient() - - // var forwardSubnet *apiv1.Subnet - // for _, subnetName := range []string{snatSubnetName, forwardSubnetName} { - // ginkgo.By("Creating subnet " + subnetName) - // cidr := framework.RandomCIDR(f.ClusterIPFamily) - // subnet := framework.MakeSubnet(subnetName, "", cidr, "", vpcName, "", nil, nil, nil) - // ginkgo.DeferCleanup(func() { - // ginkgo.By("Deleting subnet " + subnetName) - // subnetClient.DeleteSync(subnetName) - // }) - // _ = subnetClient.CreateSync(subnet) - // if subnetName == forwardSubnetName { - // forwardSubnet = subnet - // } - // } - - berName := "ber-" + framework.RandomSuffix() - ginkgo.By("Creating bgp edge router " + berName) - ber := framework.MakeBgpEdgeRouter(namespaceName, berName, vpcName, replicas, internalSubnetName, externalSubnetName) - if rand.Int32N(2) == 0 { - ber.Spec.Prefix = fmt.Sprintf("e2e-%s-", framework.RandomSuffix()) - } - ber.Spec.BFD.Enabled = bfd - ber.Spec.BFD.MinRX = 300 - ber.Spec.BFD.MinTX = 300 - ber.Spec.BFD.Multiplier = 3 - - ber.Spec.Policies = []apiv1.BgpEdgeRouterPolicy{{ - SNAT: false, - Subnets: []string{internalSubnetName}, - // IPBlocks: strings.Split(forwardSubnet.Spec.CIDRBlock, ","), - }} - - ber.Spec.BGP.Enabled = bgp - ber.Spec.BGP.Image = "inyong7635/kube-ovn:v1.15.0" - ber.Spec.BGP.ASN = 65001 - ber.Spec.BGP.RemoteASN = 65002 - ber.Spec.BGP.Neighbors = []string{"10.9.103.190"} - ber.Spec.BGP.RouterID = "10.9.103.189" - ber.Spec.BGP.EnableGracefulRestart = true - - if vpcName == util.DefaultVpc { - ber.Spec.VPC = "" // test whether the ber works without specifying VPC - ber.Spec.TrafficPolicy = apiv1.TrafficPolicyLocal - } - if util.IsOvnProvider(provider) { - ber.Spec.Selectors = []apiv1.BgpEdgeRouterSelector{{ - NamespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - corev1.LabelMetadataName: namespaceName, - }, - }, - PodSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "snat": strconv.FormatBool(true), - }, - }, - }} - } else { - ber.Spec.Policies = append(ber.Spec.Policies, apiv1.BgpEdgeRouterPolicy{ - SNAT: true, - Subnets: []string{internalSubnetName}, - }) - } - - ginkgo.DeferCleanup(func() { - ginkgo.By("Deleting bgp edge router " + berName) - berClient.DeleteSync(berName) - }) - ber = berClient.CreateSync(ber) - - ginkgo.By("Validating bgp edge router status") - framework.ExpectTrue(ber.Status.Ready) - framework.ExpectEqual(ber.Status.Phase, apiv1.PhaseCompleted) - framework.ExpectHaveLen(ber.Status.InternalIPs, int(replicas)) - framework.ExpectHaveLen(ber.Status.ExternalIPs, int(replicas)) - - ginkgo.By("Validating bgp edge router workload") - framework.ExpectEqual(ber.Status.Workload.Name, ber.Spec.Prefix+ber.Name) - deploy := deployClient.Get(ber.Status.Workload.Name) - framework.ExpectEqual(deploy.Status.Replicas, replicas) - framework.ExpectEqual(deploy.Status.ReadyReplicas, replicas) - gvk := appsv1.SchemeGroupVersion.WithKind(reflect.TypeOf(deploy).Elem().Name()) - framework.ExpectEqual(ber.Status.Workload.APIVersion, gvk.GroupVersion().String()) - framework.ExpectEqual(ber.Status.Workload.Kind, gvk.Kind) - framework.ExpectHaveLen(ber.Status.Workload.Nodes, int(replicas)) - workloadPods, err := deployClient.GetPods(deploy) - framework.ExpectNoError(err) - framework.ExpectHaveLen(workloadPods.Items, int(replicas)) - podNodes := make([]string, 0, len(workloadPods.Items)) - intIPs := make(map[string][]string, len(workloadPods.Items)) - for _, pod := range workloadPods.Items { - framework.ExpectNotContainElement(podNodes, pod.Spec.NodeName) - podNodes = append(podNodes, pod.Spec.NodeName) - intIPs[pod.Spec.NodeName] = util.PodIPs(pod) - } - framework.ExpectConsistOf(ber.Status.Workload.Nodes, podNodes) - - svrPodName := "svr-" + framework.RandomSuffix() - ginkgo.By("Creating netexec server pod " + svrPodName) - routes := util.NewPodRoutes() - // dstV4, dstV6 := util.SplitStringIP(forwardSubnet.Spec.CIDRBlock) - // gwV4, gwV6 := util.SplitStringIP(ber.Status.ExternalIPs[0]) - // routes.Add(provider, dstV4, gwV4) - // routes.Add(provider, dstV6, gwV6) - annotations, err := routes.ToAnnotations() - framework.ExpectNoError(err) - attachmentNetworkName := fmt.Sprintf("%s/%s", namespaceName, nadName) - annotations[nadv1.NetworkAttachmentAnnot] = attachmentNetworkName - port := strconv.Itoa(8000 + rand.IntN(1000)) - args := []string{"netexec", "--http-port", port} - svrPod := framework.MakePrivilegedPod(namespaceName, svrPodName, nil, annotations, framework.AgnhostImage, nil, args) - ginkgo.DeferCleanup(func() { - ginkgo.By("Deleting pod " + svrPodName) - podClient.DeleteSync(svrPodName) - }) - svrPod = podClient.CreateSync(svrPod) - svrIPs, err := util.PodAttachmentIPs(svrPod, attachmentNetworkName) - framework.ExpectNoError(err) - - image := workloadPods.Items[0].Spec.Containers[0].Image - extIPs := make([]string, 0, len(ber.Status.ExternalIPs)*2) - for _, ips := range ber.Status.ExternalIPs { - extIPs = append(extIPs, strings.Split(ips, ",")...) - } - - var nodeName string - if ber.Spec.TrafficPolicy == apiv1.TrafficPolicyLocal { - nodeName = ber.Status.Workload.Nodes[0] - } - checkBgpEdgeRouterAccess(f, namespaceName, svrPodName, image, port, svrIPs, extIPs, intIPs, externalSubnetName, nodeName, true) - checkBgpEdgeRouterAccess(f, namespaceName, svrPodName, image, port, svrIPs, extIPs, intIPs, internalSubnetName, nodeName, false) -} From dbccc91161035febc1589c77e6af51d936b2ae73 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Wed, 27 Aug 2025 20:58:09 -0700 Subject: [PATCH 61/66] Erase bgpEdgeRouterInfo key in gobgp-config --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 13 ++++--------- pkg/apis/kubeovn/v1/gobgp-config.go | 9 ++------- pkg/controller/gobgp_config.go | 10 +++++----- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index 3584df2e497..3d572647ef8 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3713,7 +3713,7 @@ spec: additionalPrinterColumns: - name: bgp-edge-router type: string - jsonPath: .spec.bgpEdgeRouterInfo.name + jsonPath: .spec.bgpEdgeRouter - name: neighbor-address type: string jsonPath: .spec.neighbors[*].address @@ -3730,16 +3730,11 @@ spec: spec: type: object required: - - bgpEdgeRouterInfo + - bgpEdgeRouter - neighbors properties: - bgpEdgeRouterInfo: - type: object - properties: - name: - type: string - namespace: - type: string + bgpEdgeRouter: + type: string neighbors: type: array x-kubernetes-list-type: map diff --git a/pkg/apis/kubeovn/v1/gobgp-config.go b/pkg/apis/kubeovn/v1/gobgp-config.go index 81a9860bb93..5522a08f672 100644 --- a/pkg/apis/kubeovn/v1/gobgp-config.go +++ b/pkg/apis/kubeovn/v1/gobgp-config.go @@ -41,8 +41,8 @@ func (g *GobgpConfig) Subnet(subnets []string) []string { // } type GobgpConfigSpec struct { - BgpEdgeRouterInfo BgpEdgeRouterInfo `json:"bgpEdgeRouterInfo"` - Neighbors []Neighbors `json:"neighbors,omitempty"` + BgpEdgeRouter string `json:"bgpEdgeRouter"` + Neighbors []Neighbors `json:"neighbors,omitempty"` } type GobgpConfigStatus struct { @@ -50,11 +50,6 @@ type GobgpConfigStatus struct { Conditions Conditions `json:"conditions,omitempty"` } -type BgpEdgeRouterInfo struct { - Name string `json:"name"` - Namespace string `json:"namespace"` -} - // Neighbors defines the BGP neighbors configuration // +k8s:openapi-gen=true // +genclient:nonNamespaced diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go index 76520c1c3a2..fb302d0f225 100644 --- a/pkg/controller/gobgp_config.go +++ b/pkg/controller/gobgp_config.go @@ -346,11 +346,11 @@ func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpCo } func (c *Controller) validateGobgpConfig(gobgpConfig *kubeovnv1.GobgpConfig) ([]*corev1.Pod, error) { - klog.Infof("gobgpConfignamespace: %s name: %s", gobgpConfig.Spec.BgpEdgeRouterInfo.Namespace, gobgpConfig.Spec.BgpEdgeRouterInfo.Name) - ber, err := c.bgpEdgeRouterLister.BgpEdgeRouters(gobgpConfig.Spec.BgpEdgeRouterInfo.Namespace).Get(gobgpConfig.Spec.BgpEdgeRouterInfo.Name) + klog.Infof("gobgpConfignamespace: %s name: %s", gobgpConfig.Namespace, gobgpConfig.Name) + ber, err := c.bgpEdgeRouterLister.BgpEdgeRouters(gobgpConfig.Namespace).Get(gobgpConfig.Spec.BgpEdgeRouter) if err != nil { if k8serrors.IsNotFound(err) { - return nil, fmt.Errorf("bgp edge router %s not found: %w", gobgpConfig.Spec.BgpEdgeRouterInfo.Name, err) + return nil, fmt.Errorf("bgp edge router %s not found: %w", gobgpConfig.Spec.BgpEdgeRouter, err) } } berNeighbors := ber.Spec.BGP.Neighbors @@ -364,12 +364,12 @@ func (c *Controller) validateGobgpConfig(gobgpConfig *kubeovnv1.GobgpConfig) ([] } if !neighborFlag { - err = fmt.Errorf("no matching neighbor found in BgpEdgeRouter %s for GobgpConfig %s", gobgpConfig.Spec.BgpEdgeRouterInfo.Name, gobgpConfig.Name) + err = fmt.Errorf("no matching neighbor found in BgpEdgeRouter %s for GobgpConfig %s", gobgpConfig.Spec.BgpEdgeRouter, gobgpConfig.Name) klog.Error(err) return nil, err } - deploy, err := c.berDeploymentsLister.Deployments(gobgpConfig.Spec.BgpEdgeRouterInfo.Namespace).Get(gobgpConfig.Spec.BgpEdgeRouterInfo.Name) + deploy, err := c.berDeploymentsLister.Deployments(gobgpConfig.Namespace).Get(gobgpConfig.Spec.BgpEdgeRouter) if err != nil { gobgpConfig.Status.Ready = false msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) From d0ab33071756f9835b9f4a23158f41bbefc779c9 Mon Sep 17 00:00:00 2001 From: inyongma1 Date: Wed, 27 Aug 2025 21:40:55 -0700 Subject: [PATCH 62/66] bgp edge router selector key is deleted. --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 61 +------------------- pkg/apis/kubeovn/v1/bgp-edge-router.go | 7 --- pkg/apis/kubeovn/v1/gobgp-config.go | 5 -- pkg/apis/kubeovn/v1/zz_generated.deepcopy.go | 33 ----------- pkg/controller/bgp_edge_router.go | 60 ------------------- 5 files changed, 2 insertions(+), 164 deletions(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index 3d572647ef8..e5fe517521e 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3327,8 +3327,8 @@ spec: - rule: "!has(self.externalIPs) || size(self.externalIPs) == 0 || size(self.externalIPs) >= self.replicas" message: 'Size of External IPs MUST be equal to or greater than Replicas' fieldPath: ".externalIPs" - - rule: "size(self.policies) != 0 || size(self.selectors) != 0" - message: 'Each BGP Edeg Router MUST have at least one policy or selector' + - rule: "size(self.policies) != 0" + message: 'Each BGP Edge Router MUST have at least one policy' properties: replicas: type: integer @@ -3390,63 +3390,6 @@ spec: type: integer format: int32 default: 3 - selectors: - type: array - items: - type: object - properties: - namespaceSelector: - type: object - properties: - matchLabels: - additionalProperties: - type: string - type: object - matchExpressions: - type: array - items: - type: object - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - x-kubernetes-validations: - - rule: "size(self.matchLabels) != 0 || size(self.matchExpressions) != 0" - message: 'Each namespace selector MUST have at least one matchLabels or matchExpressions' - podSelector: - type: object - properties: - matchLabels: - additionalProperties: - type: string - type: object - matchExpressions: - type: array - items: - type: object - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - x-kubernetes-validations: - - rule: "size(self.matchLabels) != 0 || size(self.matchExpressions) != 0" - message: 'Each pod selector MUST have at least one matchLabels or matchExpressions' policies: type: array items: diff --git a/pkg/apis/kubeovn/v1/bgp-edge-router.go b/pkg/apis/kubeovn/v1/bgp-edge-router.go index 8c397c5f8c8..bae4b0fd4fd 100644 --- a/pkg/apis/kubeovn/v1/bgp-edge-router.go +++ b/pkg/apis/kubeovn/v1/bgp-edge-router.go @@ -64,8 +64,6 @@ type BgpEdgeRouterSpec struct { // the IPs count must NOT be less than the replicas count InternalIPs []string `json:"internalIPs,omitempty"` ExternalIPs []string `json:"externalIPs,omitempty"` - // namespace/pod selectors - Selectors []BgpEdgeRouterSelector `json:"selectors,omitempty"` // optional traffic policy used to control the traffic routing // if not specified, the default traffic policy "Cluster" will be used // if set to "Local", traffic will be routed to the gateway pod/instance on the same node when available @@ -87,11 +85,6 @@ type BgpEdgeRouterSpec struct { // KubeApiSubnet string `json:"kubeApiSubnet,omitempty"` } -type BgpEdgeRouterSelector struct { - NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"` - PodSelector *metav1.LabelSelector `json:"podSelector,omitempty"` -} - type BgpEdgeRouterBFDConfig struct { // whether to enable BFD // if set to true, the egress gateway will establish BFD session(s) with the VPC BFD LRP diff --git a/pkg/apis/kubeovn/v1/gobgp-config.go b/pkg/apis/kubeovn/v1/gobgp-config.go index 5522a08f672..328d667ee80 100644 --- a/pkg/apis/kubeovn/v1/gobgp-config.go +++ b/pkg/apis/kubeovn/v1/gobgp-config.go @@ -35,11 +35,6 @@ func (g *GobgpConfig) Subnet(subnets []string) []string { return nil } -// Ready returns true if the BgpEdgeRouter has been processed successfully and is ready to serve traffic -// func (g *BgpEdgeRouterAdvertisement) Ready() bool { -// return g.Status.Ready && g.Status.Conditions.IsReady(g.Generation) -// } - type GobgpConfigSpec struct { BgpEdgeRouter string `json:"bgpEdgeRouter"` Neighbors []Neighbors `json:"neighbors,omitempty"` diff --git a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go index baa7bc636ee..d721f5afd93 100644 --- a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go +++ b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go @@ -2981,32 +2981,6 @@ func (in *BgpEdgeRouterPolicy) DeepCopy() *BgpEdgeRouterPolicy { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BgpEdgeRouterSelector) DeepCopyInto(out *BgpEdgeRouterSelector) { - *out = *in - if in.NamespaceSelector != nil { - in, out := &in.NamespaceSelector, &out.NamespaceSelector - *out = new(metav1.LabelSelector) - (*in).DeepCopyInto(*out) - } - if in.PodSelector != nil { - in, out := &in.PodSelector, &out.PodSelector - *out = new(metav1.LabelSelector) - (*in).DeepCopyInto(*out) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewaySelector. -func (in *BgpEdgeRouterSelector) DeepCopy() *BgpEdgeRouterSelector { - if in == nil { - return nil - } - out := new(BgpEdgeRouterSelector) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BgpEdgeRouterSpec) DeepCopyInto(out *BgpEdgeRouterSpec) { *out = *in @@ -3020,13 +2994,6 @@ func (in *BgpEdgeRouterSpec) DeepCopyInto(out *BgpEdgeRouterSpec) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.Selectors != nil { - in, out := &in.Selectors, &out.Selectors - *out = make([]BgpEdgeRouterSelector, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } out.BFD = in.BFD if in.Policies != nil { in, out := &in.Policies, &out.Policies diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go index 2ad161255ad..835c79d27b4 100644 --- a/pkg/controller/bgp_edge_router.go +++ b/pkg/controller/bgp_edge_router.go @@ -562,48 +562,6 @@ func (c *Controller) reconcileBgpEdgeRouterOVNRoutes(router *kubeovnv1.BgpEdgeRo // reconcile OVN port group ports := set.New[string]() - for _, selector := range router.Spec.Selectors { - sel := labels.Everything() - if selector.NamespaceSelector != nil { - if sel, err = metav1.LabelSelectorAsSelector(selector.NamespaceSelector); err != nil { - err = fmt.Errorf("failed to create label selector for namespace selector %#v: %w", selector.NamespaceSelector, err) - klog.Error(err) - return err - } - } - namespaces, err := c.namespacesLister.List(sel) - if err != nil { - err = fmt.Errorf("failed to list namespaces with selector %s: %w", sel, err) - klog.Error(err) - return err - } - sel = labels.Everything() - if selector.PodSelector != nil { - if sel, err = metav1.LabelSelectorAsSelector(selector.PodSelector); err != nil { - err = fmt.Errorf("failed to create label selector for pod selector %#v: %w", selector.PodSelector, err) - klog.Error(err) - return err - } - } - for _, ns := range namespaces { - pods, err := c.podsLister.Pods(ns.Name).List(sel) - if err != nil { - err = fmt.Errorf("failed to list pods with selector %s in namespace %s: %w", sel, ns.Name, err) - klog.Error(err) - return err - } - for _, pod := range pods { - if pod.Spec.HostNetwork || - pod.Annotations[util.AllocatedAnnotation] != "true" || - pod.Annotations[util.LogicalRouterAnnotation] != router.VPC(c.config.ClusterRouter) || - !isPodAlive(pod) { - continue - } - podName := c.getNameByPod(pod) - ports.Insert(ovs.PodNameToPortName(podName, pod.Namespace, util.OvnProvider)) - } - } - } key := cache.MetaObjectToName(router).String() pgName := berPortGroupName(key) if err = c.OVNNbClient.CreatePortGroup(pgName, externalIDs); err != nil { @@ -1027,15 +985,6 @@ func (c *Controller) handlePodEventForBgpEdgeRouter(pod *corev1.Pod) error { return nil } - ns, err := c.namespacesLister.Get(pod.Namespace) - if err != nil { - if k8serrors.IsNotFound(err) { - return nil - } - klog.Errorf("failed to get namespace %s: %v", pod.Namespace, err) - utilruntime.HandleError(err) - return err - } router, err := c.bgpEdgeRouterLister.List(labels.Everything()) if err != nil { @@ -1049,15 +998,6 @@ func (c *Controller) handlePodEventForBgpEdgeRouter(pod *corev1.Pod) error { continue } - for _, selector := range ber.Spec.Selectors { - if selector.NamespaceSelector != nil && !util.ObjectMatchesLabelSelector(ns, selector.NamespaceSelector) { - continue - } - if selector.PodSelector != nil && !util.ObjectMatchesLabelSelector(pod, selector.PodSelector) { - continue - } - c.addOrUpdateBgpEdgeRouterQueue.Add(cache.MetaObjectToName(ber).String()) - } } return nil } From 0d986f125c558bd1aac4f9056ee5dce77adc640c Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Wed, 27 Aug 2025 22:10:59 -0700 Subject: [PATCH 63/66] [fix] ber asn to localAsn --- charts/kube-ovn-v2/crds/kube-ovn-crd.yaml | 2 +- dist/images/install.sh | 2 +- pkg/apis/kubeovn/v1/bgp-edge-router.go | 2 +- pkg/controller/bgp_edge_router.go | 2 -- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/charts/kube-ovn-v2/crds/kube-ovn-crd.yaml b/charts/kube-ovn-v2/crds/kube-ovn-crd.yaml index f1655a4ef1e..313733f6949 100644 --- a/charts/kube-ovn-v2/crds/kube-ovn-crd.yaml +++ b/charts/kube-ovn-v2/crds/kube-ovn-crd.yaml @@ -508,7 +508,7 @@ spec: properties: enabled: type: boolean - asn: + localAsn: type: integer remoteAsn: type: integer diff --git a/dist/images/install.sh b/dist/images/install.sh index a3c8e08f1fd..852863079cb 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -3745,7 +3745,7 @@ spec: default: false image: type: string - asn: + localAsn: type: integer format: int32 minimum: 0 diff --git a/pkg/apis/kubeovn/v1/bgp-edge-router.go b/pkg/apis/kubeovn/v1/bgp-edge-router.go index bae4b0fd4fd..a3e30bdd730 100644 --- a/pkg/apis/kubeovn/v1/bgp-edge-router.go +++ b/pkg/apis/kubeovn/v1/bgp-edge-router.go @@ -141,7 +141,7 @@ type BgpEdgeRouterBGPConfig struct { // optional bgp image used by the workload // if not specified, the default image passed in by kube-ovn-controller will be used Image string `json:"image,omitempty"` - ASN uint32 `json:"asn"` + ASN uint32 `json:"localAsn"` RemoteASN uint32 `json:"remoteAsn"` Neighbors []string `json:"neighbors"` HoldTime metav1.Duration `json:"holdTime"` diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go index 835c79d27b4..86a4121589f 100644 --- a/pkg/controller/bgp_edge_router.go +++ b/pkg/controller/bgp_edge_router.go @@ -985,7 +985,6 @@ func (c *Controller) handlePodEventForBgpEdgeRouter(pod *corev1.Pod) error { return nil } - router, err := c.bgpEdgeRouterLister.List(labels.Everything()) if err != nil { klog.Errorf("failed to list bgp edge router: %v", err) @@ -997,7 +996,6 @@ func (c *Controller) handlePodEventForBgpEdgeRouter(pod *corev1.Pod) error { if ber.VPC(c.config.ClusterRouter) != vpc { continue } - } return nil } From 82bbad9d8dee0a6a2281265845abe064bcd4ea40 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Wed, 27 Aug 2025 22:55:51 -0700 Subject: [PATCH 64/66] [fix] ber change crd --- charts/kube-ovn/templates/kube-ovn-crd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index e5fe517521e..157958f5a95 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -3493,7 +3493,7 @@ spec: default: false image: type: string - asn: + localAsn: type: integer format: int32 minimum: 0 From ae18294ddd47e07c662b18211664aef995b028c6 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Tue, 2 Sep 2025 01:46:00 -0700 Subject: [PATCH 65/66] [fix] bgp advertisement modify address set --- .../bgp_edge_router_advertisement.go | 116 +++++++++++++++++- test/e2e/edge-router/e2e_test.go | 9 +- 2 files changed, 119 insertions(+), 6 deletions(-) diff --git a/pkg/controller/bgp_edge_router_advertisement.go b/pkg/controller/bgp_edge_router_advertisement.go index 399c7fcc169..572d01d4556 100644 --- a/pkg/controller/bgp_edge_router_advertisement.go +++ b/pkg/controller/bgp_edge_router_advertisement.go @@ -17,6 +17,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + "k8s.io/utils/set" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" @@ -147,11 +148,108 @@ func (c *Controller) handleAddBgpEdgeRouterAdvertisement(key string) error { if _, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement); err != nil { return err } + + // update ber address_set + if err := c.updateAddressSetForBer(ns, advertisement, "add"); err != nil { + klog.Error(err) + return err + } + klog.Infof("finished reconciling bgp-edge-router-advertisement %s", key) return nil } +func (c *Controller) updateAddressSetForBer(ns string, advertisement *kubeovnv1.BgpEdgeRouterAdvertisement, op string) error { + // modify ber address_set + berName := advertisement.Spec.BgpEdgeRouter + cachedRouter, err := c.bgpEdgeRouterLister.BgpEdgeRouters(ns).Get(berName) + if err != nil { + klog.Error(err) + return err + } + router := cachedRouter.DeepCopy() + // collect egress policies + ipv4ForwardSrc, ipv6ForwardSrc := set.New[string](), set.New[string]() + ipv4SNATSrc, ipv6SNATSrc := set.New[string](), set.New[string]() + for _, policy := range router.Spec.Policies { + ipv4, ipv6 := util.SplitIpsByProtocol(policy.IPBlocks) + if policy.SNAT { + ipv4SNATSrc.Insert(ipv4...) + ipv6SNATSrc.Insert(ipv6...) + } else { + ipv4ForwardSrc.Insert(ipv4...) + ipv6ForwardSrc.Insert(ipv6...) + } + for _, subnetName := range policy.Subnets { + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + klog.Error(err) + return err + } + if subnet.Status.IsNotValidated() { + err = fmt.Errorf("subnet %s is not validated", subnet.Name) + klog.Error(err) + return err + } + // TODO: check subnet's vpc and vlan + ipv4, ipv6 := util.SplitStringIP(subnet.Spec.CIDRBlock) + if policy.SNAT { + ipv4SNATSrc.Insert(ipv4) + ipv6SNATSrc.Insert(ipv6) + } else { + ipv4ForwardSrc.Insert(ipv4) + ipv6ForwardSrc.Insert(ipv6) + } + } + } + + // collect advertisement subnets + if op == "add" { + advCidrBlocks, err := c.getSubnetCidrBlock(advertisement) + if err != nil { + klog.Error(err) + return err + } + for _, advCidrBlock := range advCidrBlocks { + ipv4adv, ipv6adv := util.SplitStringIP(advCidrBlock) + ipv4ForwardSrc.Insert(ipv4adv) + ipv6ForwardSrc.Insert(ipv6adv) + } + } + + // calculate internal route destinations and forward source CIDR blocks + intRouteDstIPv4, intRouteDstIPv6 := ipv4ForwardSrc.Union(ipv4SNATSrc), ipv6ForwardSrc.Union(ipv6SNATSrc) + intRouteDstIPv4.Delete("") + intRouteDstIPv6.Delete("") + ipv4ForwardSrc.Delete("") + ipv6ForwardSrc.Delete("") + + klog.Infof("setting address set for bgp edge router : %s, intRouteDstIPv4 %v, intRouteDstIPv6 %v", berName, intRouteDstIPv4, intRouteDstIPv6) + berKey := cache.MetaObjectToName(router).String() + klog.Infof("debug bgp-edge-router %s", berKey) + if intRouteDstIPv4.Len() > 0 { + asName := berAddressSetName(berKey, 4) + klog.Infof("address set name: %s", asName) + if err = c.OVNNbClient.AddressSetUpdateAddress(asName, intRouteDstIPv4.SortedList()...); err != nil { + klog.Error(err) + err = fmt.Errorf("failed to create or update address set %s: %w", asName, err) + klog.Error(err) + return err + } + } + if intRouteDstIPv6.Len() > 0 { + asName := berAddressSetName(berKey, 6) + if err = c.OVNNbClient.AddressSetUpdateAddress(asName, intRouteDstIPv6.SortedList()...); err != nil { + klog.Error(err) + err = fmt.Errorf("failed to create or update address set %s: %w", asName, err) + klog.Error(err) + return err + } + } + return nil +} + func (c *Controller) handleUpdateBgpEdgeRouterAdvertisement(updatedObj *updateVerObject) error { key := updatedObj.key @@ -198,10 +296,17 @@ func (c *Controller) handleUpdateBgpEdgeRouterAdvertisement(updatedObj *updateVe } } + // update ber address_set + if err := c.updateAddressSetForBer(ns, advertisement, "add"); err != nil { + klog.Error(err) + return err + } + advertisement.Status.Conditions.SetReady("ReconcileSuccess", advertisement.Generation) if _, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement); err != nil { return err } + klog.Infof("finished reconciling bgp-edge-router-advertisement %s", key) return nil @@ -255,10 +360,17 @@ func (c *Controller) handleDelBgpEdgeRouterAdvertisement(key string) error { } } - advertisement.Status.Conditions.SetReady("ReconcileSuccess", advertisement.Generation) - if _, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement); err != nil { + // update ber address_set + if err := c.updateAddressSetForBer(ns, advertisement, "del"); err != nil { + klog.Error(err) return err } + + // advertisement.Status.Conditions.SetReady("ReconcileSuccess", advertisement.Generation) + // if _, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement); err != nil { + // return err + // } + klog.Infof("finished reconciling bgp-edge-router-advertisement %s", key) return nil diff --git a/test/e2e/edge-router/e2e_test.go b/test/e2e/edge-router/e2e_test.go index 34dcd06d337..e3406b53c3e 100644 --- a/test/e2e/edge-router/e2e_test.go +++ b/test/e2e/edge-router/e2e_test.go @@ -3,6 +3,7 @@ package multus import ( "context" "encoding/json" + "errors" "flag" "fmt" "maps" @@ -377,19 +378,19 @@ func parseGobgpOutput(output string) (*GlobalConfig, []NeighborEntry, error) { } if globalJSON == "" || neighborJSON == "" { - return nil, nil, fmt.Errorf("failed to extract JSON parts from output") + return nil, nil, errors.New("failed to extract JSON parts from output") } // GlobalConfig var global GlobalConfig if err := json.Unmarshal([]byte(globalJSON), &global); err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal global config: %v", err) + return nil, nil, fmt.Errorf("failed to unmarshal global config: %w", err) } // NeighborEntry var neighbors []NeighborEntry if err := json.Unmarshal([]byte(neighborJSON), &neighbors); err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal neighbor config: %v", err) + return nil, nil, fmt.Errorf("failed to unmarshal neighbor config: %w", err) } return &global, neighbors, nil @@ -489,7 +490,7 @@ func berTest(f *framework.Framework, bfd bool, provider, nadName, vpcName, inter intIPs[pod.Spec.NodeName] = util.PodIPs(pod) // exec and list ginkgo.By("Checking bgp setting " + pod.Name) - cmd := fmt.Sprintf("gobgp global -j && gobgp neighbor -j") + cmd := "gobgp global -j && gobgp neighbor -j" ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, cmd, pod.Namespace, pod.Name)) output := e2epodoutput.RunHostCmdOrDie(pod.Namespace, pod.Name, cmd) checkBgpInitSetting(ber, output) From c4eb9d6e7e7a7b3d72eea857e19002cb8faa4270 Mon Sep 17 00:00:00 2001 From: donghee-park-rsk Date: Thu, 4 Sep 2025 00:57:56 -0700 Subject: [PATCH 66/66] [feat] bgp edge router skip internal subnet cidr equals to policy cidr --- pkg/controller/bgp_edge_router.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go index 86a4121589f..4df0e85edfc 100644 --- a/pkg/controller/bgp_edge_router.go +++ b/pkg/controller/bgp_edge_router.go @@ -348,6 +348,10 @@ func (c *Controller) reconcileBgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRou routes.Add(util.OvnProvider, bfdIPv6, intGatewayIPv6) // add routes for the internal networks for _, dst := range intRouteDstIPv4.UnsortedList() { + // skip the route to the internal subnet itself + if intSubnet.Spec.CIDRBlock == dst { + continue + } routes.Add(util.OvnProvider, dst, intGatewayIPv4) } for _, dst := range intRouteDstIPv6.UnsortedList() {