diff --git a/charts/kube-ovn-v2/crds/kube-ovn-crd.yaml b/charts/kube-ovn-v2/crds/kube-ovn-crd.yaml index f1655a4ef1e..313733f6949 100644 --- a/charts/kube-ovn-v2/crds/kube-ovn-crd.yaml +++ b/charts/kube-ovn-v2/crds/kube-ovn-crd.yaml @@ -508,7 +508,7 @@ spec: properties: enabled: type: boolean - asn: + localAsn: type: integer remoteAsn: type: integer diff --git a/charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml b/charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml index e1f0b05fd66..82f34b1a461 100644 --- a/charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml +++ b/charts/kube-ovn-v2/templates/rbac/ovn-CR.yaml @@ -50,6 +50,12 @@ rules: - vpc-dnses/status - qos-policies - qos-policies/status + - bgp-edge-routers + - bgp-edge-routers/status + - bgp-edge-router-advertisements + - bgp-edge-router-advertisements/status + - gobgp-configs + - gobgp-configs/status verbs: - "*" - apiGroups: diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index 1f45c50c160..157958f5a95 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -1192,6 +1192,53 @@ spec: required: - key - operator + bgp: + type: object + properties: + edgeRouterMode: + type: boolean + default: false + enabled: + type: boolean + default: false + image: + type: string + asn: + type: integer + format: int32 + minimum: 0 + remoteAsn: + type: integer + format: int32 + minimum: 0 + neighbors: + type: array + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + holdTime: + type: string + pattern: ^[0-9]+[smhd]$ + routerId: + type: string + # The routerId can be an IPv4 or IPv6 address, but we are not enforcing it here + # routerId could be nullable, so that we can use environment value to set it + # refer pkg/speaker/config.go#L186 + anyOf: + - format: ipv4 + - format: ipv6 + - pattern: '^$' + password: + type: string + enableGracefulRestart: + type: boolean + default: false + extraArgs: + type: array + items: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3117,3 +3164,627 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgp-edge-routers.kubeovn.io +spec: + group: kubeovn.io + names: + plural: bgp-edge-routers + singular: bgp-edge-router + shortNames: + - bgp-er + - ber + kind: BgpEdgeRouter + listKind: BgpEdgeRouterList + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vpc + name: VPC + type: string + - jsonPath: .spec.replicas + name: REPLICAS + type: integer + - jsonPath: .spec.bfd.enabled + name: BFD ENABLED + type: boolean + - jsonPath: .spec.externalSubnet + name: EXTERNAL SUBNET + type: string + - jsonPath: .status.phase + name: PHASE + type: string + - jsonPath: .status.ready + name: READY + type: boolean + - jsonPath: .status.internalIPs + name: INTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.externalIPs + name: EXTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.workload.nodes + name: WORKING NODES + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + scale: + # specReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Spec.Replicas. + specReplicasPath: .spec.replicas + # statusReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Replicas. + statusReplicasPath: .status.replicas + # labelSelectorPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Selector. + labelSelectorPath: .status.labelSelector + schema: + openAPIV3Schema: + type: object + properties: + status: + properties: + replicas: + type: integer + format: int32 + labelSelector: + type: string + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + internalIPs: + items: + type: string + type: array + externalIPs: + items: + type: string + type: array + phase: + type: string + default: Pending + enum: + - Pending + - Processing + - Completed + ready: + type: boolean + default: false + workload: + type: object + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + nodes: + type: array + items: + type: string + required: + - conditions + - phase + type: object + spec: + type: object + required: + - externalSubnet + x-kubernetes-validations: + - rule: "!has(self.internalIPs) || size(self.internalIPs) == 0 || size(self.internalIPs) >= self.replicas" + message: 'Size of Internal IPs MUST be equal to or greater than Replicas' + fieldPath: ".internalIPs" + - rule: "!has(self.externalIPs) || size(self.externalIPs) == 0 || size(self.externalIPs) >= self.replicas" + message: 'Size of External IPs MUST be equal to or greater than Replicas' + fieldPath: ".externalIPs" + - rule: "size(self.policies) != 0" + message: 'Each BGP Edge Router MUST have at least one policy' + properties: + replicas: + type: integer + format: int32 + default: 1 + minimum: 0 + maximum: 10 + prefix: + type: string + anyOf: + - pattern: ^$ + - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*[-\.]?$ + x-kubernetes-validations: + - rule: "self == oldSelf" + message: "This field is immutable." + vpc: + type: string + internalSubnet: + type: string + externalSubnet: + type: string + internalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + externalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + image: + type: string + bfd: + type: object + properties: + enabled: + type: boolean + default: false + minRX: + type: integer + format: int32 + default: 1000 + minTX: + type: integer + format: int32 + default: 1000 + multiplier: + type: integer + format: int32 + default: 3 + policies: + type: array + items: + type: object + properties: + snat: + type: boolean + default: false + ipBlocks: + type: array + x-kubernetes-list-type: set + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + - format: cidr + subnets: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + x-kubernetes-validations: + - rule: "size(self.ipBlocks) != 0 || size(self.subnets) != 0" + message: 'Each policy MUST have at least one ipBlock or subnet' + trafficPolicy: + type: string + enum: + - Local + - Cluster + default: Cluster + nodeSelector: + type: array + items: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator + bgp: + type: object + properties: + edgeRouterMode: + type: boolean + default: false + routeServerClient: + type: boolean + default: false + enabled: + type: boolean + default: false + image: + type: string + localAsn: + type: integer + format: int32 + minimum: 0 + remoteAsn: + type: integer + format: int32 + minimum: 0 + neighbors: + type: array + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + holdTime: + type: string + pattern: ^[0-9]+[smhd]$ + routerId: + type: string + # The routerId can be an IPv4 or IPv6 address, but we are not enforcing it here + # routerId could be nullable, so that we can use environment value to set it + # refer pkg/speaker/config.go#L186 + anyOf: + - format: ipv4 + - format: ipv6 + - pattern: '^$' + password: + type: string + enableGracefulRestart: + type: boolean + default: false + extraArgs: + type: array + items: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgp-edge-router-advertisements.kubeovn.io +spec: + group: kubeovn.io + names: + plural: bgp-edge-router-advertisements + singular: bgp-edge-router-advertisement + shortNames: + - ber-ad + kind: BgpEdgeRouterAdvertisement + listKind: BgpEdgeRouterAdvertisementList + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.subnet + name: SUBNET + type: string + - jsonPath: .spec.bgpEdgeRouter + name: BGP-EDGE-ROUTER + type: string + - jsonPath: .status.ready + name: READY + type: boolean + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + subnet: + type: array + minItems: 1 + x-kubernetes-list-type: set + items: + type: string + bgpEdgeRouter: + type: string + required: + - subnet + - bgpEdgeRouter + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + ready: + type: boolean + default: false + required: + - conditions + type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gobgp-configs.kubeovn.io +spec: + group: kubeovn.io + names: + plural: gobgp-configs + singular: gobgp-config + shortNames: + - bgp-config + kind: GobgpConfig + listKind: GobgpConfigList + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: bgp-edge-router + type: string + jsonPath: .spec.bgpEdgeRouter + - name: neighbor-address + type: string + jsonPath: .spec.neighbors[*].address + - name: READY + type: boolean + jsonPath: .status.ready + - name: AGE + type: date + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + required: + - bgpEdgeRouter + - neighbors + properties: + bgpEdgeRouter: + type: string + neighbors: + type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - address + items: + type: object + required: + - address + - toAdvertise + - toReceive + properties: + address: + type: string + pattern: '^(([0-9]{1,3}\.){3}[0-9]{1,3}|[0-9a-fA-F:]+)$' + toAdvertise: + type: object + required: + - allowed + properties: + allowed: + type: object + required: + - mode + properties: + mode: + type: string + prefixes: + type: array + x-kubernetes-list-type: set + items: + type: string + pattern: '^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/\d{1,2})?|([0-9a-fA-F:]+(\/\d{1,3})?))$' + x-kubernetes-validations: + - rule: "self.mode != 'filtered' || (has(self.prefixes) && size(self.prefixes) > 0)" + message: "prefixes must be set and non-empty when mode is 'filtered'" + - rule: "self.mode != 'all' || !has(self.prefixes) || size(self.prefixes) == 0" + message: "If mode is 'all', cannot set prefixes" + toReceive: + type: object + required: + - allowed + properties: + allowed: + type: object + required: + - mode + properties: + mode: + type: string + prefixes: + type: array + x-kubernetes-list-type: set + items: + type: string + pattern: '^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/\d{1,2})?|([0-9a-fA-F:]+(\/\d{1,3})?))$' + x-kubernetes-validations: + - rule: "self.mode != 'filtered' || (has(self.prefixes) && size(self.prefixes) > 0)" + message: "prefixes must be set and non-empty when mode is 'filtered'" + - rule: "self.mode != 'all' || !has(self.prefixes) || size(self.prefixes) == 0" + message: "If mode is 'all', cannot set prefixes" + status: + type: object + properties: + ready: + type: boolean + default: false + conditions: + type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - type + items: + type: object + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + properties: + type: + type: string + maxLength: 316 + pattern: '^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$' + status: + type: string + enum: + - "True" + - "False" + - "Unknown" + reason: + type: string + minLength: 1 + maxLength: 1024 + pattern: '^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$' + message: + type: string + maxLength: 32768 + lastTransitionTime: + type: string + format: date-time + lastUpdateTime: + type: string + format: date-time + observedGeneration: + type: integer + format: int64 + minimum: 0 \ No newline at end of file diff --git a/charts/kube-ovn/templates/ovn-CR.yaml b/charts/kube-ovn/templates/ovn-CR.yaml index d02a15a9d6e..061cea683f4 100644 --- a/charts/kube-ovn/templates/ovn-CR.yaml +++ b/charts/kube-ovn/templates/ovn-CR.yaml @@ -50,6 +50,12 @@ rules: - vpc-dnses/status - qos-policies - qos-policies/status + - bgp-edge-routers + - bgp-edge-routers/status + - bgp-edge-router-advertisements + - bgp-edge-router-advertisements/status + - gobgp-configs + - gobgp-configs/status verbs: - "*" - apiGroups: diff --git a/cmd/speaker/speaker.go b/cmd/speaker/speaker.go index 55267cebecf..60ec54c968d 100644 --- a/cmd/speaker/speaker.go +++ b/cmd/speaker/speaker.go @@ -26,9 +26,8 @@ func CmdMain() { if err != nil { util.LogFatalAndExit(err, "failed to parse config") } - // Do not try to redirect the logs on the node if we're running in a NAT gateway - if !config.NatGwMode { + if !config.NatGwMode && !config.EdgeRouterMode { perm, err := strconv.ParseUint(config.LogPerm, 8, 32) if err != nil { util.LogFatalAndExit(err, "failed to parse log-perm") diff --git a/dist/images/install.sh b/dist/images/install.sh index 676f4236abd..852863079cb 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -3359,6 +3359,697 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgp-edge-routers.kubeovn.io +spec: + group: kubeovn.io + names: + plural: bgp-edge-routers + singular: bgp-edge-router + shortNames: + - bgp-er + - ber + kind: BgpEdgeRouter + listKind: BgpEdgeRouterList + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vpc + name: VPC + type: string + - jsonPath: .spec.replicas + name: REPLICAS + type: integer + - jsonPath: .spec.bfd.enabled + name: BFD ENABLED + type: boolean + - jsonPath: .spec.externalSubnet + name: EXTERNAL SUBNET + type: string + - jsonPath: .status.phase + name: PHASE + type: string + - jsonPath: .status.ready + name: READY + type: boolean + - jsonPath: .status.internalIPs + name: INTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.externalIPs + name: EXTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.workload.nodes + name: WORKING NODES + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + scale: + # specReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Spec.Replicas. + specReplicasPath: .spec.replicas + # statusReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Replicas. + statusReplicasPath: .status.replicas + # labelSelectorPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Selector. + labelSelectorPath: .status.labelSelector + schema: + openAPIV3Schema: + type: object + properties: + status: + properties: + replicas: + type: integer + format: int32 + labelSelector: + type: string + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + internalIPs: + items: + type: string + type: array + externalIPs: + items: + type: string + type: array + phase: + type: string + default: Pending + enum: + - Pending + - Processing + - Completed + ready: + type: boolean + default: false + workload: + type: object + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + nodes: + type: array + items: + type: string + required: + - conditions + - phase + type: object + spec: + type: object + required: + - externalSubnet + x-kubernetes-validations: + - rule: "!has(self.internalIPs) || size(self.internalIPs) == 0 || size(self.internalIPs) >= self.replicas" + message: 'Size of Internal IPs MUST be equal to or greater than Replicas' + fieldPath: ".internalIPs" + - rule: "!has(self.externalIPs) || size(self.externalIPs) == 0 || size(self.externalIPs) >= self.replicas" + message: 'Size of External IPs MUST be equal to or greater than Replicas' + fieldPath: ".externalIPs" + - rule: "size(self.policies) != 0 || size(self.selectors) != 0" + message: 'Each BGP Edeg Router MUST have at least one policy or selector' + properties: + replicas: + type: integer + format: int32 + default: 1 + minimum: 0 + maximum: 10 + prefix: + type: string + anyOf: + - pattern: ^$ + - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*[-\.]?$ + x-kubernetes-validations: + - rule: "self == oldSelf" + message: "This field is immutable." + vpc: + type: string + internalSubnet: + type: string + externalSubnet: + type: string + internalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + externalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + image: + type: string + bfd: + type: object + properties: + enabled: + type: boolean + default: false + minRX: + type: integer + format: int32 + default: 1000 + minTX: + type: integer + format: int32 + default: 1000 + multiplier: + type: integer + format: int32 + default: 3 + selectors: + type: array + items: + type: object + properties: + namespaceSelector: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + x-kubernetes-validations: + - rule: "size(self.matchLabels) != 0 || size(self.matchExpressions) != 0" + message: 'Each namespace selector MUST have at least one matchLabels or matchExpressions' + podSelector: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + x-kubernetes-validations: + - rule: "size(self.matchLabels) != 0 || size(self.matchExpressions) != 0" + message: 'Each pod selector MUST have at least one matchLabels or matchExpressions' + policies: + type: array + items: + type: object + properties: + snat: + type: boolean + default: false + ipBlocks: + type: array + x-kubernetes-list-type: set + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + - format: cidr + subnets: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + x-kubernetes-validations: + - rule: "size(self.ipBlocks) != 0 || size(self.subnets) != 0" + message: 'Each policy MUST have at least one ipBlock or subnet' + trafficPolicy: + type: string + enum: + - Local + - Cluster + default: Cluster + nodeSelector: + type: array + items: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator + bgp: + type: object + properties: + edgeRouterMode: + type: boolean + default: false + routeServerClient: + type: boolean + default: false + enabled: + type: boolean + default: false + image: + type: string + localAsn: + type: integer + format: int32 + minimum: 0 + remoteAsn: + type: integer + format: int32 + minimum: 0 + neighbors: + type: array + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + holdTime: + type: string + pattern: ^[0-9]+[smhd]$ + routerId: + type: string + # The routerId can be an IPv4 or IPv6 address, but we are not enforcing it here + # routerId could be nullable, so that we can use environment value to set it + # refer pkg/speaker/config.go#L186 + anyOf: + - format: ipv4 + - format: ipv6 + - pattern: '^$' + password: + type: string + enableGracefulRestart: + type: boolean + default: false + extraArgs: + type: array + items: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgp-edge-router-advertisements.kubeovn.io +spec: + group: kubeovn.io + names: + plural: bgp-edge-router-advertisements + singular: bgp-edge-router-advertisement + shortNames: + - ber-ad + kind: BgpEdgeRouterAdvertisement + listKind: BgpEdgeRouterAdvertisementList + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.subnet + name: SUBNET + type: string + - jsonPath: .spec.bgpEdgeRouter + name: BGP-EDGE-ROUTER + type: string + - jsonPath: .status.ready + name: READY + type: boolean + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + subnet: + type: array + minItems: 1 + x-kubernetes-list-type: set + items: + type: string + bgpEdgeRouter: + type: string + required: + - subnet + - bgpEdgeRouter + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + ready: + type: boolean + default: false + required: + - conditions + type: object +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gobgp-configs.kubeovn.io +spec: + group: kubeovn.io + names: + plural: gobgp-configs + singular: gobgp-config + shortNames: + - bgp-config + kind: GobgpConfig + listKind: GobgpConfigList + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: bgp-edge-router + type: string + jsonPath: .spec.bgpEdgeRouterInfo.name + - name: neighbor-address + type: string + jsonPath: .spec.neighbors[*].address + - name: toReceive-mode + type: string + jsonPath: .spec.neighbors[*].toReceive.allowed.mode + - name: toReceive-prefixes + type: string + jsonPath: .spec.neighbors[*].toReceive.allowed.prefixes + - name: toAdvertise-mode + type: string + jsonPath: .spec.neighbors[*].toAdvertise.allowed.mode + - name: toAdvertise-prefixes + type: string + jsonPath: .spec.neighbors[*].toAdvertise.allowed.prefixes + - name: READY + type: boolean + jsonPath: .status.ready + - name: AGE + type: date + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + required: + - bgpEdgeRouterInfo + - neighbors + properties: + bgpEdgeRouterInfo: + type: object + properties: + name: + type: string + namespace: + type: string + neighbors: + type: array + items: + type: object + required: + - address + - toAdvertise + - toReceive + properties: + address: + type: string + pattern: '^(([0-9]{1,3}\.){3}[0-9]{1,3}|[0-9a-fA-F:]+)$' + toAdvertise: + type: object + required: + - allowed + properties: + allowed: + type: object + required: + - mode + properties: + mode: + type: string + prefixes: + type: array + x-kubernetes-list-type: set + items: + type: string + pattern: '^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/\d{1,2})?|([0-9a-fA-F:]+(\/\d{1,3})?))$' + x-kubernetes-validations: + - rule: "self.mode != 'filtered' || (has(self.prefixes) && size(self.prefixes) > 0)" + message: "prefixes must be set and non-empty when mode is 'filtered'" + toReceive: + type: object + required: + - allowed + properties: + allowed: + type: object + required: + - mode + properties: + mode: + type: string + prefixes: + type: array + x-kubernetes-list-type: set + items: + type: string + pattern: '^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/\d{1,2})?|([0-9a-fA-F:]+(\/\d{1,3})?))$' + x-kubernetes-validations: + - rule: "self.mode != 'filtered' || (has(self.prefixes) && size(self.prefixes) > 0)" + message: "prefixes must be set and non-empty when mode is 'filtered'" + status: + type: object + properties: + ready: + type: boolean + default: false + conditions: + type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - type + items: + type: object + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + properties: + type: + type: string + maxLength: 316 + pattern: '^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$' + status: + type: string + enum: + - "True" + - "False" + - "Unknown" + reason: + type: string + minLength: 1 + maxLength: 1024 + pattern: '^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$' + message: + type: string + maxLength: 32768 + lastTransitionTime: + type: string + format: date-time + lastUpdateTime: + type: string + format: date-time + observedGeneration: + type: integer + format: int64 + minimum: 0 EOF cat < ovn-ovs-sa.yaml @@ -3471,6 +4162,12 @@ rules: - vpc-dnses/status - qos-policies - qos-policies/status + - bgp-edge-routers + - bgp-edge-routers/status + - bgp-edge-router-advertisements + - bgp-edge-router-advertisements/status + - gobgp-configs + - gobgp-configs/status verbs: - "*" - apiGroups: diff --git a/dist/images/update-bgp-policy.sh b/dist/images/update-bgp-policy.sh new file mode 100644 index 00000000000..696b4fedf6a --- /dev/null +++ b/dist/images/update-bgp-policy.sh @@ -0,0 +1,410 @@ +#!/usr/bin/env bash +# update-bgp-policy.sh - Hybrid Version with Smart Error Resilience +# shellcheck disable=SC2086,SC2155 + +set -u + +GOBGP_BIN=${GOBGP_BIN:-$(command -v gobgp || true)} +[[ -z "$GOBGP_BIN" ]] && { echo "ERROR: gobgp binary not found" >&2; exit 1; } + +die() { echo "ERROR: $*" >&2; exit 1; } + +usage() { + cat >&2 < + $0 flush-neighbor-policy + $0 flush-prefix-in + $0 flush-prefix-out + $0 add-prefix + $0 set-default-action + $0 --batch [ARGS1...] -- [ARGS2...] -- ... + +Examples: + # Single command execution + $0 set-neighbor-policy 1.1.1.1 + $0 flush-neighbor-policy 1.1.1.1 + $0 flush-prefix-in 1.1.1.1 + $0 flush-prefix-out 1.1.1.1 + $0 add-prefix in 1.1.1.1 "0.0.0.0/0 0..32","1.1.1.0/24","10.0.0.0/8 16..32" + + # Batch execution (multiple commands in one run) + $0 --batch set-neighbor-policy 1.1.1.1 -- add-prefix in 1.1.1.1 "10.0.0.0/8" + $0 --batch flush-prefix-in 1.1.1.1 -- add-prefix in 1.1.1.1 "192.168.0.0/16" -- flush-prefix-out 1.1.1.1 +EOF + exit 1 +} + +# Validate IPv4 format (simple regex) +validate_ip() { + local ip=$1 + if [[ ! $ip =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then + die "Invalid IPv4 address: $ip" + fi +} + +exec_cmd() { "$@" || die "failed: $*"; } + +# Global array to track failed commands for retry +declare -a FAILED_COMMANDS=() + +# Error-resilient command execution - continues even on failure and tracks failures +exec_cmd_safe() { + if "$@"; then + echo "Success: $*" + return 0 + else + echo "Warning: Failed to execute '$*' (continuing...)" >&2 + # Store the failed command for potential retry + FAILED_COMMANDS+=("$*") + return 1 + fi +} + +# Retry command execution - tries twice +exec_cmd_retry() { + local max_attempts=2 + local attempt=1 + + while [[ $attempt -le $max_attempts ]]; do + if "$@"; then + echo "Success: $* (attempt $attempt)" + return 0 + else + echo "Attempt $attempt failed: $*" >&2 + ((attempt++)) + if [[ $attempt -le $max_attempts ]]; then + echo "Retrying..." >&2 + sleep 1 + fi + fi + done + + echo "All attempts failed for: $*" >&2 + return 1 +} + +set_neighbor_policy() { + local nbr_ip=$1; validate_ip "$nbr_ip" + local prefix_in="prefix-${nbr_ip}-in" + local prefix_out="prefix-${nbr_ip}-out" + local nbr_name="neighbor-${nbr_ip}" + local stmt_in="stmt-${nbr_ip}-in" + local stmt_out="stmt-${nbr_ip}-out" + local policy_in="policy-${nbr_ip}-in" + local policy_out="policy-${nbr_ip}-out" + + echo "=== Setting policy for neighbor $nbr_ip ===" + echo "-> Creating prefix-lists" + exec_cmd $GOBGP_BIN policy prefix add $prefix_in #0.0.0.0/0 0..32 + exec_cmd $GOBGP_BIN policy prefix add $prefix_out #0.0.0.0/0 0..32 + + echo "-> Defining neighbor" + exec_cmd $GOBGP_BIN policy neighbor add $nbr_name $nbr_ip + + echo "-> Building inbound statement" + exec_cmd $GOBGP_BIN policy statement add $stmt_in + exec_cmd $GOBGP_BIN policy statement $stmt_in add action accept + exec_cmd $GOBGP_BIN policy statement $stmt_in add condition prefix $prefix_in + exec_cmd $GOBGP_BIN policy statement $stmt_in add condition neighbor $nbr_name + + echo "-> Building outbound statement" + exec_cmd $GOBGP_BIN policy statement add $stmt_out + exec_cmd $GOBGP_BIN policy statement $stmt_out add action accept + exec_cmd $GOBGP_BIN policy statement $stmt_out add condition prefix $prefix_out + exec_cmd $GOBGP_BIN policy statement $stmt_out add condition neighbor $nbr_name + + echo "-> Assembling policies" + exec_cmd $GOBGP_BIN policy add $policy_in $stmt_in + exec_cmd $GOBGP_BIN policy add $policy_out $stmt_out + + echo "-> Applying to global" + exec_cmd $GOBGP_BIN global policy import add $policy_in + exec_cmd $GOBGP_BIN global policy export add $policy_out + + echo "=== Policy set successfully for $nbr_ip ===" +} + +flush_neighbor_policy() { + local nbr_ip=$1; validate_ip "$nbr_ip" + local prefix_in="prefix-${nbr_ip}-in" + local prefix_out="prefix-${nbr_ip}-out" + local nbr_name="neighbor-${nbr_ip}" + local stmt_in="stmt-${nbr_ip}-in" + local stmt_out="stmt-${nbr_ip}-out" + local policy_in="policy-${nbr_ip}-in" + local policy_out="policy-${nbr_ip}-out" + + # Clear the failed commands array + FAILED_COMMANDS=() + + echo "=== Flushing policy for neighbor $nbr_ip (Smart Error-Resilient Mode) ===" + + # Phase 1: Remove from global policies (safe mode) + echo "-> Removing from global policies" + exec_cmd_safe $GOBGP_BIN global policy import del $policy_in + exec_cmd_safe $GOBGP_BIN global policy export del $policy_out + + # Phase 2: Remove policies (safe mode) + echo "-> Removing policies" + exec_cmd_safe $GOBGP_BIN policy del $policy_in + exec_cmd_safe $GOBGP_BIN policy del $policy_out + + # Phase 3: Remove statements (safe mode) + echo "-> Removing statements" + exec_cmd_safe $GOBGP_BIN policy statement del $stmt_in + exec_cmd_safe $GOBGP_BIN policy statement del $stmt_out + + # Phase 4: Remove neighbor definition (safe mode) + echo "-> Removing neighbor definition" + exec_cmd_safe $GOBGP_BIN policy neighbor del $nbr_name + + # Phase 5: Remove prefix-lists (safe mode) + echo "-> Removing prefix-lists" + exec_cmd_safe $GOBGP_BIN policy prefix del $prefix_in + exec_cmd_safe $GOBGP_BIN policy prefix del $prefix_out + + # Phase 6: Apply policy to neighbor (safe mode) + echo "-> Soft reset neighbor policy" + exec_cmd_safe $GOBGP_BIN neighbor $nbr_ip softreset + + # Check if any commands failed and need retry + if [[ ${#FAILED_COMMANDS[@]} -eq 0 ]]; then + echo "" + echo "=== Policy flush completed successfully for $nbr_ip ===" + echo "All commands executed successfully - no retry needed" + else + echo "" + echo "=== Retry Phase: Retrying ${#FAILED_COMMANDS[@]} failed command(s) ===" + + local retry_count=0 + for cmd in "${FAILED_COMMANDS[@]}"; do + ((retry_count++)) + echo "-> Retry $retry_count/${#FAILED_COMMANDS[@]}: $cmd" + # Convert string back to array for execution + eval "exec_cmd_retry $cmd" + done + + echo "" + echo "=== Policy flush completed for $nbr_ip (with selective retry) ===" + echo "Retried ${#FAILED_COMMANDS[@]} failed command(s)" + fi +} + +flush_prefix_in() { + local nbr_ip=$1; validate_ip "$nbr_ip" + local prefix_name="prefix-${nbr_ip}-in" + + echo "=== Flushing all entries from $prefix_name ===" + $GOBGP_BIN policy prefix $prefix_name 2>/dev/null \ + | awk 'NR>1 && NF>=2 { print $(NF-1), $NF }' \ + | while read -r iprange mask; do + echo "-> Deleting: $iprange $mask" + exec_cmd $GOBGP_BIN policy prefix del $prefix_name $iprange $mask + done + exec_cmd $GOBGP_BIN neighbor $nbr_ip softresetin + echo "=== All entries removed from $prefix_name ===" +} + +flush_prefix_out() { + local nbr_ip=$1; validate_ip "$nbr_ip" + local prefix_name="prefix-${nbr_ip}-out" + + echo "=== Flushing all entries from $prefix_name ===" + $GOBGP_BIN policy prefix $prefix_name 2>/dev/null \ + | awk 'NR>1 && NF>=2 { print $(NF-1), $NF }' \ + | while read -r iprange mask; do + echo "-> Deleting: $iprange $mask" + exec_cmd $GOBGP_BIN policy prefix del $prefix_name $iprange $mask + done + exec_cmd $GOBGP_BIN neighbor $nbr_ip softresetout + echo "=== All entries removed from $prefix_name ===" +} + +add_prefix() { + local dir=$1; shift + local nbr_ip=$1; shift + validate_ip "$nbr_ip" + [[ $dir != in && $dir != out ]] && die "Direction must be 'in' or 'out'" + local prefix_name="prefix-${nbr_ip}-${dir}" + + # split comma-separated list in first argument after IP + IFS=',' read -ra entries <<< "$*" + + echo "=== Adding prefixes to $prefix_name ===" + for entry in "${entries[@]}"; do + # Clean quotes and whitespace + entry="${entry%\"}" + entry="${entry#\"}" + entry="${entry##( )}" + entry="${entry%%( )}" + + if [[ $entry =~ ^([^[:space:]]+)[[:space:]]+(.+)$ ]]; then + local ip_pref=${BASH_REMATCH[1]} + local mask=${BASH_REMATCH[2]} + echo "-> Adding: $ip_pref $mask" + exec_cmd $GOBGP_BIN policy prefix add $prefix_name $ip_pref $mask + else + echo "-> Adding: $entry" + exec_cmd $GOBGP_BIN policy prefix add $prefix_name $entry + fi + done + exec_cmd $GOBGP_BIN neighbor $nbr_ip softreset$dir + echo "=== Done ===" +} + +validate_action() { + local action=$1 + if [[ "$action" != "accept" && "$action" != "reject" ]]; then + die "Invalid action: $action. Must be 'accept' or 'reject'" + fi +} + +set_default_action() { + local action=$1; validate_action "$action" + + echo "=== Setting default action to $action ===" + + echo "-> Applying default policy to global import" + exec_cmd $GOBGP_BIN global policy import add default $action + + echo "-> Applying default policy to global export" + exec_cmd $GOBGP_BIN global policy export add default $action + + echo "-> Soft reset neighbor policy" + # Get all neighbor IPs and perform soft reset for each + local neighbors=() + while IFS= read -r line; do + # Extract IP address from gobgp neighbor output (assuming first column is IP) + local neighbor_ip=$(echo "$line" | awk '{print $1}') + # Skip header lines and empty lines, validate IP format + if [[ "$neighbor_ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then + neighbors+=("$neighbor_ip") + fi + done < <($GOBGP_BIN neighbor 2>/dev/null | tail -n +2) + + if [[ ${#neighbors[@]} -eq 0 ]]; then + echo " No neighbors found to reset" + else + echo " Found ${#neighbors[@]} neighbor(s) to reset:" + for neighbor_ip in "${neighbors[@]}"; do + echo " -> Soft resetting neighbor: $neighbor_ip" + exec_cmd_safe $GOBGP_BIN neighbor "$neighbor_ip" softreset + done + fi + + echo "=== Default action set to $action successfully ===" +} + +# Execute a single command +execute_single_command() { + local cmd=$1; shift + + case "$cmd" in + set-neighbor-policy) + [[ $# -ne 1 ]] && die "set-neighbor-policy requires exactly 1 argument (NEIGHBOR_IP)" + set_neighbor_policy "$1" + ;; + flush-neighbor-policy) + [[ $# -ne 1 ]] && die "flush-neighbor-policy requires exactly 1 argument (NEIGHBOR_IP)" + flush_neighbor_policy "$1" + ;; + flush-prefix-in) + [[ $# -ne 1 ]] && die "flush-prefix-in requires exactly 1 argument (NEIGHBOR_IP)" + flush_prefix_in "$1" + ;; + flush-prefix-out) + [[ $# -ne 1 ]] && die "flush-prefix-out requires exactly 1 argument (NEIGHBOR_IP)" + flush_prefix_out "$1" + ;; + add-prefix) + [[ $# -lt 3 ]] && die "add-prefix requires at least 3 arguments (in|out NEIGHBOR_IP PREFIXES...)" + add_prefix "$@" + ;; + set-default-action) + [[ $# -ne 1 ]] && die "set-default-action requires exactly 1 argument (accept|reject)" + set_default_action "$1" + ;; + list-global-policy) + [[ $# -ne 0 ]] && die "list-global-policy requires exactly no argument" + list_global_policy + ;; + *) + die "Unknown command: $cmd" + ;; + esac +} + +# Parse batch commands separated by -- +parse_batch_commands() { + local -a current_cmd=() + local -a all_commands=() + + for arg in "$@"; do + if [[ "$arg" == "--" ]]; then + if [[ ${#current_cmd[@]} -gt 0 ]]; then + all_commands+=("$(printf '%s\n' "${current_cmd[@]}")") + current_cmd=() + fi + else + current_cmd+=("$arg") + fi + done + + # Add the last command if exists + if [[ ${#current_cmd[@]} -gt 0 ]]; then + all_commands+=("$(printf '%s\n' "${current_cmd[@]}")") + fi + + # Execute all commands + local cmd_count=1 + for cmd_str in "${all_commands[@]}"; do + echo "" + echo "Executing batch command #$cmd_count" + echo "-----------------------------------" + + # Convert newline-separated string back to array + local -a cmd_args=() + while IFS= read -r line; do + [[ -n "$line" ]] && cmd_args+=("$line") + done <<< "$cmd_str" + + if [[ ${#cmd_args[@]} -gt 0 ]]; then + execute_single_command "${cmd_args[@]}" + fi + + ((cmd_count++)) + done +} + +list_global_policy() { + echo "=== Global Policy ===" + $GOBGP_BIN global policy + echo "" + echo "=== Policy Prefix ===" + $GOBGP_BIN policy prefix +} + + +main() { + [[ $# -lt 1 ]] && usage + + # Check for batch mode + if [[ "$1" == "--batch" ]]; then + shift + [[ $# -lt 1 ]] && die "Batch mode requires at least one command" + + echo "Starting batch execution mode" + echo "=================================" + parse_batch_commands "$@" + echo "" + echo "All batch commands completed successfully" + + else + # Single command mode (original behavior) + execute_single_command "$@" + fi + echo "Update bgp policy completed successfully" +} + +main "$@" \ No newline at end of file diff --git a/dist/images/update-bgp-route.sh b/dist/images/update-bgp-route.sh new file mode 100644 index 00000000000..3fb58f21b69 --- /dev/null +++ b/dist/images/update-bgp-route.sh @@ -0,0 +1,363 @@ +#!/usr/bin/env bash +# update-bgp-route.sh +# shellcheck disable=SC2086,SC2155 + +set -euo pipefail + +GOBGP_BIN=${GOBGP_BIN:-$(command -v gobgp || true)} +[[ -z "$GOBGP_BIN" ]] && { echo "gobgp binary not found" >&2; exit 1; } + +die() { echo "ERROR: $*" >&2; exit 1; } + +external_iface="net1" +external_ipv4=$(ip addr show dev "${external_iface}" | grep 'inet ' | awk '{print $2}' | cut -d'/' -f1) +[[ -z "$external_ipv4" ]] && die "cannot determine external IPv4 address" + +exec_cmd() { + "$@" || die "failed: $*" +} + +check_inited() { + $GOBGP_BIN global rib &>/dev/null \ + || die "gobgp global RIB not initialized (did you 'gobgp global'?)" +} + +add_announced_route() { + check_inited + echo "Adding routes..." + for cidr in "$@"; do + if [[ $cidr == *:* ]]; then + family_flag="-a ipv6" + else + family_flag="-a ipv4" + fi + echo " + Adding: $cidr" + exec_cmd $GOBGP_BIN global rib $family_flag add \ + "$cidr" nexthop "$external_ipv4" origin igp + done + echo "" +} + +del_announced_route() { + check_inited + echo "Deleting routes..." + for cidr in "$@"; do + if [[ $cidr == *:* ]]; then + family_flag="-a ipv6" + else + family_flag="-a ipv4" + fi + echo " - Deleting: $cidr" + exec_cmd $GOBGP_BIN global rib $family_flag del \ + "$cidr" nexthop "$external_ipv4" origin igp + done + echo "" +} + +flush_announced_route() { + check_inited + echo "Flushing all routes with next-hop $external_ipv4..." + + local routes_to_delete=() + local found_routes=false + + # Get IPv4 routes with matching next-hop + local ipv4_routes + if ipv4_routes=$($GOBGP_BIN global rib -a ipv4 2>/dev/null | grep "$external_ipv4" | awk '{print $2}' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/[0-9]+$'); then + while IFS= read -r route; do + if [[ -n "$route" ]]; then + routes_to_delete+=("ipv4:$route") + found_routes=true + fi + done <<< "$ipv4_routes" + fi + + # Get IPv6 routes with matching next-hop + local ipv6_routes + if ipv6_routes=$($GOBGP_BIN global rib -a ipv6 2>/dev/null | grep "$external_ipv4" | awk '{print $2}' | grep -E '^[0-9a-fA-F:]+/[0-9]+$'); then + while IFS= read -r route; do + if [[ -n "$route" ]]; then + routes_to_delete+=("ipv6:$route") + found_routes=true + fi + done <<< "$ipv6_routes" + fi + + if [[ "$found_routes" == false ]]; then + echo " No routes found with next-hop $external_ipv4" + echo "" + return 0 + fi + + # Delete all found routes + for route_entry in "${routes_to_delete[@]}"; do + local family="${route_entry%%:*}" + local cidr="${route_entry#*:}" + echo " - Flushing: $cidr ($family)" + exec_cmd $GOBGP_BIN global rib -a "$family" del \ + "$cidr" nexthop "$external_ipv4" origin igp + done + + echo "Flushed ${#routes_to_delete[@]} routes with next-hop $external_ipv4" + echo "" +} + +list_announced_route() { + check_inited + + echo "=== BGP Global RIB ===" + echo "External Interface: $external_iface" + echo "External IPv4: $external_ipv4" + echo "" + + # Show IPv4 routes + echo "--- IPv4 Routes ---" + if $GOBGP_BIN global rib -a ipv4 2>/dev/null | grep -q "Network"; then + $GOBGP_BIN global rib -a ipv4 + else + echo "No IPv4 routes found" + fi + + echo "" + + # Show IPv6 routes + echo "--- IPv6 Routes ---" + if $GOBGP_BIN global rib -a ipv6 2>/dev/null | grep -q "Network"; then + $GOBGP_BIN global rib -a ipv6 + else + echo "No IPv6 routes found" + fi + + echo "" + + # Show routes that match our external IP as next-hop + echo "--- Routes with Next-Hop $external_ipv4 ---" + local found_matching=false + + # Check IPv4 routes with matching next-hop + if $GOBGP_BIN global rib -a ipv4 2>/dev/null | awk -v nh="$external_ipv4" 'NR==1 || $3 == nh' | grep -v "Network" | grep -q .; then + echo "IPv4 routes with next-hop $external_ipv4:" + $GOBGP_BIN global rib -a ipv4 | awk -v nh="$external_ipv4" 'NR==1 || $3 == nh' + found_matching=true + fi + + # Check IPv6 routes with matching next-hop + if $GOBGP_BIN global rib -a ipv6 2>/dev/null | awk -v nh="$external_ipv4" 'NR==1 || $3 == nh' | grep -v "Network" | grep -q .; then + echo "IPv6 routes with next-hop $external_ipv4:" + $GOBGP_BIN global rib -a ipv6 | awk -v nh="$external_ipv4" 'NR==1 || $3 == nh' + found_matching=true + fi + + if [[ "$found_matching" == false ]]; then + echo "No routes found with next-hop $external_ipv4" + fi + + echo "" + echo "==========================================" + echo "" +} + +parse_sequential_args() { + local operations=() + local operation_args=() + + # Parse arguments and store operations in order + for arg in "$@"; do + case "$arg" in + add_announced_route=*|add_announce_routes=*) + operations+=("add") + operation_args+=("${arg#*=}") + ;; + del_announced_route=*|del_announce_routes=*) + operations+=("del") + operation_args+=("${arg#*=}") + ;; + flush_announced_route) + operations+=("flush") + operation_args+=("") + ;; + list_announced_route) + operations+=("list") + operation_args+=("") + ;; + *) + echo "Unknown argument: $arg" >&2 + usage + ;; + esac + done + + # Execute operations in order + for i in "${!operations[@]}"; do + case "${operations[$i]}" in + list) + list_announced_route + ;; + flush) + flush_announced_route + ;; + del) + # Parse comma-separated CIDRs + IFS=',' read -ra del_cidrs <<< "${operation_args[$i]}" + # Remove leading and trailing spaces + for j in "${!del_cidrs[@]}"; do + del_cidrs[$j]=$(echo "${del_cidrs[$j]}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + done + del_announced_route "${del_cidrs[@]}" + ;; + add) + # Parse comma-separated CIDRs + IFS=',' read -ra add_cidrs <<< "${operation_args[$i]}" + # Remove leading and trailing spaces + for j in "${!add_cidrs[@]}"; do + add_cidrs[$j]=$(echo "${add_cidrs[$j]}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + done + add_announced_route "${add_cidrs[@]}" + ;; + esac + done +} + +parse_key_value_args() { + local add_routes="" + local del_routes="" + local list_routes=false + local flush_routes=false + + # find key=value + for arg in "$@"; do + case "$arg" in + add_announced_route=*|add_announce_routes=*) + add_routes="${arg#*=}" # extract after = values + ;; + del_announced_route=*|del_announce_routes=*) + del_routes="${arg#*=}" + ;; + flush_announced_route) + flush_routes=true + ;; + list_announced_route) + list_routes=true + ;; + *) + echo "Unknown argument: $arg" >&2 + usage + ;; + esac + done + + if [[ "$flush_routes" == true ]]; then + flush_announced_route + fi + + if [[ "$list_routes" == true ]]; then + list_announced_route + return 0 + fi + + if [[ -n "$del_routes" ]]; then + echo "Processing del_announced_route: $del_routes" + # change cidrs to array + IFS=',' read -ra del_cidrs <<< "$del_routes" + # remove leading and trailing spaces + for i in "${!del_cidrs[@]}"; do + del_cidrs[$i]=$(echo "${del_cidrs[$i]}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + done + del_announced_route "${del_cidrs[@]}" + fi + + if [[ -n "$add_routes" ]]; then + echo "Processing add_announced_route: $add_routes" + # change cidrs to array + IFS=',' read -ra add_cidrs <<< "$add_routes" + # remove leading and trailing spaces from each CIDR + for i in "${!add_cidrs[@]}"; do + add_cidrs[$i]=$(echo "${add_cidrs[$i]}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + done + add_announced_route "${add_cidrs[@]}" + fi +} + +usage() { + cat >&2 < [CIDR ...] + 2. Key-Value Arguments: $0 add_announced_route=CIDR1,CIDR2 [del_announced_route=CIDR3,CIDR4] [flush_announced_route] [list_announced_route] + 3. Sequential Processing: $0 list_announced_route del_announce_routes=CIDR1,CIDR2 add_announce_routes=CIDR3,CIDR4 flush_announced_route list_announced_route + +Examples: + $0 add_announced_route 10.100.0.0/24 192.168.1.0/24 + $0 del_announced_route 10.100.0.0/24 192.168.1.0/24 + $0 flush_announced_route + $0 list_announced_route + $0 add_announced_route=10.100.0.0/24,10.100.1.0/24 del_announced_route=10.0.0.0/24,10.0.1.0/24 + $0 flush_announced_route list_announced_route + $0 list_announced_route flush_announced_route add_announce_routes=10.0.0.0/24,10.0.1.0/24 list_announced_route + +Note: + - flush_announced_route removes ALL routes with next-hop $external_ipv4 + - Both 'del_announced_route=' and 'del_announce_routes=' are supported (same for add operations) +EOF + exit 1 +} + +has_sequential_processing() { + local has_list=false + local has_operations=false + + for arg in "$@"; do + case "$arg" in + list_announced_route|flush_announced_route) + has_list=true + ;; + add_announced_route=*|add_announce_routes=*|del_announced_route=*|del_announce_routes=*) + has_operations=true + ;; + esac + done + + # Return true if we have list/flush + operations (sequential processing) + [[ "$has_list" == true && "$has_operations" == true ]] +} + +has_key_value_args() { + for arg in "$@"; do + case "$arg" in + *=*|list_announced_route|flush_announced_route) + return 0 # key=value or list/flush command + ;; + esac + done + return 1 # key=value not found +} + +# main entry point +main() { + # if no arguments are provided, show usage + [[ $# -eq 0 ]] && usage + + # check if we need sequential processing (list/flush + operations) + if has_sequential_processing "$@"; then + parse_sequential_args "$@" + return 0 + fi + + # check if key=value arguments are used + if has_key_value_args "$@"; then + parse_key_value_args "$@" + return 0 + fi + + [[ $# -lt 1 ]] && usage + # TODO list announced routes network which nexthop is external_ipv4 + local op=$1; shift + case "$op" in + add_announced_route) add_announced_route "$@" ;; + del_announced_route) del_announced_route "$@" ;; + flush_announced_route) flush_announced_route ;; + list_announced_route) list_announced_route ;; + *) usage ;; + esac +} + +main "$@" \ No newline at end of file diff --git a/e2e.mk b/e2e.mk index f5ea1819559..92797ee3e4f 100644 --- a/e2e.mk +++ b/e2e.mk @@ -91,6 +91,7 @@ e2e-build: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/connectivity ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/metallb ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ipsec-cert-mgr + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/edge-router .PHONY: k8s-conformance-e2e k8s-conformance-e2e: @@ -278,3 +279,12 @@ kube-ovn-underlay-metallb-e2e: E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ --focus=CNI:Kube-OVN ./test/e2e/metallb/metallb.test -- $(TEST_BIN_ARGS) + +.PHONY: kube-ovn-edge-router-e2e +kube-ovn-edge-router-e2e: + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/edge-router + E2E_BRANCH=$(E2E_BRANCH) \ + E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ + E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \ + ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ + --focus=CNI:Kube-OVN ./test/e2e/edge-router/edge-router.test -- $(TEST_BIN_ARGS) diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index d048f706e4b..b31d3690272 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -4,12 +4,16 @@ set -o pipefail set -x SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. -CODEGEN_PKG=$GOPATH/src/k8s.io/code-generator +CODEGEN_PKG="/usr/local/pkg/mod/k8s.io/code-generator@v0.32.6" +MODULE=github.com/kubeovn/kube-ovn # generate the code with: # --output-base because this script should also be able to run inside the vendor dir of # k8s.io/kubernetes. The output-base is needed for the generators to output into the vendor dir # instead of the $GOPATH directly. For normal projects this can be dropped. -${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \ +# ${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \ +# github.com/kubeovn/kube-ovn/pkg/client github.com/kubeovn/kube-ovn/pkg/apis \ +# kubeovn:v1 +${CODEGEN_PKG}/kube_codegen.sh "deepcopy,client,informer,lister" \ github.com/kubeovn/kube-ovn/pkg/client github.com/kubeovn/kube-ovn/pkg/apis \ kubeovn:v1 diff --git a/pkg/apis/kubeovn/v1/bgp-edge-router-advertisement.go b/pkg/apis/kubeovn/v1/bgp-edge-router-advertisement.go new file mode 100644 index 00000000000..480b2087acf --- /dev/null +++ b/pkg/apis/kubeovn/v1/bgp-edge-router-advertisement.go @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type BgpEdgeRouterAdvertisementList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []BgpEdgeRouterAdvertisement `json:"items"` +} + +// +genclient +// +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale +// +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +resourceName=bgp-edge-router-advertisements +// bgp edge router advertisement is used to forward the egress traffic from the VPC to the external network +type BgpEdgeRouterAdvertisement struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + + Spec BgpEdgeRouterAdvertisementSpec `json:"spec"` + Status BgpEdgeRouterAdvertisementStatus `json:"status"` +} + +// If the BgpEdgeRouter has no VPC specified in the spec, it will return the default VPC name +func (g *BgpEdgeRouterAdvertisement) Subnet(subnets []string) []string { + if len(subnets) != 0 { + return subnets + } + return nil +} + +// Ready returns true if the BgpEdgeRouter has been processed successfully and is ready to serve traffic +// func (g *BgpEdgeRouterAdvertisement) Ready() bool { +// return g.Status.Ready && g.Status.Conditions.IsReady(g.Generation) +// } + +type BgpEdgeRouterAdvertisementSpec struct { + Subnet []string `json:"subnet,omitempty"` + BgpEdgeRouter string `json:"bgpEdgeRouter,omitempty"` +} + +type BgpEdgeRouterAdvertisementStatus struct { + Ready bool `json:"ready"` + Conditions Conditions `json:"conditions,omitempty"` +} diff --git a/pkg/apis/kubeovn/v1/bgp-edge-router.go b/pkg/apis/kubeovn/v1/bgp-edge-router.go new file mode 100644 index 00000000000..a3e30bdd730 --- /dev/null +++ b/pkg/apis/kubeovn/v1/bgp-edge-router.go @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: Apache-2.0 +package v1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type BgpEdgeRouterList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []BgpEdgeRouter `json:"items"` +} + +// +genclient +// +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale +// +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +resourceName=bgp-edge-routers +// bgp edge router is used to forward the egress traffic from the VPC to the external network +type BgpEdgeRouter struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + + Spec BgpEdgeRouterSpec `json:"spec"` + Status BgpEdgeRouterStatus `json:"status"` +} + +// VPC returns the VPC name +// If the BgpEdgeRouter has no VPC specified in the spec, it will return the default VPC name +func (g *BgpEdgeRouter) VPC(defaultVPC string) string { + if g.Spec.VPC != "" { + return g.Spec.VPC + } + return defaultVPC +} + +// Ready returns true if the BgpEdgeRouter has been processed successfully and is ready to serve traffic +func (g *BgpEdgeRouter) Ready() bool { + return g.Status.Ready && g.Status.Conditions.IsReady(g.Generation) +} + +type BgpEdgeRouterSpec struct { + // optional VPC name + // if not specified, the default VPC will be used + VPC string `json:"vpc,omitempty"` + // workload replicas + Replicas int32 `json:"replicas,omitempty"` + // optional name prefix used to generate the workload + // the workload name will be generated as + Prefix string `json:"prefix,omitempty"` + // optional image used by the workload + // if not specified, the default image passed in by kube-ovn-controller will be used + Image string `json:"image,omitempty"` + // optional internal subnet used to create the workload + // if not specified, the workload will be created in the default subnet of the VPC + InternalSubnet string `json:"internalSubnet,omitempty"` + // external subnet used to create the workload + ExternalSubnet string `json:"externalSubnet"` + // optional internal/external IPs used to create the workload + // these IPs must be in the internal/external subnet + // the IPs count must NOT be less than the replicas count + InternalIPs []string `json:"internalIPs,omitempty"` + ExternalIPs []string `json:"externalIPs,omitempty"` + // optional traffic policy used to control the traffic routing + // if not specified, the default traffic policy "Cluster" will be used + // if set to "Local", traffic will be routed to the gateway pod/instance on the same node when available + // currently it works only for the default vpc + TrafficPolicy string `json:"trafficPolicy,omitempty"` + + // BFD configuration + BFD BgpEdgeRouterBFDConfig `json:"bfd"` + // egress policies + // at least one policy must be specified + Policies []BgpEdgeRouterPolicy `json:"policies,omitempty"` + // optional node selector used to select the nodes where the workload will be running + NodeSelector []BgpEdgeRouterNodeSelector `json:"nodeSelector,omitempty"` + + // Add BGP configuration + BGP BgpEdgeRouterBGPConfig `json:"bgp"` + + // TODO, subnet to access kube-apiserver. this will be used to reconcile routes to vpc + // KubeApiSubnet string `json:"kubeApiSubnet,omitempty"` +} + +type BgpEdgeRouterBFDConfig struct { + // whether to enable BFD + // if set to true, the egress gateway will establish BFD session(s) with the VPC BFD LRP + // the VPC's .spec.bfd.enabled must be set to true to enable BFD + Enabled bool `json:"enabled"` + // optional BFD minRX/minTX/multiplier + MinRX int32 `json:"minRX,omitempty"` + MinTX int32 `json:"minTX,omitempty"` + Multiplier int32 `json:"multiplier,omitempty"` +} + +type BgpEdgeRouterPolicy struct { + // whether to enable SNAT/MASQUERADE for the egress traffic + SNAT bool `json:"snat"` + // CIDRs/subnets targeted by the egress traffic policy + IPBlocks []string `json:"ipBlocks,omitempty"` + Subnets []string `json:"subnets,omitempty"` +} + +type BgpEdgeRouterNodeSelector struct { + MatchLabels map[string]string `json:"matchLabels,omitempty"` + MatchExpressions []corev1.NodeSelectorRequirement `json:"matchExpressions,omitempty"` + MatchFields []corev1.NodeSelectorRequirement `json:"matchFields,omitempty"` +} + +type BgpEdgeRouterStatus struct { + // used by the scale subresource + Replicas int32 `json:"replicas,omitempty"` + LabelSelector string `json:"labelSelector,omitempty"` + + // whether the egress gateway is ready + Ready bool `json:"ready"` + Phase Phase `json:"phase"` + // internal/external IPs used by the workload + InternalIPs []string `json:"internalIPs,omitempty"` + ExternalIPs []string `json:"externalIPs,omitempty"` + Conditions Conditions `json:"conditions,omitempty"` + + // workload information + Workload BgpEdgeRouterWorkload `json:"workload"` +} + +type BgpEdgeRouterWorkload struct { + APIVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + Name string `json:"name,omitempty"` + // nodes where the workload is running + Nodes []string `json:"nodes,omitempty"` +} + +type BgpEdgeRouterBGPConfig struct { + // whether to enable BGP for the egress gateway + Enabled bool `json:"enabled"` + // optional bgp image used by the workload + // if not specified, the default image passed in by kube-ovn-controller will be used + Image string `json:"image,omitempty"` + ASN uint32 `json:"localAsn"` + RemoteASN uint32 `json:"remoteAsn"` + Neighbors []string `json:"neighbors"` + HoldTime metav1.Duration `json:"holdTime"` + RouterID string `json:"routerId"` + Password string `json:"password"` + EnableGracefulRestart bool `json:"enableGracefulRestart"` + ExtraArgs []string `json:"extraArgs"` + EdgeRouterMode bool `json:"edgeRouterMode"` + RouteServerClient bool `json:"routeServerClient"` +} diff --git a/pkg/apis/kubeovn/v1/gobgp-config.go b/pkg/apis/kubeovn/v1/gobgp-config.go new file mode 100644 index 00000000000..328d667ee80 --- /dev/null +++ b/pkg/apis/kubeovn/v1/gobgp-config.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type GobgpConfigList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []GobgpConfig `json:"items"` +} + +// +genclient +// +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale +// +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +resourceName=gobgp-configs +// bgp edge router advertisement is used to forward the egress traffic from the VPC to the external network +type GobgpConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + + Spec GobgpConfigSpec `json:"spec"` + Status GobgpConfigStatus `json:"status"` +} + +// If the GobgpConfig has no VPC specified in the spec, it will return the default VPC name +func (g *GobgpConfig) Subnet(subnets []string) []string { + if len(subnets) != 0 { + return subnets + } + return nil +} + +type GobgpConfigSpec struct { + BgpEdgeRouter string `json:"bgpEdgeRouter"` + Neighbors []Neighbors `json:"neighbors,omitempty"` +} + +type GobgpConfigStatus struct { + Ready bool `json:"ready"` + Conditions Conditions `json:"conditions,omitempty"` +} + +// Neighbors defines the BGP neighbors configuration +// +k8s:openapi-gen=true +// +genclient:nonNamespaced +type Neighbors struct { + Address string `json:"address"` + ToAdvertise ToAdvertise `json:"toAdvertise"` + ToReceive ToReceive `json:"toReceive"` +} + +type ToAdvertise struct { + Allowed Allowed `json:"allowed"` +} + +type ToReceive struct { + Allowed Allowed `json:"allowed"` +} + +type Allowed struct { + Mode string `json:"mode,omitempty"` + Prefixes []string `json:"prefixes,omitempty"` +} diff --git a/pkg/apis/kubeovn/v1/register.go b/pkg/apis/kubeovn/v1/register.go index 020f13f8dc2..d6cee08fd91 100644 --- a/pkg/apis/kubeovn/v1/register.go +++ b/pkg/apis/kubeovn/v1/register.go @@ -73,6 +73,12 @@ func addKnownTypes(scheme *runtime.Scheme) error { &VpcEgressGatewayList{}, &VpcNatGateway{}, &VpcNatGatewayList{}, + &BgpEdgeRouter{}, + &BgpEdgeRouterList{}, + &BgpEdgeRouterAdvertisement{}, + &BgpEdgeRouterAdvertisementList{}, + &GobgpConfig{}, + &GobgpConfigList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/pkg/apis/kubeovn/v1/vpc-egress-gateway.go b/pkg/apis/kubeovn/v1/vpc-egress-gateway.go index 3404804a88a..06889a67e30 100644 --- a/pkg/apis/kubeovn/v1/vpc-egress-gateway.go +++ b/pkg/apis/kubeovn/v1/vpc-egress-gateway.go @@ -95,6 +95,12 @@ type VpcEgressGatewaySpec struct { Policies []VpcEgressGatewayPolicy `json:"policies,omitempty"` // optional node selector used to select the nodes where the workload will be running NodeSelector []VpcEgressGatewayNodeSelector `json:"nodeSelector,omitempty"` + + // Add BGP configuration + BGP VpcEgressGatewayBGPConfig `json:"bgp"` + + // TODO, subnet to access kube-apiserver. this will be used to reconcile routes to vpc + // KubeApiSubnet string `json:"kubeApiSubnet,omitempty"` } type VpcEgressGatewaySelector struct { @@ -151,3 +157,20 @@ type VpcEgressWorkload struct { // nodes where the workload is running Nodes []string `json:"nodes,omitempty"` } + +type VpcEgressGatewayBGPConfig struct { + EdgeRouterMode bool `json:"edgeRouterMode"` + // whether to enable BGP for the egress gateway + Enabled bool `json:"enabled"` + // optional bgp image used by the workload + // if not specified, the default image passed in by kube-ovn-controller will be used + Image string `json:"image,omitempty"` + ASN uint32 `json:"asn"` + RemoteASN uint32 `json:"remoteAsn"` + Neighbors []string `json:"neighbors"` + HoldTime metav1.Duration `json:"holdTime"` + RouterID string `json:"routerId"` + Password string `json:"password"` + EnableGracefulRestart bool `json:"enableGracefulRestart"` + ExtraArgs []string `json:"extraArgs"` +} diff --git a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go index ae3130280da..d721f5afd93 100644 --- a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go +++ b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go @@ -2840,3 +2840,437 @@ func (in *VpcStatus) DeepCopy() *VpcStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouter) DeepCopyInto(out *BgpEdgeRouter) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGateway. +func (in *BgpEdgeRouter) DeepCopy() *BgpEdgeRouter { + if in == nil { + return nil + } + out := new(BgpEdgeRouter) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BgpEdgeRouter) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterBFDConfig) DeepCopyInto(out *BgpEdgeRouterBFDConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayBFDConfig. +func (in *BgpEdgeRouterBFDConfig) DeepCopy() *BgpEdgeRouterBFDConfig { + if in == nil { + return nil + } + out := new(BgpEdgeRouterBFDConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterList) DeepCopyInto(out *BgpEdgeRouterList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]BgpEdgeRouter, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayList. +func (in *BgpEdgeRouterList) DeepCopy() *BgpEdgeRouterList { + if in == nil { + return nil + } + out := new(BgpEdgeRouterList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BgpEdgeRouterList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterNodeSelector) DeepCopyInto(out *BgpEdgeRouterNodeSelector) { + *out = *in + if in.MatchLabels != nil { + in, out := &in.MatchLabels, &out.MatchLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.MatchExpressions != nil { + in, out := &in.MatchExpressions, &out.MatchExpressions + *out = make([]corev1.NodeSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.MatchFields != nil { + in, out := &in.MatchFields, &out.MatchFields + *out = make([]corev1.NodeSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayNodeSelector. +func (in *BgpEdgeRouterNodeSelector) DeepCopy() *BgpEdgeRouterNodeSelector { + if in == nil { + return nil + } + out := new(BgpEdgeRouterNodeSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterPolicy) DeepCopyInto(out *BgpEdgeRouterPolicy) { + *out = *in + if in.IPBlocks != nil { + in, out := &in.IPBlocks, &out.IPBlocks + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayPolicy. +func (in *BgpEdgeRouterPolicy) DeepCopy() *BgpEdgeRouterPolicy { + if in == nil { + return nil + } + out := new(BgpEdgeRouterPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterSpec) DeepCopyInto(out *BgpEdgeRouterSpec) { + *out = *in + if in.InternalIPs != nil { + in, out := &in.InternalIPs, &out.InternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalIPs != nil { + in, out := &in.ExternalIPs, &out.ExternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.BFD = in.BFD + if in.Policies != nil { + in, out := &in.Policies, &out.Policies + *out = make([]BgpEdgeRouterPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make([]BgpEdgeRouterNodeSelector, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpEdgeRouterSpec. +func (in *BgpEdgeRouterSpec) DeepCopy() *BgpEdgeRouterSpec { + if in == nil { + return nil + } + out := new(BgpEdgeRouterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterStatus) DeepCopyInto(out *BgpEdgeRouterStatus) { + *out = *in + if in.InternalIPs != nil { + in, out := &in.InternalIPs, &out.InternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalIPs != nil { + in, out := &in.ExternalIPs, &out.ExternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Workload.DeepCopyInto(&out.Workload) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpEdgeRouterStatus. +func (in *BgpEdgeRouterStatus) DeepCopy() *BgpEdgeRouterStatus { + if in == nil { + return nil + } + out := new(BgpEdgeRouterStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterWorkload) DeepCopyInto(out *BgpEdgeRouterWorkload) { + *out = *in + if in.Nodes != nil { + in, out := &in.Nodes, &out.Nodes + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpEdgeRouterWorkload. +func (in *BgpEdgeRouterWorkload) DeepCopy() *BgpEdgeRouterWorkload { + if in == nil { + return nil + } + out := new(BgpEdgeRouterWorkload) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterAdvertisement) DeepCopyInto(out *BgpEdgeRouterAdvertisement) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGateway. +func (in *BgpEdgeRouterAdvertisement) DeepCopy() *BgpEdgeRouterAdvertisement { + if in == nil { + return nil + } + out := new(BgpEdgeRouterAdvertisement) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BgpEdgeRouterAdvertisement) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterAdvertisementList) DeepCopyInto(out *BgpEdgeRouterAdvertisementList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]BgpEdgeRouterAdvertisement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayList. +func (in *BgpEdgeRouterAdvertisementList) DeepCopy() *BgpEdgeRouterAdvertisementList { + if in == nil { + return nil + } + out := new(BgpEdgeRouterAdvertisementList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BgpEdgeRouterAdvertisementList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpEdgeRouterAdvertisementSpec) DeepCopyInto(out *BgpEdgeRouterAdvertisementSpec) { + *out = *in + if in.Subnet != nil { + in, out := &in.Subnet, &out.Subnet + *out = *in + } + if in.BgpEdgeRouter != "" { + in, out := &in.BgpEdgeRouter, &out.BgpEdgeRouter + *out = *in + } + + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpEdgeRouterAdvertisementSpec. +func (in *BgpEdgeRouterAdvertisementSpec) DeepCopy() *BgpEdgeRouterAdvertisementSpec { + if in == nil { + return nil + } + out := new(BgpEdgeRouterAdvertisementSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GobgpConfig) DeepCopyInto(out *GobgpConfig) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGateway. +func (in *GobgpConfig) DeepCopy() *GobgpConfig { + if in == nil { + return nil + } + out := new(GobgpConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GobgpConfig) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GobgpConfigList) DeepCopyInto(out *GobgpConfigList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GobgpConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayList. +func (in *GobgpConfigList) DeepCopy() *GobgpConfigList { + if in == nil { + return nil + } + out := new(GobgpConfigList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GobgpConfigList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GobgpConfigSpec) DeepCopyInto(out *GobgpConfigSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcDNSSpec. +func (in *GobgpConfigSpec) DeepCopy() *GobgpConfigSpec { + if in == nil { + return nil + } + out := new(GobgpConfigSpec) + in.DeepCopyInto(out) + return out +} + +// // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +// func (in *ToAdvertise) DeepCopyInto(out *ToAdvertise) { +// *out = *in +// if in != nil { +// in, out := &in.Allowed, &out.Allowed +// out = new(Allowed) +// (in).DeepCopyInto(out) +// } + +// return +// } + +// func (in *ToReceive) DeepCopyInto(out *ToReceive) { +// *out = *in +// if in != nil { +// in, out := &in.Allowed, &out.Allowed +// out = new(Allowed) +// (in).DeepCopyInto(out) +// } + +// return +// } + +// func (in *Allowed) DeepCopyInto(out *Allowed) { +// *out = *in +// if in.Mode != "" { +// in, out := &in.Mode, &out.Mode +// *out = *in +// } +// if in.Prefixes != nil { +// in, out := &in.Prefixes, &out.Prefixes +// *out = make([]string, len(*in)) +// copy(*out, *in) +// } +// return +// } diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go new file mode 100644 index 00000000000..5c186b15062 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouter.go @@ -0,0 +1,103 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + scheme "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" + autoscalingv1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// BgpEdgeRoutersGetter has a method to return a BgpEdgeRouterInterface. +// A group's client should implement this interface. +type BgpEdgeRoutersGetter interface { + BgpEdgeRouters(namespace string) BgpEdgeRouterInterface +} + +// BgpEdgeRouterInterface has methods to work with BgpEdgeRouter resources. +type BgpEdgeRouterInterface interface { + Create(ctx context.Context, bgpEdgeRouter *kubeovnv1.BgpEdgeRouter, opts metav1.CreateOptions) (*kubeovnv1.BgpEdgeRouter, error) + Update(ctx context.Context, bgpEdgeRouter *kubeovnv1.BgpEdgeRouter, opts metav1.UpdateOptions) (*kubeovnv1.BgpEdgeRouter, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, bgpEdgeRouter *kubeovnv1.BgpEdgeRouter, opts metav1.UpdateOptions) (*kubeovnv1.BgpEdgeRouter, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*kubeovnv1.BgpEdgeRouter, error) + List(ctx context.Context, opts metav1.ListOptions) (*kubeovnv1.BgpEdgeRouterList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *kubeovnv1.BgpEdgeRouter, err error) + GetScale(ctx context.Context, bgpEdgeRouterName string, options metav1.GetOptions) (*autoscalingv1.Scale, error) + UpdateScale(ctx context.Context, bgpEdgeRouterName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (*autoscalingv1.Scale, error) + + BgpEdgeRouterExpansion +} + +// bgpEdgeRouters implements BgpEdgeRouterInterface +type bgpEdgeRouters struct { + *gentype.ClientWithList[*kubeovnv1.BgpEdgeRouter, *kubeovnv1.BgpEdgeRouterList] +} + +// newBgpEdgeRouters returns a BgpEdgeRouters +func newBgpEdgeRouters(c *KubeovnV1Client, namespace string) *bgpEdgeRouters { + return &bgpEdgeRouters{ + gentype.NewClientWithList[*kubeovnv1.BgpEdgeRouter, *kubeovnv1.BgpEdgeRouterList]( + "bgp-edge-routers", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *kubeovnv1.BgpEdgeRouter { return &kubeovnv1.BgpEdgeRouter{} }, + func() *kubeovnv1.BgpEdgeRouterList { return &kubeovnv1.BgpEdgeRouterList{} }, + ), + } +} + +// GetScale takes name of the bgpEdgeRouter, and returns the corresponding autoscalingv1.Scale object, and an error if there is any. +func (c *bgpEdgeRouters) GetScale(ctx context.Context, bgpEdgeRouterName string, options metav1.GetOptions) (result *autoscalingv1.Scale, err error) { + result = &autoscalingv1.Scale{} + err = c.GetClient().Get(). + Namespace(c.GetNamespace()). + Resource("bgp-edge-routers"). + Name(bgpEdgeRouterName). + SubResource("scale"). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// UpdateScale takes the top resource name and the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *bgpEdgeRouters) UpdateScale(ctx context.Context, bgpEdgeRouterName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (result *autoscalingv1.Scale, err error) { + result = &autoscalingv1.Scale{} + err = c.GetClient().Put(). + Namespace(c.GetNamespace()). + Resource("bgp-edge-routers"). + Name(bgpEdgeRouterName). + SubResource("scale"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(scale). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouteradvertisement.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouteradvertisement.go new file mode 100644 index 00000000000..c0ee6dcd3d9 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/bgpedgerouteradvertisement.go @@ -0,0 +1,103 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + scheme "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" + autoscalingv1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// BgpEdgeRouterAdvertisementsGetter has a method to return a BgpEdgeRouterAdvertisementInterface. +// A group's client should implement this interface. +type BgpEdgeRouterAdvertisementsGetter interface { + BgpEdgeRouterAdvertisements(namespace string) BgpEdgeRouterAdvertisementInterface +} + +// BgpEdgeRouterAdvertisementInterface has methods to work with BgpEdgeRouterAdvertisement resources. +type BgpEdgeRouterAdvertisementInterface interface { + Create(ctx context.Context, bgpEdgeRouterAdvertisement *kubeovnv1.BgpEdgeRouterAdvertisement, opts metav1.CreateOptions) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) + Update(ctx context.Context, bgpEdgeRouterAdvertisement *kubeovnv1.BgpEdgeRouterAdvertisement, opts metav1.UpdateOptions) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, bgpEdgeRouterAdvertisement *kubeovnv1.BgpEdgeRouterAdvertisement, opts metav1.UpdateOptions) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) + List(ctx context.Context, opts metav1.ListOptions) (*kubeovnv1.BgpEdgeRouterAdvertisementList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *kubeovnv1.BgpEdgeRouterAdvertisement, err error) + GetScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, options metav1.GetOptions) (*autoscalingv1.Scale, error) + UpdateScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (*autoscalingv1.Scale, error) + + BgpEdgeRouterAdvertisementExpansion +} + +// bgpEdgeRouterAdvertisements implements BgpEdgeRouterAdvertisementInterface +type bgpEdgeRouterAdvertisements struct { + *gentype.ClientWithList[*kubeovnv1.BgpEdgeRouterAdvertisement, *kubeovnv1.BgpEdgeRouterAdvertisementList] +} + +// newBgpEdgeRouters returns a BgpEdgeRouters +func newBgpEdgeRouterAdvertisements(c *KubeovnV1Client, namespace string) *bgpEdgeRouterAdvertisements { + return &bgpEdgeRouterAdvertisements{ + gentype.NewClientWithList[*kubeovnv1.BgpEdgeRouterAdvertisement, *kubeovnv1.BgpEdgeRouterAdvertisementList]( + "bgp-edge-router-advertisements", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *kubeovnv1.BgpEdgeRouterAdvertisement { return &kubeovnv1.BgpEdgeRouterAdvertisement{} }, + func() *kubeovnv1.BgpEdgeRouterAdvertisementList { return &kubeovnv1.BgpEdgeRouterAdvertisementList{} }, + ), + } +} + +// GetScale takes name of the bgpEdgeRouter, and returns the corresponding autoscalingv1.Scale object, and an error if there is any. +func (c *bgpEdgeRouterAdvertisements) GetScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, options metav1.GetOptions) (result *autoscalingv1.Scale, err error) { + result = &autoscalingv1.Scale{} + err = c.GetClient().Get(). + Namespace(c.GetNamespace()). + Resource("bgp-edge-router-advertisements"). + Name(bgpEdgeRouterAdvertisementName). + SubResource("scale"). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// UpdateScale takes the top resource name and the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *bgpEdgeRouterAdvertisements) UpdateScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (result *autoscalingv1.Scale, err error) { + result = &autoscalingv1.Scale{} + err = c.GetClient().Put(). + Namespace(c.GetNamespace()). + Resource("bgp-edge-router-advertisements"). + Name(bgpEdgeRouterAdvertisementName). + SubResource("scale"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(scale). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouter.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouter.go new file mode 100644 index 00000000000..15bfe5383b1 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouter.go @@ -0,0 +1,79 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + context "context" + + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gentype "k8s.io/client-go/gentype" + testing "k8s.io/client-go/testing" +) + +// fakeBgpEdgeRouters implements BgpEdgeRouterInterface +type fakeBgpEdgeRouters struct { + *gentype.FakeClientWithList[*v1.BgpEdgeRouter, *v1.BgpEdgeRouterList] + Fake *FakeKubeovnV1 +} + +func newFakeBgpEdgeRouters(fake *FakeKubeovnV1, namespace string) kubeovnv1.BgpEdgeRouterInterface { + return &fakeBgpEdgeRouters{ + gentype.NewFakeClientWithList[*v1.BgpEdgeRouter, *v1.BgpEdgeRouterList]( + fake.Fake, + namespace, + v1.SchemeGroupVersion.WithResource("bgp-edge-routers"), + v1.SchemeGroupVersion.WithKind("BgpEdgeRouter"), + func() *v1.BgpEdgeRouter { return &v1.BgpEdgeRouter{} }, + func() *v1.BgpEdgeRouterList { return &v1.BgpEdgeRouterList{} }, + func(dst, src *v1.BgpEdgeRouterList) { dst.ListMeta = src.ListMeta }, + func(list *v1.BgpEdgeRouterList) []*v1.BgpEdgeRouter { return gentype.ToPointerSlice(list.Items) }, + func(list *v1.BgpEdgeRouterList, items []*v1.BgpEdgeRouter) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} + +// GetScale takes name of the bgpEdgeRouter, and returns the corresponding scale object, and an error if there is any. +func (c *fakeBgpEdgeRouters) GetScale(ctx context.Context, bgpEdgeRouterName string, options metav1.GetOptions) (result *autoscalingv1.Scale, err error) { + emptyResult := &autoscalingv1.Scale{} + obj, err := c.Fake. + Invokes(testing.NewGetSubresourceActionWithOptions(c.Resource(), c.Namespace(), "scale", bgpEdgeRouterName, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*autoscalingv1.Scale), err +} + +// UpdateScale takes the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *fakeBgpEdgeRouters) UpdateScale(ctx context.Context, bgpEdgeRouterName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (result *autoscalingv1.Scale, err error) { + emptyResult := &autoscalingv1.Scale{} + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceActionWithOptions(c.Resource(), "scale", c.Namespace(), scale, opts), &autoscalingv1.Scale{}) + + if obj == nil { + return emptyResult, err + } + return obj.(*autoscalingv1.Scale), err +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouteradvertisement.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouteradvertisement.go new file mode 100644 index 00000000000..a73909af703 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_bgpedgerouteradvertisement.go @@ -0,0 +1,81 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + context "context" + + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gentype "k8s.io/client-go/gentype" + testing "k8s.io/client-go/testing" +) + +// fakeBgpEdgeRouters implements BgpEdgeRouterInterface +type fakeBgpEdgeRouterAdvertisements struct { + *gentype.FakeClientWithList[*v1.BgpEdgeRouterAdvertisement, *v1.BgpEdgeRouterAdvertisementList] + Fake *FakeKubeovnV1 +} + +func newFakeBgpEdgeRouterAdvertisements(fake *FakeKubeovnV1, namespace string) kubeovnv1.BgpEdgeRouterAdvertisementInterface { + return &fakeBgpEdgeRouterAdvertisements{ + gentype.NewFakeClientWithList[*v1.BgpEdgeRouterAdvertisement, *v1.BgpEdgeRouterAdvertisementList]( + fake.Fake, + namespace, + v1.SchemeGroupVersion.WithResource("bgp-edge-router-advertisements"), + v1.SchemeGroupVersion.WithKind("BgpEdgeRouterAdvertisement"), + func() *v1.BgpEdgeRouterAdvertisement { return &v1.BgpEdgeRouterAdvertisement{} }, + func() *v1.BgpEdgeRouterAdvertisementList { return &v1.BgpEdgeRouterAdvertisementList{} }, + func(dst, src *v1.BgpEdgeRouterAdvertisementList) { dst.ListMeta = src.ListMeta }, + func(list *v1.BgpEdgeRouterAdvertisementList) []*v1.BgpEdgeRouterAdvertisement { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1.BgpEdgeRouterAdvertisementList, items []*v1.BgpEdgeRouterAdvertisement) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} + +// GetScale takes name of the bgpEdgeRouter, and returns the corresponding scale object, and an error if there is any. +func (c *fakeBgpEdgeRouterAdvertisements) GetScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, options metav1.GetOptions) (result *autoscalingv1.Scale, err error) { + emptyResult := &autoscalingv1.Scale{} + obj, err := c.Fake. + Invokes(testing.NewGetSubresourceActionWithOptions(c.Resource(), c.Namespace(), "scale", bgpEdgeRouterAdvertisementName, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*autoscalingv1.Scale), err +} + +// UpdateScale takes the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *fakeBgpEdgeRouterAdvertisements) UpdateScale(ctx context.Context, bgpEdgeRouterAdvertisementName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (result *autoscalingv1.Scale, err error) { + emptyResult := &autoscalingv1.Scale{} + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceActionWithOptions(c.Resource(), "scale", c.Namespace(), scale, opts), &autoscalingv1.Scale{}) + + if obj == nil { + return emptyResult, err + } + return obj.(*autoscalingv1.Scale), err +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_gobgpconfig.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_gobgpconfig.go new file mode 100644 index 00000000000..e8c05c5449f --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_gobgpconfig.go @@ -0,0 +1,81 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + context "context" + + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gentype "k8s.io/client-go/gentype" + testing "k8s.io/client-go/testing" +) + +// fakeGobgpConfigs implements GobgpConfigInterface +type fakeGobgpConfigs struct { + *gentype.FakeClientWithList[*v1.GobgpConfig, *v1.GobgpConfigList] + Fake *FakeKubeovnV1 +} + +func newFakeGobgpConfigs(fake *FakeKubeovnV1, namespace string) kubeovnv1.GobgpConfigInterface { + return &fakeGobgpConfigs{ + gentype.NewFakeClientWithList[*v1.GobgpConfig, *v1.GobgpConfigList]( + fake.Fake, + namespace, + v1.SchemeGroupVersion.WithResource("gobgp-configs"), + v1.SchemeGroupVersion.WithKind("GobgpConfig"), + func() *v1.GobgpConfig { return &v1.GobgpConfig{} }, + func() *v1.GobgpConfigList { return &v1.GobgpConfigList{} }, + func(dst, src *v1.GobgpConfigList) { dst.ListMeta = src.ListMeta }, + func(list *v1.GobgpConfigList) []*v1.GobgpConfig { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1.GobgpConfigList, items []*v1.GobgpConfig) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} + +// GetScale takes name of the gobgpConfig, and returns the corresponding scale object, and an error if there is any. +func (c *fakeGobgpConfigs) GetScale(ctx context.Context, gobgpConfigName string, options metav1.GetOptions) (result *autoscalingv1.Scale, err error) { + emptyResult := &autoscalingv1.Scale{} + obj, err := c.Fake. + Invokes(testing.NewGetSubresourceActionWithOptions(c.Resource(), c.Namespace(), "scale", gobgpConfigName, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*autoscalingv1.Scale), err +} + +// UpdateScale takes the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *fakeGobgpConfigs) UpdateScale(ctx context.Context, gobgpConfigName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (result *autoscalingv1.Scale, err error) { + emptyResult := &autoscalingv1.Scale{} + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceActionWithOptions(c.Resource(), "scale", c.Namespace(), scale, opts), &autoscalingv1.Scale{}) + + if obj == nil { + return emptyResult, err + } + return obj.(*autoscalingv1.Scale), err +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go index 8be54ee7e18..dec025c5dd7 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go @@ -112,8 +112,20 @@ func (c *FakeKubeovnV1) VpcNatGateways() v1.VpcNatGatewayInterface { return newFakeVpcNatGateways(c) } +func (c *FakeKubeovnV1) BgpEdgeRouters(namespace string) v1.BgpEdgeRouterInterface { + return newFakeBgpEdgeRouters(c, namespace) +} + +func (c *FakeKubeovnV1) BgpEdgeRouterAdvertisements(namespace string) v1.BgpEdgeRouterAdvertisementInterface { + return newFakeBgpEdgeRouterAdvertisements(c, namespace) +} + +func (c *FakeKubeovnV1) GobgpConfigs(namespace string) v1.GobgpConfigInterface { + return newFakeGobgpConfigs(c, namespace) +} + // RESTClient returns a RESTClient that is used to communicate -// with API server by this client implementation. +// with API server by this client implementation.p func (c *FakeKubeovnV1) RESTClient() rest.Interface { var ret *rest.RESTClient return ret diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go index a4a3dcf75f0..f487e31b210 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go @@ -59,3 +59,9 @@ type VpcDnsExpansion interface{} type VpcEgressGatewayExpansion interface{} type VpcNatGatewayExpansion interface{} + +type BgpEdgeRouterExpansion interface{} + +type BgpEdgeRouterAdvertisementExpansion interface{} + +type GobgpConfigExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/gobgpconfig.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/gobgpconfig.go new file mode 100644 index 00000000000..3e31d04a744 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/gobgpconfig.go @@ -0,0 +1,103 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + scheme "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" + autoscalingv1 "k8s.io/api/autoscaling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// GobgpConfigsGetter has a method to return a GobgpConfigInterface. +// A group's client should implement this interface. +type GobgpConfigsGetter interface { + GobgpConfigs(namespace string) GobgpConfigInterface +} + +// GobgpConfigInterface has methods to work with gobgpConfig resources. +type GobgpConfigInterface interface { + Create(ctx context.Context, gobgpConfig *kubeovnv1.GobgpConfig, opts metav1.CreateOptions) (*kubeovnv1.GobgpConfig, error) + Update(ctx context.Context, gobgpConfig *kubeovnv1.GobgpConfig, opts metav1.UpdateOptions) (*kubeovnv1.GobgpConfig, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, gobgpConfig *kubeovnv1.GobgpConfig, opts metav1.UpdateOptions) (*kubeovnv1.GobgpConfig, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*kubeovnv1.GobgpConfig, error) + List(ctx context.Context, opts metav1.ListOptions) (*kubeovnv1.GobgpConfigList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *kubeovnv1.GobgpConfig, err error) + GetScale(ctx context.Context, gobgpConfigName string, options metav1.GetOptions) (*autoscalingv1.Scale, error) + UpdateScale(ctx context.Context, gobgpConfigName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (*autoscalingv1.Scale, error) + + GobgpConfigExpansion +} + +// gobgpConfigs implements GobgpConfigInterface +type gobgpConfigs struct { + *gentype.ClientWithList[*kubeovnv1.GobgpConfig, *kubeovnv1.GobgpConfigList] +} + +// newGobgpConfigs returns a GobgpConfigs +func newGobgpConfigs(c *KubeovnV1Client, namespace string) *gobgpConfigs { + return &gobgpConfigs{ + gentype.NewClientWithList[*kubeovnv1.GobgpConfig, *kubeovnv1.GobgpConfigList]( + "gobgp-configs", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *kubeovnv1.GobgpConfig { return &kubeovnv1.GobgpConfig{} }, + func() *kubeovnv1.GobgpConfigList { return &kubeovnv1.GobgpConfigList{} }, + ), + } +} + +// GetScale takes name of the GobgpConfig, and returns the corresponding autoscalingv1.Scale object, and an error if there is any. +func (c *gobgpConfigs) GetScale(ctx context.Context, gobgpConfigName string, options metav1.GetOptions) (result *autoscalingv1.Scale, err error) { + result = &autoscalingv1.Scale{} + err = c.GetClient().Get(). + Namespace(c.GetNamespace()). + Resource("gobgp-configs"). + Name(gobgpConfigName). + SubResource("scale"). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// UpdateScale takes the top resource name and the representation of a scale and updates it. Returns the server's representation of the scale, and an error, if there is any. +func (c *gobgpConfigs) UpdateScale(ctx context.Context, gobgpConfigName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (result *autoscalingv1.Scale, err error) { + result = &autoscalingv1.Scale{} + err = c.GetClient().Put(). + Namespace(c.GetNamespace()). + Resource("gobgp-configs"). + Name(gobgpConfigName). + SubResource("scale"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(scale). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go index 24dc5ea5a1f..de04c5416b9 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go @@ -49,6 +49,9 @@ type KubeovnV1Interface interface { VpcDnsesGetter VpcEgressGatewaysGetter VpcNatGatewaysGetter + BgpEdgeRoutersGetter + BgpEdgeRouterAdvertisementsGetter + GobgpConfigsGetter } // KubeovnV1Client is used to interact with features provided by the kubeovn.io group. @@ -140,6 +143,18 @@ func (c *KubeovnV1Client) VpcNatGateways() VpcNatGatewayInterface { return newVpcNatGateways(c) } +func (c *KubeovnV1Client) BgpEdgeRouters(namespace string) BgpEdgeRouterInterface { + return newBgpEdgeRouters(c, namespace) +} + +func (c *KubeovnV1Client) BgpEdgeRouterAdvertisements(namespace string) BgpEdgeRouterAdvertisementInterface { + return newBgpEdgeRouterAdvertisements(c, namespace) +} + +func (c *KubeovnV1Client) GobgpConfigs(namespace string) GobgpConfigInterface { + return newGobgpConfigs(c, namespace) +} + // NewForConfig creates a new KubeovnV1Client for the given config. // NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), // where httpClient was generated with rest.HTTPClientFor(c). diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index 01c67b1c70c..a6482a5d8c6 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -93,6 +93,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcDnses().Informer()}, nil case v1.SchemeGroupVersion.WithResource("vpc-egress-gateways"): return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcEgressGateways().Informer()}, nil + case v1.SchemeGroupVersion.WithResource("bgp-edge-routers"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().BgpEdgeRouters().Informer()}, nil case v1.SchemeGroupVersion.WithResource("vpc-nat-gateways"): return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcNatGateways().Informer()}, nil diff --git a/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouter.go b/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouter.go new file mode 100644 index 00000000000..3453626947a --- /dev/null +++ b/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouter.go @@ -0,0 +1,90 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + time "time" + + apiskubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + versioned "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions/internalinterfaces" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// BgpEdgeRouterInformer provides access to a shared informer and lister for +// BgpEdgeRouters. +type BgpEdgeRouterInformer interface { + Informer() cache.SharedIndexInformer + Lister() kubeovnv1.BgpEdgeRouterLister +} + +type bgpEdgeRouterInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewBgpEdgeRouterInformer constructs a new informer for BgpEdgeRouter type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewBgpEdgeRouterInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredBgpEdgeRouterInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredBgpEdgeRouterInformer constructs a new informer for BgpEdgeRouter type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredBgpEdgeRouterInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().BgpEdgeRouters(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().BgpEdgeRouters(namespace).Watch(context.TODO(), options) + }, + }, + &apiskubeovnv1.BgpEdgeRouter{}, + resyncPeriod, + indexers, + ) +} + +func (f *bgpEdgeRouterInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredBgpEdgeRouterInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *bgpEdgeRouterInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apiskubeovnv1.BgpEdgeRouter{}, f.defaultInformer) +} + +func (f *bgpEdgeRouterInformer) Lister() kubeovnv1.BgpEdgeRouterLister { + return kubeovnv1.NewBgpEdgeRouterLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouteradvertisement.go b/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouteradvertisement.go new file mode 100644 index 00000000000..bdfccf1813a --- /dev/null +++ b/pkg/client/informers/externalversions/kubeovn/v1/bgpedgerouteradvertisement.go @@ -0,0 +1,89 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + time "time" + + apiskubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + versioned "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions/internalinterfaces" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// BgpEdgeRouterAdvertisementInformer provides access to a shared informer and lister for +type BgpEdgeRouterAdvertisementInformer interface { + Informer() cache.SharedIndexInformer + Lister() kubeovnv1.BgpEdgeRouterAdvertisementLister +} + +type bgpEdgeRouterAdvertisementInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewBgpEdgeRouterAdvertisementInformer constructs a new informer for BgpEdgeRouter type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewBgpEdgeRouterAdvertisementInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredBgpEdgeRouterAdvertisementInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredBgpEdgeRouterInformer constructs a new informer for BgpEdgeRouter type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredBgpEdgeRouterAdvertisementInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().BgpEdgeRouterAdvertisements(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().BgpEdgeRouterAdvertisements(namespace).Watch(context.TODO(), options) + }, + }, + &apiskubeovnv1.BgpEdgeRouterAdvertisement{}, + resyncPeriod, + indexers, + ) +} + +func (f *bgpEdgeRouterAdvertisementInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredBgpEdgeRouterAdvertisementInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *bgpEdgeRouterAdvertisementInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apiskubeovnv1.BgpEdgeRouterAdvertisement{}, f.defaultInformer) +} + +func (f *bgpEdgeRouterAdvertisementInformer) Lister() kubeovnv1.BgpEdgeRouterAdvertisementLister { + return kubeovnv1.NewBgpEdgeRouterAdvertisementLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/kubeovn/v1/gobgpconfig.go b/pkg/client/informers/externalversions/kubeovn/v1/gobgpconfig.go new file mode 100644 index 00000000000..317ad6d1ffb --- /dev/null +++ b/pkg/client/informers/externalversions/kubeovn/v1/gobgpconfig.go @@ -0,0 +1,89 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + context "context" + time "time" + + apiskubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + versioned "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions/internalinterfaces" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// GobgpConfigInformer provides access to a shared informer and lister for +type GobgpConfigInformer interface { + Informer() cache.SharedIndexInformer + Lister() kubeovnv1.GobgpConfigLister +} + +type gobgpConfigInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewBgpEdgeRouterAdvertisementInformer constructs a new informer for BgpEdgeRouter type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewGobgpConfigInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredGobgpConfigInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredGobgpConfigInformer constructs a new informer for GobgpConfig type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredGobgpConfigInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().GobgpConfigs(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().GobgpConfigs(namespace).Watch(context.TODO(), options) + }, + }, + &apiskubeovnv1.GobgpConfig{}, + resyncPeriod, + indexers, + ) +} + +func (f *gobgpConfigInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredGobgpConfigInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *gobgpConfigInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apiskubeovnv1.GobgpConfig{}, f.defaultInformer) +} + +func (f *gobgpConfigInformer) Lister() kubeovnv1.GobgpConfigLister { + return kubeovnv1.NewGobgpConfigLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/kubeovn/v1/interface.go b/pkg/client/informers/externalversions/kubeovn/v1/interface.go index 69cad95925a..deda5a9333c 100644 --- a/pkg/client/informers/externalversions/kubeovn/v1/interface.go +++ b/pkg/client/informers/externalversions/kubeovn/v1/interface.go @@ -66,6 +66,12 @@ type Interface interface { VpcEgressGateways() VpcEgressGatewayInformer // VpcNatGateways returns a VpcNatGatewayInformer. VpcNatGateways() VpcNatGatewayInformer + // BgpEdgeRouters returns a BgpEdgeRouterInformer. + BgpEdgeRouters() BgpEdgeRouterInformer + // BgpEdgeRouterAdvertisements returns a BgpEdgeRouterAdvertisementInformer. + BgpEdgeRouterAdvertisements() BgpEdgeRouterAdvertisementInformer + // GobgpConfigs returns a GobgpConfigInformer. + GobgpConfigs() GobgpConfigInformer } type version struct { @@ -183,3 +189,18 @@ func (v *version) VpcEgressGateways() VpcEgressGatewayInformer { func (v *version) VpcNatGateways() VpcNatGatewayInformer { return &vpcNatGatewayInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} } + +// BgpEdgeRouters returns a BgpEdgeRouterInformer. +func (v *version) BgpEdgeRouters() BgpEdgeRouterInformer { + return &bgpEdgeRouterInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + +// BgpEdgeRouters returns a BgpEdgeRouterInformer. +func (v *version) BgpEdgeRouterAdvertisements() BgpEdgeRouterAdvertisementInformer { + return &bgpEdgeRouterAdvertisementInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + +// BgpEdgeRouters returns a BgpEdgeRouterInformer. +func (v *version) GobgpConfigs() GobgpConfigInformer { + return &gobgpConfigInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/client/listers/kubeovn/v1/bgpedgerouter.go b/pkg/client/listers/kubeovn/v1/bgpedgerouter.go new file mode 100644 index 00000000000..70b4b99d211 --- /dev/null +++ b/pkg/client/listers/kubeovn/v1/bgpedgerouter.go @@ -0,0 +1,82 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// type BgpEdgeRouterInformer interface { +// Informer() cache.SharedIndexInformer +// Lister() kubeovnv1.BgpEdgeRouterLister +// } + +// type bgpEdgeRouterInformer struct { +// factory internalinterfaces.SharedInformerFactory +// tweakListOptions internalinterfaces.TweakListOptionsFunc +// namespace string +// } + +// BgpEdgeRouterLister helps list BgpEdgeRouters. +// All objects returned here must be treated as read-only. +type BgpEdgeRouterLister interface { + // List lists all BgpEdgeRouters in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*kubeovnv1.BgpEdgeRouter, err error) + // Get(name string) (*kubeovnv1.BgpEdgeRouter, error) + // BgpEdgeRouters returns an object that can list and get BgpEdgeRouters. + BgpEdgeRouters(namespace string) BgpEdgeRouterNamespaceLister + BgpEdgeRouterListerExpansion +} + +// bgpEdgeRouterLister implements the bgpEdgeRouterLister interface. +type bgpEdgeRouterLister struct { + listers.ResourceIndexer[*kubeovnv1.BgpEdgeRouter] +} + +// NewBgpEdgeRouterLister returns a new bgpEdgeRouterLister. +func NewBgpEdgeRouterLister(indexer cache.Indexer) BgpEdgeRouterLister { + return &bgpEdgeRouterLister{listers.New[*kubeovnv1.BgpEdgeRouter](indexer, kubeovnv1.Resource("bgpedgerouter"))} +} + +// bgpEdgeRouters returns an object that can list and get bgpEdgeRouters. +func (s *bgpEdgeRouterLister) BgpEdgeRouters(namespace string) BgpEdgeRouterNamespaceLister { + return bgpEdgeRouterNamespaceLister{listers.NewNamespaced[*kubeovnv1.BgpEdgeRouter](s.ResourceIndexer, namespace)} +} + +// bgpEdgeRouterNamespaceLister helps list and get bgpEdgeRouters. +// All objects returned here must be treated as read-only. +type BgpEdgeRouterNamespaceLister interface { + // List lists all bgpEdgeRouters in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*kubeovnv1.BgpEdgeRouter, err error) + // Get retrieves the bgpEdgeRouter from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*kubeovnv1.BgpEdgeRouter, error) + BgpEdgeRouterNamespaceListerExpansion +} + +// bgpEdgeRouterNamespaceLister implements the bgpEdgeRouterNamespaceLister +// interface. +type bgpEdgeRouterNamespaceLister struct { + listers.ResourceIndexer[*kubeovnv1.BgpEdgeRouter] +} diff --git a/pkg/client/listers/kubeovn/v1/bgpedgerouteradvertisement.go b/pkg/client/listers/kubeovn/v1/bgpedgerouteradvertisement.go new file mode 100644 index 00000000000..11e95058e84 --- /dev/null +++ b/pkg/client/listers/kubeovn/v1/bgpedgerouteradvertisement.go @@ -0,0 +1,70 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// BgpEdgeRouterAdvertisementLister helps list BgpEdgeRouterAdvertisements. +// All objects returned here must be treated as read-only. +type BgpEdgeRouterAdvertisementLister interface { + // List lists all BgpEdgeRouterAdvertisements in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*kubeovnv1.BgpEdgeRouterAdvertisement, err error) + // BgpEdgeRouterAdvertisements returns an object that can list and get BgpEdgeRouterAdvertisements. + BgpEdgeRouterAdvertisements(namespace string) BgpEdgeRouterAdvertisementNamespaceLister + BgpEdgeRouterAdvertisementListerExpansion +} + +// bgpEdgeRouterLister implements the bgpEdgeRouterLister interface. +type bgpEdgeRouterAdvertisementLister struct { + listers.ResourceIndexer[*kubeovnv1.BgpEdgeRouterAdvertisement] +} + +// NewBgpEdgeRouterAdvertisementLister returns a new bgpEdgeRouterAdvertisementLister. +func NewBgpEdgeRouterAdvertisementLister(indexer cache.Indexer) BgpEdgeRouterAdvertisementLister { + return &bgpEdgeRouterAdvertisementLister{listers.New[*kubeovnv1.BgpEdgeRouterAdvertisement](indexer, kubeovnv1.Resource("bgpedgerouteradvertisement"))} +} + +// bgpEdgeRouterAdvertisements returns an object that can list and get bgpEdgeRouterAdvertisements. +func (s *bgpEdgeRouterAdvertisementLister) BgpEdgeRouterAdvertisements(namespace string) BgpEdgeRouterAdvertisementNamespaceLister { + return bgpEdgeRouterAdvertisementNamespaceLister{listers.NewNamespaced[*kubeovnv1.BgpEdgeRouterAdvertisement](s.ResourceIndexer, namespace)} +} + +// bgpEdgeRouterAdvertisementNamespaceLister helps list and get bgpEdgeRouterAdvertisements. +// All objects returned here must be treated as read-only. +type BgpEdgeRouterAdvertisementNamespaceLister interface { + // List lists all bgpEdgeRouterAdvertisements in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*kubeovnv1.BgpEdgeRouterAdvertisement, err error) + // Get retrieves the bgpEdgeRouter from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) + BgpEdgeRouterAdvertisementNamespaceListerExpansion +} + +// bgpEdgeRouterAdvertisementNamespaceLister implements the bgpEdgeRouterAdvertisementNamespaceLister +// interface. +type bgpEdgeRouterAdvertisementNamespaceLister struct { + listers.ResourceIndexer[*kubeovnv1.BgpEdgeRouterAdvertisement] +} diff --git a/pkg/client/listers/kubeovn/v1/expansion_generated.go b/pkg/client/listers/kubeovn/v1/expansion_generated.go index b474d11b77d..6af94fabcb7 100644 --- a/pkg/client/listers/kubeovn/v1/expansion_generated.go +++ b/pkg/client/listers/kubeovn/v1/expansion_generated.go @@ -105,3 +105,27 @@ type VpcEgressGatewayNamespaceListerExpansion interface{} // VpcNatGatewayListerExpansion allows custom methods to be added to // VpcNatGatewayLister. type VpcNatGatewayListerExpansion interface{} + +// BgpEdgeRouterListerExpansion allows custom methods to be added to +// BgpEdgeRouterLister. +type BgpEdgeRouterListerExpansion interface{} + +// BgpEdgeRouterNamespaceListerExpansion allows custom methods to be added to +// BgpEdgeRouterNamespaceLister. +type BgpEdgeRouterNamespaceListerExpansion interface{} + +// BgpEdgeRouterListerExpansion allows custom methods to be added to +// BgpEdgeRouterLister. +type BgpEdgeRouterAdvertisementListerExpansion interface{} + +// BgpEdgeRouterNamespaceListerExpansion allows custom methods to be added to +// BgpEdgeRouterNamespaceLister. +type BgpEdgeRouterAdvertisementNamespaceListerExpansion interface{} + +// BgpEdgeRouterListerExpansion allows custom methods to be added to +// BgpEdgeRouterLister. +type GobgpConfigListerExpansion interface{} + +// BgpEdgeRouterNamespaceListerExpansion allows custom methods to be added to +// BgpEdgeRouterNamespaceLister. +type GobgpConfigNamespaceListerExpansion interface{} diff --git a/pkg/client/listers/kubeovn/v1/gobgpconfig.go b/pkg/client/listers/kubeovn/v1/gobgpconfig.go new file mode 100644 index 00000000000..fbf3974c660 --- /dev/null +++ b/pkg/client/listers/kubeovn/v1/gobgpconfig.go @@ -0,0 +1,70 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// GobgpConfigLister helps list GobgpConfigs. +// All objects returned here must be treated as read-only. +type GobgpConfigLister interface { + // List lists all GobgpConfigs in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*kubeovnv1.GobgpConfig, err error) + // GobgpConfigs returns an object that can list and get GobgpConfigs. + GobgpConfigs(namespace string) GobgpConfigNamespaceLister + GobgpConfigListerExpansion +} + +// gobgpConfigLister implements the gobgpConfigLister interface. +type gobgpConfigLister struct { + listers.ResourceIndexer[*kubeovnv1.GobgpConfig] +} + +// NewGobgpConfigLister returns a new gobgpConfigLister. +func NewGobgpConfigLister(indexer cache.Indexer) GobgpConfigLister { + return &gobgpConfigLister{listers.New[*kubeovnv1.GobgpConfig](indexer, kubeovnv1.Resource("gobgpconfig"))} +} + +// gobgpConfigs returns an object that can list and get gobgpConfigs. +func (s *gobgpConfigLister) GobgpConfigs(namespace string) GobgpConfigNamespaceLister { + return gobgpConfigNamespaceLister{listers.NewNamespaced[*kubeovnv1.GobgpConfig](s.ResourceIndexer, namespace)} +} + +// gobgpConfigNamespaceLister helps list and get gobgpConfigs. +// All objects returned here must be treated as read-only. +type GobgpConfigNamespaceLister interface { + // List lists all gobgpConfigs in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*kubeovnv1.GobgpConfig, err error) + // Get retrieves the GobgpConfig from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*kubeovnv1.GobgpConfig, error) + GobgpConfigNamespaceListerExpansion +} + +// gobgpConfigNamespaceLister implements the gobgpConfigNamespaceLister +// interface. +type gobgpConfigNamespaceLister struct { + listers.ResourceIndexer[*kubeovnv1.GobgpConfig] +} diff --git a/pkg/controller/bgp_edge_router.go b/pkg/controller/bgp_edge_router.go new file mode 100644 index 00000000000..4df0e85edfc --- /dev/null +++ b/pkg/controller/bgp_edge_router.go @@ -0,0 +1,1111 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "maps" + "reflect" + "slices" + "strconv" + "strings" + + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + "k8s.io/utils/set" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +func (c *Controller) enqueueAddBgpEdgeRouter(obj any) { + key := cache.MetaObjectToName(obj.(*kubeovnv1.BgpEdgeRouter)).String() + klog.V(3).Infof("enqueue add bgp-edge-router %s", key) + c.addOrUpdateBgpEdgeRouterQueue.Add(key) +} + +func (c *Controller) enqueueUpdateBgpEdgeRouter(_, newObj any) { + key := cache.MetaObjectToName(newObj.(*kubeovnv1.BgpEdgeRouter)).String() + klog.V(3).Infof("enqueue update bgp-edge-router %s", key) + c.addOrUpdateBgpEdgeRouterQueue.Add(key) +} + +func (c *Controller) enqueueDeleteBgpEdgeRouter(obj any) { + key := cache.MetaObjectToName(obj.(*kubeovnv1.BgpEdgeRouter)).String() + klog.V(3).Infof("enqueue delete bgp-edge-router %s", key) + c.delBgpEdgeRouterQueue.Add(key) +} + +func bgpEdgeRouterWorkloadLabels(bgpEdgeRouterName string) map[string]string { + return map[string]string{"app": "bgp-edge-router", util.BgpEdgeRouterLabel: bgpEdgeRouterName} +} + +func (c *Controller) handleAddOrUpdateBgpEdgeRouter(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.bgpEdgeRouterKeyMutex.LockKey(key) + defer func() { _ = c.bgpEdgeRouterKeyMutex.UnlockKey(key) }() + + cachedRouter, err := c.bgpEdgeRouterLister.BgpEdgeRouters(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + if !cachedRouter.DeletionTimestamp.IsZero() { + c.delBgpEdgeRouterQueue.Add(key) + return nil + } + + klog.Infof("reconciling bgp-edge-router %s", key) + router := cachedRouter.DeepCopy() + if router, err = c.initBgpEdgeRouterStatus(router); err != nil { + return err + } + + vpcName := router.Spec.VPC + if vpcName == "" { + vpcName = c.config.ClusterRouter + } + vpc, err := c.vpcsLister.Get(vpcName) + if err != nil { + klog.Error(err) + return err + } + if router.Spec.BFD.Enabled && vpc.Status.BFDPort.IP == "" { + err = fmt.Errorf("vpc %s bfd port is not enabled or not ready", vpc.Name) + klog.Error(err) + router.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "VpcBfdPortNotEnabled", err.Error(), router.Generation) + _, _ = c.updatebgpEdgeRouterStatus(router) + return err + } + + if controllerutil.AddFinalizer(router, util.KubeOVNControllerFinalizer) { + updatedGateway, err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouters(router.Namespace). + Update(context.Background(), router, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to add finalizer for bgp-edge-router %s/%s: %w", router.Namespace, router.Name, err) + klog.Error(err) + return err + } + router = updatedGateway + } + + var bfdIP, bfdIPv4, bfdIPv6 string + if router.Spec.BFD.Enabled { + bfdIP = vpc.Status.BFDPort.IP + bfdIPv4, bfdIPv6 = util.SplitStringIP(bfdIP) + } + + // reconcile the bgp edge router workload and get the route sources for later OVN resources reconciliation + attachmentNetworkName, ipv4Src, ipv6Src, deploy, err := c.reconcileBgpEdgeRouterWorkload(router, vpc, bfdIP, bfdIPv4, bfdIPv6) + router.Status.Replicas = router.Spec.Replicas + router.Status.LabelSelector = labels.FormatLabels(bgpEdgeRouterWorkloadLabels(router.Name)) + if err != nil { + klog.Error(err) + router.Status.Replicas = 0 + router.Status.Conditions.SetCondition(kubeovnv1.Ready, corev1.ConditionFalse, "ReconcileWorkloadFailed", err.Error(), router.Generation) + _, _ = c.updatebgpEdgeRouterStatus(router) + return err + } + + router.Status.InternalIPs = nil + router.Status.ExternalIPs = nil + router.Status.Workload.APIVersion = deploy.APIVersion + router.Status.Workload.Kind = deploy.Kind + router.Status.Workload.Name = deploy.Name + router.Status.Workload.Nodes = nil + nodeNexthopIPv4 := make(map[string]string, int(router.Spec.Replicas)) + nodeNexthopIPv6 := make(map[string]string, int(router.Spec.Replicas)) + ready := util.DeploymentIsReady(deploy) + if !ready { + router.Status.Ready = false + msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) + router.Status.Conditions.SetCondition(kubeovnv1.Ready, corev1.ConditionFalse, "Processing", msg, router.Generation) + } + // get the pods of the deployment to collect the pod IPs + podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) + if err != nil { + err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return err + } + + pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) + if err != nil { + err = fmt.Errorf("failed to list pods of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return err + } + + // update router status including the internal/external IPs and the nodes where the pods are running + router.Status.Workload.Nodes = make([]string, 0, len(pods)) + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + extIPs, err := util.PodAttachmentIPs(pod, attachmentNetworkName) + if err != nil { + klog.Error(err) + continue + } + + ips := util.PodIPs(*pod) + ipv4, ipv6 := util.SplitIpsByProtocol(ips) + if len(ipv4) != 0 { + nodeNexthopIPv4[pod.Spec.NodeName] = ipv4[0] + } + if len(ipv6) != 0 { + nodeNexthopIPv6[pod.Spec.NodeName] = ipv6[0] + } + router.Status.InternalIPs = append(router.Status.InternalIPs, strings.Join(ips, ",")) + router.Status.ExternalIPs = append(router.Status.ExternalIPs, strings.Join(extIPs, ",")) + router.Status.Workload.Nodes = append(router.Status.Workload.Nodes, pod.Spec.NodeName) + } + if router, err = c.updatebgpEdgeRouterStatus(router); err != nil { + klog.Error(err) + return err + } + + // reconcile OVN routes + if err = c.reconcileBgpEdgeRouterOVNRoutes(router, 4, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv4, nodeNexthopIPv4, ipv4Src); err != nil { + klog.Error(err) + return err + } + if err = c.reconcileBgpEdgeRouterOVNRoutes(router, 6, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv6, nodeNexthopIPv6, ipv6Src); err != nil { + klog.Error(err) + return err + } + + if ready { + router.Status.Ready = true + router.Status.Phase = kubeovnv1.PhaseCompleted + router.Status.Conditions.SetReady("ReconcileSuccess", router.Generation) + if _, err = c.updatebgpEdgeRouterStatus(router); err != nil { + return err + } + } + + klog.Infof("finished reconciling bgp-edge-router %s", key) + + return nil +} + +func (c *Controller) initBgpEdgeRouterStatus(router *kubeovnv1.BgpEdgeRouter) (*kubeovnv1.BgpEdgeRouter, error) { + var err error + if router.Status.Phase == "" || router.Status.Phase == kubeovnv1.PhasePending { + router.Status.Phase = kubeovnv1.PhaseProcessing + router, err = c.updatebgpEdgeRouterStatus(router) + } + return router, err +} + +func (c *Controller) updatebgpEdgeRouterStatus(router *kubeovnv1.BgpEdgeRouter) (*kubeovnv1.BgpEdgeRouter, error) { + if len(router.Status.Conditions) == 0 { + router.Status.Conditions.SetCondition(kubeovnv1.Init, corev1.ConditionUnknown, "Processing", "", router.Generation) + } + if !router.Status.Ready { + router.Status.Phase = kubeovnv1.PhaseProcessing + } + + updateRouter, err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouters(router.Namespace). + UpdateStatus(context.Background(), router, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to update status of bgp-edge-router %s/%s: %w", router.Namespace, router.Name, err) + klog.Error(err) + return nil, err + } + + return updateRouter, nil +} + +// create or update bgp edge router workload +func (c *Controller) reconcileBgpEdgeRouterWorkload(router *kubeovnv1.BgpEdgeRouter, vpc *kubeovnv1.Vpc, bfdIP, bfdIPv4, bfdIPv6 string) (string, set.Set[string], set.Set[string], *appsv1.Deployment, error) { + image := c.config.Image + bgpImage := c.config.Image + if router.Spec.Image != "" { + image = router.Spec.Image + } + if router.Spec.BGP.Image != "" { + bgpImage = router.Spec.BGP.Image + } + if image == "" { + err := fmt.Errorf("no image specified for bgp edge router %s/%s", router.Namespace, router.Name) + klog.Error(err) + return "", nil, nil, nil, err + } + + if len(router.Spec.InternalIPs) != 0 && len(router.Spec.InternalIPs) < int(router.Spec.Replicas) { + err := fmt.Errorf("internal IPs count %d is less than replicas %d", len(router.Spec.InternalIPs), router.Spec.Replicas) + klog.Error(err) + return "", nil, nil, nil, err + } + if len(router.Spec.ExternalIPs) != 0 && len(router.Spec.ExternalIPs) < int(router.Spec.Replicas) { + err := fmt.Errorf("external IPs count %d is less than replicas %d", len(router.Spec.ExternalIPs), router.Spec.Replicas) + klog.Error(err) + return "", nil, nil, nil, err + } + + internalSubnet := router.Spec.InternalSubnet + if internalSubnet == "" { + internalSubnet = vpc.Status.DefaultLogicalSwitch + } + if internalSubnet == "" { + err := fmt.Errorf("default subnet of vpc %s not found, please set internal subnet of the bgp edge router", vpc.Name) + klog.Error(err) + return "", nil, nil, nil, err + } + intSubnet, err := c.subnetsLister.Get(internalSubnet) + if err != nil { + klog.Error(err) + return "", nil, nil, nil, err + } + extSubnet, err := c.subnetsLister.Get(router.Spec.ExternalSubnet) + if err != nil { + klog.Error(err) + return "", nil, nil, nil, err + } + if !strings.ContainsRune(extSubnet.Spec.Provider, '.') { + err = fmt.Errorf("please set correct provider of subnet %s to get the network-attachment-definition", extSubnet.Name) + klog.Error(err) + return "", nil, nil, nil, err + } + subStrings := strings.Split(extSubnet.Spec.Provider, ".") + nadName, nadNamespace := subStrings[0], subStrings[1] + if _, err = c.netAttachLister.NetworkAttachmentDefinitions(nadNamespace).Get(nadName); err != nil { + klog.Errorf("failed to get net-attach-def %s/%s: %v", nadNamespace, nadName, err) + return "", nil, nil, nil, err + } + attachmentNetworkName := fmt.Sprintf("%s/%s", nadNamespace, nadName) + + // collect egress policies + ipv4ForwardSrc, ipv6ForwardSrc := set.New[string](), set.New[string]() + ipv4SNATSrc, ipv6SNATSrc := set.New[string](), set.New[string]() + for _, policy := range router.Spec.Policies { + ipv4, ipv6 := util.SplitIpsByProtocol(policy.IPBlocks) + if policy.SNAT { + ipv4SNATSrc.Insert(ipv4...) + ipv6SNATSrc.Insert(ipv6...) + } else { + ipv4ForwardSrc.Insert(ipv4...) + ipv6ForwardSrc.Insert(ipv6...) + } + for _, subnetName := range policy.Subnets { + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + if subnet.Status.IsNotValidated() { + err = fmt.Errorf("subnet %s is not validated", subnet.Name) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + // TODO: check subnet's vpc and vlan + ipv4, ipv6 := util.SplitStringIP(subnet.Spec.CIDRBlock) + if policy.SNAT { + ipv4SNATSrc.Insert(ipv4) + ipv6SNATSrc.Insert(ipv6) + } else { + ipv4ForwardSrc.Insert(ipv4) + ipv6ForwardSrc.Insert(ipv6) + } + } + } + + // calculate internal route destinations and forward source CIDR blocks + intRouteDstIPv4, intRouteDstIPv6 := ipv4ForwardSrc.Union(ipv4SNATSrc), ipv6ForwardSrc.Union(ipv6SNATSrc) + intRouteDstIPv4.Delete("") + intRouteDstIPv6.Delete("") + ipv4ForwardSrc.Delete("") + ipv6ForwardSrc.Delete("") + + // generate route annotations used to configure routes in the pod + routes := util.NewPodRoutes() + intGatewayIPv4, intGatewayIPv6 := util.SplitStringIP(intSubnet.Spec.Gateway) + extGatewayIPv4, extGatewayIPv6 := util.SplitStringIP(extSubnet.Spec.Gateway) + // add routes for the VPC BFD Port so that the bgp edge router can establish BFD session(s) with it + routes.Add(util.OvnProvider, bfdIPv4, intGatewayIPv4) + routes.Add(util.OvnProvider, bfdIPv6, intGatewayIPv6) + // add routes for the internal networks + for _, dst := range intRouteDstIPv4.UnsortedList() { + // skip the route to the internal subnet itself + if intSubnet.Spec.CIDRBlock == dst { + continue + } + routes.Add(util.OvnProvider, dst, intGatewayIPv4) + } + for _, dst := range intRouteDstIPv6.UnsortedList() { + routes.Add(util.OvnProvider, dst, intGatewayIPv6) + } + // add default routes to forward traffic to the external network + routes.Add(extSubnet.Spec.Provider, "0.0.0.0/0", extGatewayIPv4) + routes.Add(extSubnet.Spec.Provider, "::/0", extGatewayIPv6) + + // generate pod annotations + annotations, err := routes.ToAnnotations() + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + annotations[nadv1.NetworkAttachmentAnnot] = attachmentNetworkName + annotations[util.LogicalSwitchAnnotation] = intSubnet.Name + if len(router.Spec.InternalIPs) != 0 { + // set internal IPs + annotations[util.IPPoolAnnotation] = strings.Join(router.Spec.InternalIPs, ";") + } + if len(router.Spec.ExternalIPs) != 0 { + // set external IPs + annotations[fmt.Sprintf(util.IPPoolAnnotationTemplate, extSubnet.Spec.Provider)] = strings.Join(router.Spec.ExternalIPs, ";") + } + + // generate init container environment variables + // the init container is responsible for adding routes and SNAT rules to the pod network namespace + initEnv, err := bgpEdgeRouterInitContainerEnv(4, intGatewayIPv4, extGatewayIPv4, ipv4ForwardSrc) + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + ipv6Env, err := bgpEdgeRouterInitContainerEnv(6, intGatewayIPv6, extGatewayIPv6, ipv6ForwardSrc) + if err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + initEnv = append(initEnv, ipv6Env...) + + // generate workload + labels := bgpEdgeRouterWorkloadLabels(router.Name) + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: router.Spec.Prefix + router.Name, + Namespace: router.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: ptr.To(intstr.FromInt(1)), + MaxSurge: ptr.To(intstr.FromInt(0)), + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: berMergeNodeSelector(router.Spec.NodeSelector), + }, + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: corev1.LabelHostname, + }}, + }, + }, + InitContainers: []corev1.Container{{ + Name: "init", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"bash", "/kube-ovn/init-vpc-egress-gateway.sh"}, + Env: initEnv, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "usr-local-sbin", + MountPath: "/usr/local/sbin", + }}, + }}, + Containers: []corev1.Container{{ + Name: "gateway", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sleep", "infinity"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](65534), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "usr-local-sbin", + MountPath: "/usr/local/sbin", + }}, + }}, + Volumes: []corev1.Volume{ + { + Name: "usr-local-sbin", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "kube-ovn-logs", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + TerminationGracePeriodSeconds: ptr.To[int64](0), + }, + }, + }, + } + // set owner reference so that the workload will be deleted automatically when the bgp edge router is deleted + if err = util.SetOwnerReference(router, deploy); err != nil { + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + + if bfdIP != "" { + // run BFD in the router container to establish BFD session(s) with the VPC BFD LRP + container := bgpEdgeRouterContainerBFDD(image, bfdIP, router.Spec.BFD.MinTX, router.Spec.BFD.MinRX, router.Spec.BFD.Multiplier) + deploy.Spec.Template.Spec.Containers[0] = container + } + + // bgp sidecar container logic + if router.Spec.BGP.Enabled { + // run BGP in the router container + bgpContainer, err := bgpEdgeRouterContainerBGP(bgpImage, router.Name, &router.Spec.BGP) + if err != nil { + klog.Errorf("failed to create a BGP speaker container for router %s: %v", router.Name, err) + return "", nil, nil, nil, err + } + deploy.Spec.Template.Spec.Containers = append(deploy.Spec.Template.Spec.Containers, *bgpContainer) + } + + // generate hash for the workload to determine whether to update the existing workload or not + hash, err := util.Sha256HashObject(deploy) + if err != nil { + err = fmt.Errorf("failed to hash generated deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + + hash = hash[:12] + // replicas and the hash annotation should be excluded from hash calculation + deploy.Spec.Replicas = ptr.To(router.Spec.Replicas) + deploy.Annotations = map[string]string{util.GenerateHashAnnotation: hash} + + if currentDeploy, err := c.berDeploymentsLister.Deployments(router.Namespace).Get(deploy.Name); err != nil { + if !k8serrors.IsNotFound(err) { + err = fmt.Errorf("failed to get deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + if deploy, err = c.config.KubeClient.AppsV1().Deployments(router.Namespace). + Create(context.Background(), deploy, metav1.CreateOptions{}); err != nil { + err = fmt.Errorf("failed to create deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + } else if !reflect.DeepEqual(currentDeploy.Spec.Replicas, deploy.Spec.Replicas) || + currentDeploy.Annotations[util.GenerateHashAnnotation] != hash { + // update the deployment if replicas or hash annotation is changed + if deploy, err = c.config.KubeClient.AppsV1().Deployments(router.Namespace). + Update(context.Background(), deploy, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to update deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return attachmentNetworkName, nil, nil, nil, err + } + } else { + // no need to create or update the deployment + deploy = currentDeploy + } + + // return the source CIDR blocks for later OVN resources reconciliation + deploy.APIVersion, deploy.Kind = deploymentGroupVersion, deploymentKind + return attachmentNetworkName, intRouteDstIPv4, intRouteDstIPv6, deploy, nil +} + +func (c *Controller) reconcileBgpEdgeRouterOVNRoutes(router *kubeovnv1.BgpEdgeRouter, af int, lrName, lrpName, bfdIP string, nextHops map[string]string, sources set.Set[string]) error { + if len(nextHops) == 0 { + return nil + } + + externalIDs := map[string]string{ + ovs.ExternalIDVendor: util.CniTypeName, + ovs.ExternalIDBgpEdgeRouter: fmt.Sprintf("%s/%s", router.Namespace, router.Name), + "af": strconv.Itoa(af), + } + bfdList, err := c.OVNNbClient.FindBFD(externalIDs) + if err != nil { + klog.Error(err) + return err + } + + // reconcile OVN port group + ports := set.New[string]() + key := cache.MetaObjectToName(router).String() + pgName := berPortGroupName(key) + if err = c.OVNNbClient.CreatePortGroup(pgName, externalIDs); err != nil { + err = fmt.Errorf("failed to create port group %s: %w", pgName, err) + klog.Error(err) + return err + } + if err = c.OVNNbClient.PortGroupSetPorts(pgName, ports.UnsortedList()); err != nil { + err = fmt.Errorf("failed to set ports of port group %s: %w", pgName, err) + klog.Error(err) + return err + } + + // reconcile OVN address set + asName := berAddressSetName(key, af) + if err = c.OVNNbClient.CreateAddressSet(asName, externalIDs); err != nil { + err = fmt.Errorf("failed to create address set %s: %w", asName, err) + klog.Error(err) + return err + } + if err = c.OVNNbClient.AddressSetUpdateAddress(asName, sources.SortedList()...); err != nil { + err = fmt.Errorf("failed to update address set %s: %w", asName, err) + klog.Error(err) + return err + } + + // reconcile OVN BFD entries + bfdIDs := set.New[string]() + staleBFDIDs := set.New[string]() + bfdDstIPs := set.New(slices.Collect(maps.Values(nextHops))...) + bfdMap := make(map[string]string, bfdDstIPs.Len()) + for _, bfd := range bfdList { + if bfdIP == "" || bfd.LogicalPort != lrpName || !bfdDstIPs.Has(bfd.DstIP) { + staleBFDIDs.Insert(bfd.UUID) + } + if bfdIP == "" || (bfd.LogicalPort == lrpName && bfdDstIPs.Has(bfd.DstIP)) { + // TODO: update min_rx, min_tx and multiplier + if bfdIP != "" { + bfdIDs.Insert(bfd.UUID) + bfdMap[bfd.DstIP] = bfd.UUID + } + bfdDstIPs.Delete(bfd.DstIP) + } + } + if bfdIP != "" { + for _, dstIP := range bfdDstIPs.UnsortedList() { + bfd, err := c.OVNNbClient.CreateBFD(lrpName, dstIP, int(router.Spec.BFD.MinRX), int(router.Spec.BFD.MinTX), int(router.Spec.BFD.Multiplier), externalIDs) + if err != nil { + klog.Error(err) + return err + } + bfdIDs.Insert(bfd.UUID) + bfdMap[bfd.DstIP] = bfd.UUID + } + } + + // reconcile LR policy + if router.Spec.TrafficPolicy == kubeovnv1.TrafficPolicyLocal { + rules := make(map[string]string, len(nextHops)) + for nodeName, nexthop := range nextHops { + node, err := c.nodesLister.Get(nodeName) + if err != nil { + if k8serrors.IsNotFound(err) { + continue + } + klog.Errorf("failed to get node %s: %v", nodeName, err) + return err + } + portName := node.Annotations[util.PortNameAnnotation] + if portName == "" { + err = fmt.Errorf("node %s does not have port name annotation", nodeName) + klog.Error(err) + return err + } + localPgName := strings.ReplaceAll(portName, "-", ".") + rules[fmt.Sprintf("ip%d.src == $%s_ip%d && ip%d.src == $%s_ip%d", af, localPgName, af, af, pgName, af)] = nexthop + rules[fmt.Sprintf("ip%d.src == $%s_ip%d && ip%d.src == $%s", af, localPgName, af, af, asName)] = nexthop + } + policies, err := c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayLocalPolicyPriority, externalIDs, false) + if err != nil { + klog.Error(err) + return err + } + // update/delete existing policies + for _, policy := range policies { + nexthop := rules[policy.Match] + bfdSessions := set.New(bfdMap[nexthop]).Delete("") + if nexthop == "" { + if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { + err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) + klog.Error(err) + return err + } + } else { + var changed bool + if len(policy.Nexthops) != 1 || policy.Nexthops[0] != nexthop { + policy.Nexthops = []string{nexthop} + changed = true + } + if !bfdSessions.Equal(set.New(policy.BFDSessions...)) { + policy.BFDSessions = bfdSessions.UnsortedList() + changed = true + } + if changed { + if err = c.OVNNbClient.UpdateLogicalRouterPolicy(policy, &policy.Nexthops, &policy.BFDSessions); err != nil { + err = fmt.Errorf("failed to update logical router policy %s: %w", policy.UUID, err) + klog.Error(err) + return err + } + } + } + delete(rules, policy.Match) + } + // create new policies + for match, nexthop := range rules { + if err = c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayLocalPolicyPriority, match, + ovnnb.LogicalRouterPolicyActionReroute, []string{nexthop}, []string{bfdMap[nexthop]}, externalIDs); err != nil { + klog.Error(err) + return err + } + } + } else { + if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, util.EgressGatewayLocalPolicyPriority, externalIDs); err != nil { + klog.Error(err) + return err + } + } + policies, err := c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayPolicyPriority, externalIDs, false) + if err != nil { + klog.Error(err) + return err + } + matches := set.New( + fmt.Sprintf("ip%d.src == $%s_ip%d", af, pgName, af), + fmt.Sprintf("ip%d.src == $%s", af, asName), + ) + bfdIPs := set.New(slices.Collect(maps.Values(nextHops))...) + bfdSessions := bfdIDs.UnsortedList() + for _, policy := range policies { + if matches.Has(policy.Match) { + if !bfdIPs.Equal(set.New(policy.Nexthops...)) || !bfdIDs.Equal(set.New(policy.BFDSessions...)) { + policy.Nexthops, policy.BFDSessions = bfdIPs.UnsortedList(), bfdSessions + if err = c.OVNNbClient.UpdateLogicalRouterPolicy(policy, &policy.Nexthops, &policy.BFDSessions); err != nil { + err = fmt.Errorf("failed to update bfd sessions of logical router policy %s: %w", policy.UUID, err) + klog.Error(err) + return err + } + } + matches.Delete(policy.Match) + continue + } + if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { + err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) + klog.Error(err) + return err + } + } + for _, match := range matches.UnsortedList() { + if err = c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayPolicyPriority, match, + ovnnb.LogicalRouterPolicyActionReroute, bfdIPs.UnsortedList(), bfdSessions, externalIDs); err != nil { + klog.Error(err) + return err + } + } + + if router.Spec.BFD.Enabled { + // drop traffic if no nexthop is available + if policies, err = c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayDropPolicyPriority, externalIDs, false); err != nil { + klog.Error(err) + return err + } + matches = set.New( + fmt.Sprintf("ip%d.src == $%s_ip%d", af, pgName, af), + fmt.Sprintf("ip%d.src == $%s", af, asName), + ) + for _, policy := range policies { + if matches.Has(policy.Match) { + matches.Delete(policy.Match) + continue + } + if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { + err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) + klog.Error(err) + return err + } + } + for _, match := range matches.UnsortedList() { + if err = c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayDropPolicyPriority, match, + ovnnb.LogicalRouterPolicyActionDrop, nil, nil, externalIDs); err != nil { + klog.Error(err) + return err + } + } + } else if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, util.EgressGatewayDropPolicyPriority, externalIDs); err != nil { + klog.Error(err) + return err + } + + for _, bfdID := range staleBFDIDs.UnsortedList() { + if err = c.OVNNbClient.DeleteBFD(bfdID); err != nil { + err = fmt.Errorf("failed to delete bfd %s: %w", bfdID, err) + klog.Error(err) + return err + } + } + + return nil +} + +func berMergeNodeSelector(nodeSelector []kubeovnv1.BgpEdgeRouterNodeSelector) *corev1.NodeSelector { + if len(nodeSelector) == 0 { + return nil + } + + result := &corev1.NodeSelector{ + NodeSelectorTerms: make([]corev1.NodeSelectorTerm, len(nodeSelector)), + } + for i, selector := range nodeSelector { + result.NodeSelectorTerms[i] = corev1.NodeSelectorTerm{ + MatchExpressions: make([]corev1.NodeSelectorRequirement, len(selector.MatchExpressions), len(selector.MatchLabels)+len(selector.MatchExpressions)), + MatchFields: make([]corev1.NodeSelectorRequirement, len(selector.MatchFields)), + } + for j := range selector.MatchExpressions { + selector.MatchExpressions[j].DeepCopyInto(&result.NodeSelectorTerms[i].MatchExpressions[j]) + } + for _, key := range slices.Sorted(maps.Keys(selector.MatchLabels)) { + result.NodeSelectorTerms[i].MatchExpressions = append(result.NodeSelectorTerms[i].MatchExpressions, corev1.NodeSelectorRequirement{ + Key: key, + Operator: corev1.NodeSelectorOpIn, + Values: []string{selector.MatchLabels[key]}, + }) + } + for j := range selector.MatchFields { + selector.MatchFields[j].DeepCopyInto(&result.NodeSelectorTerms[i].MatchFields[j]) + } + } + + return result +} + +func bgpEdgeRouterInitContainerEnv(af int, internalGateway, externalGateway string, forwardSrc set.Set[string]) ([]corev1.EnvVar, error) { + if internalGateway == "" { + return nil, nil + } + + return []corev1.EnvVar{{ + Name: fmt.Sprintf("INTERNAL_GATEWAY_IPV%d", af), + Value: internalGateway, + }, { + Name: fmt.Sprintf("EXTERNAL_GATEWAY_IPV%d", af), + Value: externalGateway, + }, { + Name: fmt.Sprintf("NO_SNAT_SOURCES_IPV%d", af), + Value: strings.Join(forwardSrc.SortedList(), ","), + }}, nil +} + +func bgpEdgeRouterContainerBFDD(image, bfdIP string, minTX, minRX, multiplier int32) corev1.Container { + return corev1.Container{ + Name: "bfdd", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"bash", "/kube-ovn/start-bfdd.sh"}, + Env: []corev1.EnvVar{{ + Name: "POD_IPS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIPs", + }, + }, + }, { + Name: "BFD_PEER_IPS", + Value: bfdIP, + }, { + Name: "BFD_MIN_TX", + Value: strconv.Itoa(int(minTX)), + }, { + Name: "BFD_MIN_RX", + Value: strconv.Itoa(int(minRX)), + }, { + Name: "BFD_MULTI", + Value: strconv.Itoa(int(multiplier)), + }}, + // wait for the BFD process to be running and initialize the BFD configuration + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bash", "/kube-ovn/bfdd-prestart.sh"}, + }, + }, + InitialDelaySeconds: 1, + FailureThreshold: 1, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bfdd-control", "status"}, + }, + }, + InitialDelaySeconds: 1, + PeriodSeconds: 5, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bfdd-control", "status"}, + }, + }, + InitialDelaySeconds: 3, + PeriodSeconds: 3, + FailureThreshold: 1, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](65534), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_BIND_SERVICE", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "usr-local-sbin", + MountPath: "/usr/local/sbin", + }}, + } +} + +func (c *Controller) handleDelBgpEdgeRouter(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.bgpEdgeRouterKeyMutex.LockKey(key) + defer func() { _ = c.bgpEdgeRouterKeyMutex.UnlockKey(key) }() + + cachedGateway, err := c.bgpEdgeRouterLister.BgpEdgeRouters(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + err = fmt.Errorf("failed to get bgp-edge-router %s: %w", key, err) + klog.Error(err) + return err + } + return nil + } + + klog.Infof("handle deleting bgp-edge-router %s", key) + if err = c.cleanOVNForBgpEdgeRouter(key, cachedGateway.Spec.VPC); err != nil { + klog.Error(err) + return err + } + + router := cachedGateway.DeepCopy() + if controllerutil.RemoveFinalizer(router, util.KubeOVNControllerFinalizer) { + if _, err = c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouters(router.Namespace). + Update(context.Background(), router, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to remove finalizer from bgp-edge-router %s: %w", key, err) + klog.Error(err) + } + } + + return nil +} + +func (c *Controller) cleanOVNForBgpEdgeRouter(key, lrName string) error { + externalIDs := map[string]string{ + ovs.ExternalIDVendor: util.CniTypeName, + ovs.ExternalIDBgpEdgeRouter: key, + } + + bfdList, err := c.OVNNbClient.FindBFD(externalIDs) + if err != nil { + klog.Error(err) + return err + } + for _, bfd := range bfdList { + if err = c.OVNNbClient.DeleteBFD(bfd.UUID); err != nil { + klog.Error(err) + return err + } + } + + if lrName == "" { + lrName = c.config.ClusterRouter + } + if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, -1, externalIDs); err != nil { + klog.Error(err) + return err + } + if err = c.OVNNbClient.DeletePortGroup(berPortGroupName(key)); err != nil { + klog.Error(err) + return err + } + for _, af := range [...]int{4, 6} { + if err = c.OVNNbClient.DeleteAddressSet(berAddressSetName(key, af)); err != nil { + klog.Error(err) + return err + } + } + + return nil +} + +func berPortGroupName(key string) string { + hash := util.Sha256Hash([]byte(key)) + return "BER." + hash[:12] +} + +func berAddressSetName(key string, af int) string { + hash := util.Sha256Hash([]byte(key)) + return fmt.Sprintf("BER.%s.ipv%d", hash[:12], af) +} + +func (c *Controller) handlePodEventForBgpEdgeRouter(pod *corev1.Pod) error { + if !pod.DeletionTimestamp.IsZero() || pod.Annotations[util.AllocatedAnnotation] != "true" { + return nil + } + vpc := pod.Annotations[util.LogicalRouterAnnotation] + if vpc == "" { + return nil + } + + router, err := c.bgpEdgeRouterLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list bgp edge router: %v", err) + utilruntime.HandleError(err) + return err + } + + for _, ber := range router { + if ber.VPC(c.config.ClusterRouter) != vpc { + continue + } + } + return nil +} + +func bgpEdgeRouterContainerBGP(speakerImage, routerName string, speakerParams *kubeovnv1.BgpEdgeRouterBGPConfig) (*corev1.Container, error) { + if speakerImage == "" { + return nil, errors.New("BGP speaker image must be specified") + } + if speakerParams == nil { + return nil, errors.New("BGP config must not be nil") + } + if speakerParams.ASN == 0 { + return nil, errors.New("ASN not set, but must be non-zero value") + } + if speakerParams.RemoteASN == 0 { + return nil, errors.New("remote ASN not set, but must be non-zero value") + } + if len(speakerParams.Neighbors) == 0 { + return nil, errors.New("no BGP neighbors specified") + } + + args := []string{} + if speakerParams.RouterID != "" { + args = append(args, "--router-id="+speakerParams.RouterID) + } + if speakerParams.Password != "" { + args = append(args, "--auth-password="+speakerParams.Password) + } + if speakerParams.EnableGracefulRestart { + args = append(args, "--graceful-restart") + } + if speakerParams.HoldTime != (metav1.Duration{}) { + args = append(args, "--holdtime="+speakerParams.HoldTime.Duration.String()) + } + if speakerParams.EdgeRouterMode { + args = append(args, "--edge-router-mode=true") + } + if speakerParams.RouteServerClient { + args = append(args, "--route-server-client=true") + } + args = append(args, fmt.Sprintf("--cluster-as=%d", speakerParams.ASN)) + args = append(args, fmt.Sprintf("--neighbor-as=%d", speakerParams.RemoteASN)) + + var neighIPv4, neighIPv6 []string + for _, neighbor := range speakerParams.Neighbors { + switch util.CheckProtocol(neighbor) { + case kubeovnv1.ProtocolIPv4: + neighIPv4 = append(neighIPv4, neighbor) + case kubeovnv1.ProtocolIPv6: + neighIPv6 = append(neighIPv6, neighbor) + default: + return nil, fmt.Errorf("unsupported protocol for peer %s", neighbor) + } + } + if len(neighIPv4) > 0 { + args = append(args, "--neighbor-address="+strings.Join(neighIPv4, ",")) + } + if len(neighIPv6) > 0 { + args = append(args, "--neighbor-ipv6-address="+strings.Join(neighIPv6, ",")) + } + + args = append(args, speakerParams.ExtraArgs...) + + container := &corev1.Container{ + Name: "bgp-router-speaker", + Image: speakerImage, + Command: []string{"/kube-ovn/kube-ovn-speaker"}, + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + { + Name: "EGRESS_GATEWAY_NAME", + Value: routerName, + }, + { + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + { + Name: "MULTI_NET_STATUS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.annotations['k8s.v1.cni.cncf.io/network-status']", + }, + }, + }, + }, + Args: args, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "kube-ovn-logs", + MountPath: "/var/log/kube-ovn", + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](0), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_BIND_SERVICE", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, + } + + return container, nil +} diff --git a/pkg/controller/bgp_edge_router_advertisement.go b/pkg/controller/bgp_edge_router_advertisement.go new file mode 100644 index 00000000000..572d01d4556 --- /dev/null +++ b/pkg/controller/bgp_edge_router_advertisement.go @@ -0,0 +1,743 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "reflect" + "regexp" + "slices" + "sort" + "strings" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + "k8s.io/utils/set" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +type updateVerObject struct { + key string + oldVer *kubeovnv1.BgpEdgeRouterAdvertisement + newVer *kubeovnv1.BgpEdgeRouterAdvertisement +} + +func (c *Controller) enqueueAddBgpEdgeRouterAdvertisement(obj any) { + key := cache.MetaObjectToName(obj.(*kubeovnv1.BgpEdgeRouterAdvertisement)).String() + klog.V(3).Infof("enqueue add bgp-edge-router-advertisement %s", key) + c.addBgpEdgeRouterAdvertisementQueue.Add(key) +} + +func (c *Controller) enqueueUpdateBgpEdgeRouterAdvertisement(oldObj, newObj any) { + key := cache.MetaObjectToName(newObj.(*kubeovnv1.BgpEdgeRouterAdvertisement)).String() + klog.V(3).Infof("enqueue update bgp-edge-router-advertisement %s", key) + + if oldObj == nil || newObj == nil { + klog.Warningf("enqueue update bgp-edge-router-advertisement %s, but old object is nil", key) + return + } + + oldRouter := oldObj.(*kubeovnv1.BgpEdgeRouterAdvertisement) + newRouter := newObj.(*kubeovnv1.BgpEdgeRouterAdvertisement) + updateVer := &updateVerObject{ + key: key, + oldVer: oldRouter, + newVer: newRouter, + } + + if !newRouter.DeletionTimestamp.IsZero() { + c.deleteBgpEdgeRouterAdvertisementQueue.Add(key) + return + } + + if !reflect.DeepEqual(oldRouter.Spec, newRouter.Spec) { + klog.Infof("enqueue update bgp-edge-router-advertisement %s", key) + c.updateBgpEdgeRouterAdvertisementQueue.Add(updateVer) + } +} + +func (c *Controller) enqueueDeleteBgpEdgeRouterAdvertisement(obj any) { + var berAd *kubeovnv1.BgpEdgeRouterAdvertisement + switch t := obj.(type) { + case *kubeovnv1.BgpEdgeRouterAdvertisement: + berAd = t + case cache.DeletedFinalStateUnknown: + if v, ok := t.Obj.(*kubeovnv1.BgpEdgeRouterAdvertisement); ok { + berAd = v + } + } + if berAd == nil { + klog.Warning("enqueueDeleteBgpEdgeRouterAdvertisement: object is not BgpEdgeRouterAdvertisement") + return + } + key := cache.MetaObjectToName(obj.(*kubeovnv1.BgpEdgeRouterAdvertisement)).String() + klog.V(3).Infof("enqueue delete bgp-edge-router-advertisement %s", key) + c.deleteBgpEdgeRouterAdvertisementQueue.Add(key) +} + +func (c *Controller) handleAddBgpEdgeRouterAdvertisement(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.bgpEdgeRouterAdvertisementKeyMutex.LockKey(key) + defer func() { _ = c.bgpEdgeRouterAdvertisementKeyMutex.UnlockKey(key) }() + + cachedAdvertisement, err := c.bgpEdgeRouterAdvertisementLister.BgpEdgeRouterAdvertisements(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + if !cachedAdvertisement.DeletionTimestamp.IsZero() { + c.deleteBgpEdgeRouterAdvertisementQueue.Add(key) + return nil + } + klog.V(3).Infof("debug bgp-edge-router-advertisement %s", cachedAdvertisement.Name) + + if _, err := c.initBgpEdgeRouterAdvertisementStatus(cachedAdvertisement); err != nil { + klog.Error(err) + return err + } + + klog.Infof("reconciling bgp-edge-router-advertisement %s", key) + advertisement := cachedAdvertisement.DeepCopy() + + if controllerutil.AddFinalizer(advertisement, util.KubeOVNControllerFinalizer) { + updatedAdvertisement, err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouterAdvertisements(advertisement.Namespace). + Update(context.Background(), advertisement, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to add finalizer for bgp-edge-router %s/%s: %w", advertisement.Namespace, advertisement.Name, err) + klog.Error(err) + return err + } + advertisement = updatedAdvertisement + } + + pods, err := c.validateBgpEdgeRouterAdvertisement(advertisement) + if err != nil || pods == nil { + klog.Error(err) + return err + } + + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + klog.Infof("handle adding bgp-edge-router-advertisement %s", key) + if err = c.addOrDeleteBgpEdgeRouterAdvertisementRule("add", key, pod, advertisement.Spec.Subnet); err != nil { + klog.Error(err) + return err + } + } + + advertisement.Status.Conditions.SetReady("ReconcileSuccess", advertisement.Generation) + if _, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement); err != nil { + return err + } + + // update ber address_set + if err := c.updateAddressSetForBer(ns, advertisement, "add"); err != nil { + klog.Error(err) + return err + } + + klog.Infof("finished reconciling bgp-edge-router-advertisement %s", key) + + return nil +} + +func (c *Controller) updateAddressSetForBer(ns string, advertisement *kubeovnv1.BgpEdgeRouterAdvertisement, op string) error { + // modify ber address_set + berName := advertisement.Spec.BgpEdgeRouter + cachedRouter, err := c.bgpEdgeRouterLister.BgpEdgeRouters(ns).Get(berName) + if err != nil { + klog.Error(err) + return err + } + router := cachedRouter.DeepCopy() + // collect egress policies + ipv4ForwardSrc, ipv6ForwardSrc := set.New[string](), set.New[string]() + ipv4SNATSrc, ipv6SNATSrc := set.New[string](), set.New[string]() + for _, policy := range router.Spec.Policies { + ipv4, ipv6 := util.SplitIpsByProtocol(policy.IPBlocks) + if policy.SNAT { + ipv4SNATSrc.Insert(ipv4...) + ipv6SNATSrc.Insert(ipv6...) + } else { + ipv4ForwardSrc.Insert(ipv4...) + ipv6ForwardSrc.Insert(ipv6...) + } + for _, subnetName := range policy.Subnets { + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + klog.Error(err) + return err + } + if subnet.Status.IsNotValidated() { + err = fmt.Errorf("subnet %s is not validated", subnet.Name) + klog.Error(err) + return err + } + // TODO: check subnet's vpc and vlan + ipv4, ipv6 := util.SplitStringIP(subnet.Spec.CIDRBlock) + if policy.SNAT { + ipv4SNATSrc.Insert(ipv4) + ipv6SNATSrc.Insert(ipv6) + } else { + ipv4ForwardSrc.Insert(ipv4) + ipv6ForwardSrc.Insert(ipv6) + } + } + } + + // collect advertisement subnets + if op == "add" { + advCidrBlocks, err := c.getSubnetCidrBlock(advertisement) + if err != nil { + klog.Error(err) + return err + } + for _, advCidrBlock := range advCidrBlocks { + ipv4adv, ipv6adv := util.SplitStringIP(advCidrBlock) + ipv4ForwardSrc.Insert(ipv4adv) + ipv6ForwardSrc.Insert(ipv6adv) + } + } + + // calculate internal route destinations and forward source CIDR blocks + intRouteDstIPv4, intRouteDstIPv6 := ipv4ForwardSrc.Union(ipv4SNATSrc), ipv6ForwardSrc.Union(ipv6SNATSrc) + intRouteDstIPv4.Delete("") + intRouteDstIPv6.Delete("") + ipv4ForwardSrc.Delete("") + ipv6ForwardSrc.Delete("") + + klog.Infof("setting address set for bgp edge router : %s, intRouteDstIPv4 %v, intRouteDstIPv6 %v", berName, intRouteDstIPv4, intRouteDstIPv6) + berKey := cache.MetaObjectToName(router).String() + klog.Infof("debug bgp-edge-router %s", berKey) + if intRouteDstIPv4.Len() > 0 { + asName := berAddressSetName(berKey, 4) + klog.Infof("address set name: %s", asName) + if err = c.OVNNbClient.AddressSetUpdateAddress(asName, intRouteDstIPv4.SortedList()...); err != nil { + klog.Error(err) + err = fmt.Errorf("failed to create or update address set %s: %w", asName, err) + klog.Error(err) + return err + } + } + if intRouteDstIPv6.Len() > 0 { + asName := berAddressSetName(berKey, 6) + if err = c.OVNNbClient.AddressSetUpdateAddress(asName, intRouteDstIPv6.SortedList()...); err != nil { + klog.Error(err) + err = fmt.Errorf("failed to create or update address set %s: %w", asName, err) + klog.Error(err) + return err + } + } + return nil +} + +func (c *Controller) handleUpdateBgpEdgeRouterAdvertisement(updatedObj *updateVerObject) error { + key := updatedObj.key + + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.bgpEdgeRouterAdvertisementKeyMutex.LockKey(key) + defer func() { _ = c.bgpEdgeRouterAdvertisementKeyMutex.UnlockKey(key) }() + + cachedAdvertisement, err := c.bgpEdgeRouterAdvertisementLister.BgpEdgeRouterAdvertisements(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + if !cachedAdvertisement.DeletionTimestamp.IsZero() { + c.deleteBgpEdgeRouterAdvertisementQueue.Add(key) + return nil + } + + klog.Infof("reconciling bgp-edge-router-advertisement %s", key) + advertisement := cachedAdvertisement.DeepCopy() + + pods, err := c.validateBgpEdgeRouterAdvertisement(advertisement) + if err != nil || pods == nil { + klog.Error(err) + return err + } + + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + klog.Infof("handle adding bgp-edge-router-advertisement %s", key) + if err = c.updateBgpEdgeRouterAdvertisementRule(key, pod, updatedObj.oldVer, updatedObj.newVer); err != nil { + klog.Error(err) + return err + } + } + + // update ber address_set + if err := c.updateAddressSetForBer(ns, advertisement, "add"); err != nil { + klog.Error(err) + return err + } + + advertisement.Status.Conditions.SetReady("ReconcileSuccess", advertisement.Generation) + if _, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement); err != nil { + return err + } + + klog.Infof("finished reconciling bgp-edge-router-advertisement %s", key) + + return nil +} + +func (c *Controller) handleDelBgpEdgeRouterAdvertisement(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.bgpEdgeRouterKeyMutex.LockKey(key) + defer func() { _ = c.bgpEdgeRouterKeyMutex.UnlockKey(key) }() + + cachedAdvertisement, err := c.bgpEdgeRouterAdvertisementLister.BgpEdgeRouterAdvertisements(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + klog.Infof("reconciling bgp-edge-router-advertisement %s", key) + advertisement := cachedAdvertisement.DeepCopy() + + pods, err := c.validateBgpEdgeRouterAdvertisement(advertisement) + if err != nil || pods == nil { + klog.Error(err) + return err + } + + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + klog.Infof("handle deleting bgp-edge-router-advertisement %s", key) + if err = c.addOrDeleteBgpEdgeRouterAdvertisementRule("del", key, pod, advertisement.Spec.Subnet); err != nil { + klog.Error(err) + return err + } + } + + advertisement = cachedAdvertisement.DeepCopy() + if controllerutil.RemoveFinalizer(advertisement, util.KubeOVNControllerFinalizer) { + if _, err = c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouterAdvertisements(advertisement.Namespace). + Update(context.Background(), advertisement, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to remove finalizer from bgp-edge-router-advertisement %s: %w", key, err) + klog.Error(err) + } + } + + // update ber address_set + if err := c.updateAddressSetForBer(ns, advertisement, "del"); err != nil { + klog.Error(err) + return err + } + + // advertisement.Status.Conditions.SetReady("ReconcileSuccess", advertisement.Generation) + // if _, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement); err != nil { + // return err + // } + + klog.Infof("finished reconciling bgp-edge-router-advertisement %s", key) + + return nil +} + +func (c *Controller) updateBgpEdgeRouterAdvertisementRule(key string, pod *corev1.Pod, oldBerAd, newBerAd *kubeovnv1.BgpEdgeRouterAdvertisement) error { + if pod.Name == "" { + err := fmt.Errorf("failed to get pod name %s", pod.Name) + klog.Error(err) + return err + } + var oldSubnetArray []string + var newSubnetArray []string + + for _, subnetName := range oldBerAd.Spec.Subnet { + var subnet *kubeovnv1.Subnet + var err error + if subnet, err = c.subnetsLister.Get(subnetName); err != nil { + err = fmt.Errorf("failed to get subnet %s: %w", subnetName, err) + klog.Error(err) + return err + } + if subnet.Spec.CIDRBlock != "" { + oldSubnetArray = append(oldSubnetArray, subnet.Spec.CIDRBlock) + } + klog.Infof("cleaning bgp-edge-router-advertisement %s for subnet %s", key, subnet.Name) + } + + for _, subnetName := range newBerAd.Spec.Subnet { + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + err = fmt.Errorf("failed to get subnet %s: %w", subnetName, err) + klog.Error(err) + return err + } + if subnet.Spec.CIDRBlock != "" { + newSubnetArray = append(newSubnetArray, subnet.Spec.CIDRBlock) + } + klog.Infof("cleaning bgp-edge-router-advertisement %s for subnet %s", key, subnet.Name) + } + + if err := c.execUpdateBgpRoute(pod, oldSubnetArray, newSubnetArray); err != nil { + klog.Error(err) + return err + } + + return nil +} + +func (c *Controller) addOrDeleteBgpEdgeRouterAdvertisementRule(op, key string, pod *corev1.Pod, subnetNames []string) error { + if pod.Name == "" { + err := fmt.Errorf("failed to get pod name %s", pod.Name) + klog.Error(err) + return err + } + SubnetCidrArray := []string{} + for _, subnetName := range subnetNames { + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + err = fmt.Errorf("failed to get subnet %s: %w", subnetName, err) + klog.Error(err) + return err + } + if subnet.Spec.CIDRBlock != "" { + SubnetCidrArray = append(SubnetCidrArray, subnet.Spec.CIDRBlock) + } + klog.Infof("cleaning bgp-edge-router-advertisement %s for subnet %s", key, subnet.Name) + } + + if op == "add" { + if err := c.execUpdateBgpRoute(pod, nil, SubnetCidrArray); err != nil { + klog.Error(err) + return err + } + } else { + if err := c.execUpdateBgpRoute(pod, SubnetCidrArray, nil); err != nil { + klog.Error(err) + return err + } + } + + return nil +} + +func (c *Controller) resyncBgpRules() { + klog.Info("resync bgp edge router") + // resync all bgp edge routers + bgpEdgeRouterAds, err := c.bgpEdgeRouterAdvertisementLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list bgp edge routers: %v", err) + return + } + + for _, bgpEdgeRouterAd := range bgpEdgeRouterAds { + // Check router.Spec.BGP.AdvertisedRoutes same with pods bgp advertised routes + if err := c.syncAdvertisedRoutes(bgpEdgeRouterAd); err != nil { + klog.Errorf("failed to sync advertised routes for bgp edge router %s: %v", bgpEdgeRouterAd.Name, err) + continue + } + klog.Infof("resync bgp edge router %s", bgpEdgeRouterAd.Name) + } +} + +func (c *Controller) validateBgpEdgeRouterAdvertisement(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement) ([]*corev1.Pod, error) { + deploy, err := c.berDeploymentsLister.Deployments(advertisement.Namespace).Get(advertisement.Spec.BgpEdgeRouter) + if err != nil { + advertisement.Status.Ready = false + msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) + advertisement.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "BgpEdgeRouterDeployNotFound", msg, advertisement.Generation) + _, _ = c.updatebgpEdgeRouterAdvertisementStatus(advertisement) + klog.Error(err) + return nil, err + } + + ready := util.DeploymentIsReady(deploy) + if !ready { + advertisement.Status.Ready = false + msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) + advertisement.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "BgpEdgeRouterNotEnabled", msg, advertisement.Generation) + _, _ = c.updatebgpEdgeRouterAdvertisementStatus(advertisement) + readyErr := fmt.Sprintf("Kind %s, Deployment %s is not ready", deploy.Kind, deploy.Name) + klog.Error(readyErr) + return nil, fmt.Errorf("%s", readyErr) + } + // get the pods of the deployment to collect the pod IPs + podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) + if err != nil { + err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return nil, err + } + + pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) + if err != nil { + err = fmt.Errorf("failed to list pods of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return nil, err + } + + if ready { + advertisement.Status.Ready = true + } + + return pods, nil +} + +func (c *Controller) initBgpEdgeRouterAdvertisementStatus(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) { + var err error + advertisement, err = c.updatebgpEdgeRouterAdvertisementStatus(advertisement) + return advertisement, err +} + +func (c *Controller) updatebgpEdgeRouterAdvertisementStatus(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement) (*kubeovnv1.BgpEdgeRouterAdvertisement, error) { + if len(advertisement.Status.Conditions) == 0 { + advertisement.Status.Conditions.SetCondition(kubeovnv1.Init, corev1.ConditionUnknown, "Processing", "", advertisement.Generation) + } + + updateAdvertisement, err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouterAdvertisements(advertisement.Namespace). + UpdateStatus(context.Background(), advertisement, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to update status of bgp-edge-router %s/%s: %w", advertisement.Namespace, advertisement.Name, err) + klog.Error(err) + return nil, err + } + + return updateAdvertisement, nil +} + +func (c *Controller) execUpdateBgpRoute(pod *corev1.Pod, oldCidrs, newCidrs []string) error { + // add_announced_route + cmdArs := []string{} + if len(oldCidrs) > 0 { + cmdArs = append(cmdArs, "del_announced_route="+strings.Join(oldCidrs, ",")) + } + if len(newCidrs) > 0 { + cmdArs = append(cmdArs, "add_announced_route="+strings.Join(newCidrs, ",")) + } + cmdArs = append(cmdArs, "list_announced_route") + cmd := fmt.Sprintf("bash /kube-ovn/update-bgp-route.sh %s", strings.Join(cmdArs, " ")) + + klog.Infof("exec command : %s", cmd) + stdOutput, errOutput, err := util.ExecuteCommandInContainer(c.config.KubeClient, c.config.KubeRestConfig, pod.Namespace, pod.Name, "bgp-router-speaker", []string{"/bin/bash", "-c", cmd}...) + if err != nil { + if len(errOutput) > 0 { + klog.Errorf("failed to ExecuteCommandInContainer, errOutput: %v", errOutput) + } + if len(stdOutput) > 0 { + klog.Infof("failed to ExecuteCommandInContainer, stdOutput: %v", stdOutput) + } + klog.Error(err) + return err + } + + if len(stdOutput) > 0 { + klog.Infof("ExecuteCommandInContainer stdOutput: %v", stdOutput) + } + + if len(errOutput) > 0 { + klog.Errorf("failed to ExecuteCommandInContainer errOutput: %v", errOutput) + return errors.New(errOutput) + } + + // list the current rule and check if the routes are updated + + return nil +} + +func (c *Controller) syncAdvertisedRoutes(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement) error { + key := cache.MetaObjectToName(advertisement).String() + + c.bgpEdgeRouterAdvertisementKeyMutex.LockKey(key) + defer func() { _ = c.bgpEdgeRouterAdvertisementKeyMutex.UnlockKey(key) }() + + if !advertisement.DeletionTimestamp.IsZero() { + c.deleteBgpEdgeRouterAdvertisementQueue.Add(key) + return nil + } + klog.Infof("reconciling bgp-edge-router %s", key) + // Deep copy because we might mutate Status below. + cachedAdvertisement := advertisement.DeepCopy() + + pods, err := c.validateBgpEdgeRouterAdvertisement(cachedAdvertisement) + if err != nil || pods == nil { + klog.Error(err) + return err + } + cidrBlock, err := c.getSubnetCidrBlock(cachedAdvertisement) + if err != nil { + klog.Error(err) + return err + } + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + podCidr, err := c.execGetBgpRoute(pod) + if err != nil { + return err + } + klog.Infof("current router advertised routes: %v", cidrBlock) + klog.Infof("router pod %s/%s advertised routes: %v", pod.Namespace, pod.Name, podCidr) + routesDiff := !slicesEqual(podCidr, cidrBlock) + if routesDiff { + if err := c.execUpdateBgpRoute(pod, podCidr, cidrBlock); err != nil { + return err + } + klog.Infof("synced advertised routes for bgp-router-speaker %s pod %s/%s", key, pod.Namespace, pod.Name) + } + } + + klog.Infof("finished sync bgp-edge-router %s advertised routes", key) + return nil +} + +func (c *Controller) execGetBgpRoute(routerPod *corev1.Pod) ([]string, error) { + cmd := "bash /kube-ovn/update-bgp-route.sh list_announced_route" + klog.Infof("exec command : %s", cmd) + stdOutput, errOutput, err := util.ExecuteCommandInContainer(c.config.KubeClient, c.config.KubeRestConfig, routerPod.Namespace, routerPod.Name, "bgp-router-speaker", []string{"/bin/bash", "-c", cmd}...) + if err != nil { + if len(errOutput) > 0 { + klog.Errorf("failed to ExecuteCommandInContainer, errOutput: %v", errOutput) + } + klog.Error(err) + return nil, err + } + + if len(stdOutput) > 0 { + klog.Infof("ExecuteCommandInContainer stdOutput: %v", stdOutput) + } + if len(errOutput) > 0 { + klog.Errorf("failed to ExecuteCommandInContainer errOutput: %v", errOutput) + return nil, errors.New(errOutput) + } + + // Parse the output to extract announced routes + announcedRoutes, err := c.parseBgpAnnouncedRoutes(stdOutput) + if err != nil { + klog.Errorf("failed to parse BGP announced routes: %v", err) + return nil, err + } + + return announcedRoutes, nil +} + +func (c *Controller) getSubnetCidrBlock(advertisement *kubeovnv1.BgpEdgeRouterAdvertisement) ([]string, error) { + var cirdBlock []string + for _, subnetName := range advertisement.Spec.Subnet { + var subnet *kubeovnv1.Subnet + var err error + subnet, err = c.subnetsLister.Get(subnetName) + if err != nil { + err = fmt.Errorf("failed to get subnet %s: %w", subnetName, err) + klog.Error(err) + return nil, err + } + if subnet.Spec.CIDRBlock != "" { + cirdBlock = append(cirdBlock, subnet.Spec.CIDRBlock) + } + } + return cirdBlock, nil +} + +func slicesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + + // Create copies and sort them + aCopy := make([]string, len(a)) + bCopy := make([]string, len(b)) + copy(aCopy, a) + copy(bCopy, b) + + sort.Strings(aCopy) + sort.Strings(bCopy) + + return slices.Equal(aCopy, bCopy) +} + +func (c *Controller) parseBgpAnnouncedRoutes(output string) ([]string, error) { + var routes []string + + // Look for the specific section with next-hop routes + lines := strings.Split(output, "\n") + inTargetSection := false + foundRoutesSection := false + + // Regex to match route lines starting with "*>" followed by CIDR + routeRegex := regexp.MustCompile(`^\*>\s+(\d+\.\d+\.\d+\.\d+/\d+)`) + + for _, line := range lines { + line = strings.TrimSpace(line) + + // Start parsing when we find the target section with any IP address + if strings.Contains(line, "--- Routes with Next-Hop") && strings.Contains(line, "---") { + inTargetSection = true + continue + } + + // Look for the IPv4 routes subsection + if inTargetSection && strings.Contains(line, "IPv4 routes with next-hop") { + foundRoutesSection = true + continue + } + + // Stop parsing if we hit another section starting with "---" or "===" + if inTargetSection && foundRoutesSection && (strings.HasPrefix(line, "---") || strings.HasPrefix(line, "===")) { + break + } + + // Skip header lines (Network, Next Hop, AS_PATH, etc.) + if inTargetSection && (strings.Contains(line, "Network") && strings.Contains(line, "Next Hop")) { + continue + } + + // Parse route lines in the target section that start with "*>" + if inTargetSection && foundRoutesSection && routeRegex.MatchString(line) { + matches := routeRegex.FindStringSubmatch(line) + if len(matches) > 1 { + routes = append(routes, matches[1]) + } + } + } + + if len(routes) == 0 { + return nil, errors.New("no announced routes found in BGP output") + } + + return routes, nil +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 73fe6a70c46..214d2e95fff 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -107,6 +107,26 @@ type Controller struct { delVpcEgressGatewayQueue workqueue.TypedRateLimitingInterface[string] vpcEgressGatewayKeyMutex keymutex.KeyMutex + bgpEdgeRouterLister kubeovnlister.BgpEdgeRouterLister + bgpEdgeRouterSynced cache.InformerSynced + addOrUpdateBgpEdgeRouterQueue workqueue.TypedRateLimitingInterface[string] + delBgpEdgeRouterQueue workqueue.TypedRateLimitingInterface[string] + bgpEdgeRouterKeyMutex keymutex.KeyMutex + + bgpEdgeRouterAdvertisementLister kubeovnlister.BgpEdgeRouterAdvertisementLister + bgpEdgeRouterAdvertisementSynced cache.InformerSynced + addBgpEdgeRouterAdvertisementQueue workqueue.TypedRateLimitingInterface[string] + updateBgpEdgeRouterAdvertisementQueue workqueue.TypedRateLimitingInterface[*updateVerObject] + deleteBgpEdgeRouterAdvertisementQueue workqueue.TypedRateLimitingInterface[string] + bgpEdgeRouterAdvertisementKeyMutex keymutex.KeyMutex + + gobgpConfigLister kubeovnlister.GobgpConfigLister + gobgpConfigSynced cache.InformerSynced + addGobgpConfigQueue workqueue.TypedRateLimitingInterface[string] + updateGobgpConfigQueue workqueue.TypedRateLimitingInterface[*updateVerGobgpConfigObject] + deleteGobgpConfigQueue workqueue.TypedRateLimitingInterface[string] + gobgpConfigKeyMutex keymutex.KeyMutex + switchLBRuleLister kubeovnlister.SwitchLBRuleLister switchLBRuleSynced cache.InformerSynced addSwitchLBRuleQueue workqueue.TypedRateLimitingInterface[string] @@ -233,6 +253,9 @@ type Controller struct { deploymentsLister appsv1.DeploymentLister deploymentsSynced cache.InformerSynced + berDeploymentsLister appsv1.DeploymentLister + berDeploymentsSynced cache.InformerSynced + npsLister netv1.NetworkPolicyLister npsSynced cache.InformerSynced updateNpQueue workqueue.TypedRateLimitingInterface[string] @@ -281,12 +304,13 @@ type Controller struct { netAttachSynced cache.InformerSynced netAttachInformerFactory netAttach.SharedInformerFactory - recorder record.EventRecorder - informerFactory kubeinformers.SharedInformerFactory - cmInformerFactory kubeinformers.SharedInformerFactory - deployInformerFactory kubeinformers.SharedInformerFactory - kubeovnInformerFactory kubeovninformer.SharedInformerFactory - anpInformerFactory anpinformer.SharedInformerFactory + recorder record.EventRecorder + informerFactory kubeinformers.SharedInformerFactory + cmInformerFactory kubeinformers.SharedInformerFactory + deployInformerFactory kubeinformers.SharedInformerFactory + berDeployInformerFactory kubeinformers.SharedInformerFactory + kubeovnInformerFactory kubeovninformer.SharedInformerFactory + anpInformerFactory anpinformer.SharedInformerFactory // Database health check dbFailureCount int @@ -316,6 +340,11 @@ func Run(ctx context.Context, config *Configuration) { util.LogFatalAndExit(err, "failed to create label selector for vpc egress gateway workload") } + berSelector, berErr := labels.Parse(util.BgpEdgeRouterLabel) + if berErr != nil { + util.LogFatalAndExit(berErr, "failed to create label selector for bgp edge router workload") + } + informerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeFactoryClient, 0, kubeinformers.WithTweakListOptions(func(listOption *metav1.ListOptions) { listOption.AllowWatchBookmarks = true @@ -330,6 +359,12 @@ func Run(ctx context.Context, config *Configuration) { listOption.AllowWatchBookmarks = true listOption.LabelSelector = selector.String() })) + // deployment informer used to list/watch bgp edge router workloads + berDeployInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeFactoryClient, 0, + kubeinformers.WithTweakListOptions(func(listOption *metav1.ListOptions) { + listOption.AllowWatchBookmarks = true + listOption.LabelSelector = berSelector.String() + })) kubeovnInformerFactory := kubeovninformer.NewSharedInformerFactoryWithOptions(config.KubeOvnFactoryClient, 0, kubeovninformer.WithTweakListOptions(func(listOption *metav1.ListOptions) { listOption.AllowWatchBookmarks = true @@ -350,6 +385,9 @@ func Run(ctx context.Context, config *Configuration) { vpcInformer := kubeovnInformerFactory.Kubeovn().V1().Vpcs() vpcNatGatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcNatGateways() vpcEgressGatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcEgressGateways() + bgpEdgeRouterInformer := kubeovnInformerFactory.Kubeovn().V1().BgpEdgeRouters() + bgpEdgeRouterAdvertisementInformer := kubeovnInformerFactory.Kubeovn().V1().BgpEdgeRouterAdvertisements() + gobgpConfigInformer := kubeovnInformerFactory.Kubeovn().V1().GobgpConfigs() subnetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets() ippoolInformer := kubeovnInformerFactory.Kubeovn().V1().IPPools() ipInformer := kubeovnInformerFactory.Kubeovn().V1().IPs() @@ -367,6 +405,7 @@ func Run(ctx context.Context, config *Configuration) { serviceInformer := informerFactory.Core().V1().Services() endpointSliceInformer := informerFactory.Discovery().V1().EndpointSlices() deploymentInformer := deployInformerFactory.Apps().V1().Deployments() + berDeploymentInformer := berDeployInformerFactory.Apps().V1().Deployments() qosPolicyInformer := kubeovnInformerFactory.Kubeovn().V1().QoSPolicies() configMapInformer := cmInformerFactory.Core().V1().ConfigMaps() npInformer := informerFactory.Networking().V1().NetworkPolicies() @@ -414,6 +453,26 @@ func Run(ctx context.Context, config *Configuration) { delVpcEgressGatewayQueue: newTypedRateLimitingQueue("DeleteVpcEgressGateway", custCrdRateLimiter), vpcEgressGatewayKeyMutex: keymutex.NewHashed(numKeyLocks), + bgpEdgeRouterLister: bgpEdgeRouterInformer.Lister(), + bgpEdgeRouterSynced: bgpEdgeRouterInformer.Informer().HasSynced, + addOrUpdateBgpEdgeRouterQueue: newTypedRateLimitingQueue("AddOrUpdateBgpEdgeRouter", custCrdRateLimiter), + delBgpEdgeRouterQueue: newTypedRateLimitingQueue("DeleteBgpEdgeRouter", custCrdRateLimiter), + bgpEdgeRouterKeyMutex: keymutex.NewHashed(numKeyLocks), + + bgpEdgeRouterAdvertisementLister: bgpEdgeRouterAdvertisementInformer.Lister(), + bgpEdgeRouterAdvertisementSynced: bgpEdgeRouterAdvertisementInformer.Informer().HasSynced, + addBgpEdgeRouterAdvertisementQueue: newTypedRateLimitingQueue("AddBgpEdgeRouterAdvertisement", custCrdRateLimiter), + updateBgpEdgeRouterAdvertisementQueue: newTypedRateLimitingQueue[*updateVerObject]("UpdateBgpEdgeRouterAdvertisement", nil), + deleteBgpEdgeRouterAdvertisementQueue: newTypedRateLimitingQueue("DeleteBgpEdgeRouterAdvertisement", custCrdRateLimiter), + bgpEdgeRouterAdvertisementKeyMutex: keymutex.NewHashed(numKeyLocks), + + gobgpConfigLister: gobgpConfigInformer.Lister(), + gobgpConfigSynced: gobgpConfigInformer.Informer().HasSynced, + addGobgpConfigQueue: newTypedRateLimitingQueue("AddGobgpConfig", custCrdRateLimiter), + updateGobgpConfigQueue: newTypedRateLimitingQueue[*updateVerGobgpConfigObject]("UpdateGobgpConfig", nil), + deleteGobgpConfigQueue: newTypedRateLimitingQueue("DeleteGobgpConfig", custCrdRateLimiter), + gobgpConfigKeyMutex: keymutex.NewHashed(numKeyLocks), + subnetsLister: subnetInformer.Lister(), subnetSynced: subnetInformer.Informer().HasSynced, addOrUpdateSubnetQueue: newTypedRateLimitingQueue[string]("AddSubnet", nil), @@ -517,6 +576,9 @@ func Run(ctx context.Context, config *Configuration) { deploymentsLister: deploymentInformer.Lister(), deploymentsSynced: deploymentInformer.Informer().HasSynced, + berDeploymentsLister: berDeploymentInformer.Lister(), + berDeploymentsSynced: berDeploymentInformer.Informer().HasSynced, + qosPoliciesLister: qosPolicyInformer.Lister(), qosPolicySynced: qosPolicyInformer.Informer().HasSynced, addQoSPolicyQueue: newTypedRateLimitingQueue("AddQoSPolicy", custCrdRateLimiter), @@ -570,12 +632,13 @@ func Run(ctx context.Context, config *Configuration) { netAttachSynced: netAttachInformer.Informer().HasSynced, netAttachInformerFactory: attachNetInformerFactory, - recorder: recorder, - informerFactory: informerFactory, - cmInformerFactory: cmInformerFactory, - deployInformerFactory: deployInformerFactory, - kubeovnInformerFactory: kubeovnInformerFactory, - anpInformerFactory: anpInformerFactory, + recorder: recorder, + informerFactory: informerFactory, + cmInformerFactory: cmInformerFactory, + deployInformerFactory: deployInformerFactory, + berDeployInformerFactory: berDeployInformerFactory, + kubeovnInformerFactory: kubeovnInformerFactory, + anpInformerFactory: anpInformerFactory, } if controller.OVNNbClient, err = ovs.NewOvnNbClient( @@ -652,6 +715,7 @@ func Run(ctx context.Context, config *Configuration) { controller.informerFactory.Start(ctx.Done()) controller.cmInformerFactory.Start(ctx.Done()) controller.deployInformerFactory.Start(ctx.Done()) + controller.berDeployInformerFactory.Start(ctx.Done()) controller.kubeovnInformerFactory.Start(ctx.Done()) controller.anpInformerFactory.Start(ctx.Done()) controller.StartKubevirtInformerFactory(ctx, kubevirtInformerFactory) @@ -664,9 +728,9 @@ func Run(ctx context.Context, config *Configuration) { controller.ipSynced, controller.virtualIpsSynced, controller.iptablesEipSynced, controller.iptablesFipSynced, controller.iptablesDnatRuleSynced, controller.iptablesSnatRuleSynced, controller.vlanSynced, controller.podsSynced, controller.namespacesSynced, controller.nodesSynced, - controller.serviceSynced, controller.endpointSlicesSynced, controller.deploymentsSynced, controller.configMapsSynced, + controller.serviceSynced, controller.endpointSlicesSynced, controller.deploymentsSynced, controller.berDeploymentsSynced, controller.configMapsSynced, controller.ovnEipSynced, controller.ovnFipSynced, controller.ovnSnatRuleSynced, - controller.ovnDnatRuleSynced, + controller.ovnDnatRuleSynced, controller.bgpEdgeRouterSynced, } if controller.config.EnableLb { cacheSyncs = append(cacheSyncs, controller.switchLBRuleSynced, controller.vpcDNSSynced) @@ -728,6 +792,13 @@ func Run(ctx context.Context, config *Configuration) { util.LogFatalAndExit(err, "failed to add deployment event handler") } + if _, err = berDeploymentInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddDeployment, + UpdateFunc: controller.enqueueUpdateDeployment, + }); err != nil { + util.LogFatalAndExit(err, "failed to add deployment event handler") + } + if _, err = vpcInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueAddVpc, UpdateFunc: controller.enqueueUpdateVpc, @@ -744,6 +815,30 @@ func Run(ctx context.Context, config *Configuration) { util.LogFatalAndExit(err, "failed to add vpc nat gateway event handler") } + if _, err = bgpEdgeRouterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddBgpEdgeRouter, + UpdateFunc: controller.enqueueUpdateBgpEdgeRouter, + DeleteFunc: controller.enqueueDeleteBgpEdgeRouter, + }); err != nil { + util.LogFatalAndExit(err, "failed to add bgp edge router event handler") + } + + if _, err = bgpEdgeRouterAdvertisementInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddBgpEdgeRouterAdvertisement, + UpdateFunc: controller.enqueueUpdateBgpEdgeRouterAdvertisement, + DeleteFunc: controller.enqueueDeleteBgpEdgeRouterAdvertisement, + }); err != nil { + util.LogFatalAndExit(err, "failed to add bgp edge router advertisement event handler") + } + + if _, err = gobgpConfigInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddGobgpConfig, + UpdateFunc: controller.enqueueUpdateGobgpConfig, + DeleteFunc: controller.enqueueDeleteGobgpConfig, + }); err != nil { + util.LogFatalAndExit(err, "failed to add gobgp config event handler") + } + if _, err = vpcEgressGatewayInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueAddVpcEgressGateway, UpdateFunc: controller.enqueueUpdateVpcEgressGateway, @@ -1093,6 +1188,17 @@ func (c *Controller) shutdown() { c.addOrUpdateVpcEgressGatewayQueue.ShutDown() c.delVpcEgressGatewayQueue.ShutDown() + c.addOrUpdateBgpEdgeRouterQueue.ShutDown() + c.delBgpEdgeRouterQueue.ShutDown() + + c.addBgpEdgeRouterAdvertisementQueue.ShutDown() + c.updateBgpEdgeRouterAdvertisementQueue.ShutDown() + c.deleteBgpEdgeRouterAdvertisementQueue.ShutDown() + + c.addGobgpConfigQueue.ShutDown() + c.updateGobgpConfigQueue.ShutDown() + c.deleteGobgpConfigQueue.ShutDown() + if c.config.EnableLb { c.addSwitchLBRuleQueue.ShutDown() c.delSwitchLBRuleQueue.ShutDown() @@ -1186,6 +1292,17 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(runWorker("delete vpc nat gateway", c.delVpcNatGatewayQueue, c.handleDelVpcNatGw), time.Second, ctx.Done()) go wait.Until(runWorker("add/update vpc egress gateway", c.addOrUpdateVpcEgressGatewayQueue, c.handleAddOrUpdateVpcEgressGateway), time.Second, ctx.Done()) go wait.Until(runWorker("delete vpc egress gateway", c.delVpcEgressGatewayQueue, c.handleDelVpcEgressGateway), time.Second, ctx.Done()) + go wait.Until(runWorker("add/update bgp edge router", c.addOrUpdateBgpEdgeRouterQueue, c.handleAddOrUpdateBgpEdgeRouter), time.Second, ctx.Done()) + go wait.Until(runWorker("delete bgp edge router", c.delBgpEdgeRouterQueue, c.handleDelBgpEdgeRouter), time.Second, ctx.Done()) + go wait.Until(runWorker("add bgp edge router advertisement", c.addBgpEdgeRouterAdvertisementQueue, c.handleAddBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) + go wait.Until(runWorker("update bgp edge router advertisement", c.updateBgpEdgeRouterAdvertisementQueue, c.handleUpdateBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) + go wait.Until(runWorker("delete bgp edge router advertisement", c.deleteBgpEdgeRouterAdvertisementQueue, c.handleDelBgpEdgeRouterAdvertisement), time.Second, ctx.Done()) + go wait.Until(c.resyncBgpRules, 60*time.Second, ctx.Done()) + go wait.Until(c.resyncBgpPolicyRules, 60*time.Second, ctx.Done()) + go wait.Until(runWorker("add bgp edge router advertisement", c.addGobgpConfigQueue, c.handleAddGobgpConfig), time.Second, ctx.Done()) + go wait.Until(runWorker("update bgp edge router advertisement", c.updateGobgpConfigQueue, c.handleUpdateGobgpConfig), time.Second, ctx.Done()) + go wait.Until(runWorker("delete bgp edge router advertisement", c.deleteGobgpConfigQueue, c.handleDelGobgpConfig), time.Second, ctx.Done()) + go wait.Until(runWorker("update fip for vpc nat gateway", c.updateVpcFloatingIPQueue, c.handleUpdateVpcFloatingIP), time.Second, ctx.Done()) go wait.Until(runWorker("update eip for vpc nat gateway", c.updateVpcEipQueue, c.handleUpdateVpcEip), time.Second, ctx.Done()) go wait.Until(runWorker("update dnat for vpc nat gateway", c.updateVpcDnatQueue, c.handleUpdateVpcDnat), time.Second, ctx.Done()) diff --git a/pkg/controller/deployment.go b/pkg/controller/deployment.go index c0828e47fe2..edc5b031544 100644 --- a/pkg/controller/deployment.go +++ b/pkg/controller/deployment.go @@ -15,6 +15,8 @@ var ( deploymentKind string vpcEgressGatewayGroupVersion string vpcEgressGatewayKind string + bgpEdgeRouterGroupVersion string + bgpEdgeRouterKind string ) func init() { @@ -27,6 +29,11 @@ func init() { gvk = kubeovnv1.SchemeGroupVersion.WithKind(name) vpcEgressGatewayGroupVersion = gvk.GroupVersion().String() vpcEgressGatewayKind = gvk.Kind + + name = reflect.TypeOf(&kubeovnv1.BgpEdgeRouter{}).Elem().Name() + gvk = kubeovnv1.SchemeGroupVersion.WithKind(name) + bgpEdgeRouterGroupVersion = gvk.GroupVersion().String() + bgpEdgeRouterKind = gvk.Kind } func (c *Controller) enqueueAddDeployment(obj any) { @@ -37,6 +44,11 @@ func (c *Controller) enqueueAddDeployment(obj any) { klog.V(3).Infof("enqueue update vpc-egress-gateway %s", key) c.addOrUpdateVpcEgressGatewayQueue.Add(key) return + } else if ref.APIVersion == bgpEdgeRouterGroupVersion && ref.Kind == bgpEdgeRouterKind { + key := types.NamespacedName{Namespace: deploy.Namespace, Name: ref.Name}.String() + klog.V(3).Infof("enqueue update bgp-edge-router %s", key) + c.addOrUpdateBgpEdgeRouterQueue.Add(key) + return } } } diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go index c2d7e84c891..f3893022621 100644 --- a/pkg/controller/gc.go +++ b/pkg/controller/gc.go @@ -50,6 +50,7 @@ func (c *Controller) gc() error { c.gcVip, c.gcLbSvcPods, c.gcVPCDNS, + c.gcEdgeRouter, } for _, gcFunc := range gcFunctions { if err := gcFunc(); err != nil { @@ -60,6 +61,54 @@ func (c *Controller) gc() error { return nil } +func (c *Controller) gcEdgeRouter() error { + klog.Infof("start to gc edge router") + + edgeRouterAdvertisement, errAdv := c.bgpEdgeRouterAdvertisementLister.List(labels.Everything()) + if errAdv != nil { + klog.Errorf("failed to list edge router advertisements, %v", errAdv) + return errAdv + } + for _, advertisement := range edgeRouterAdvertisement { + if err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouterAdvertisements(advertisement.Namespace).Delete(context.Background(), advertisement.Name, metav1.DeleteOptions{}); err != nil { + if !k8serrors.IsNotFound(err) { + klog.Errorf("failed to delete edge router advertisement %s, %v", advertisement.Name, err) + return err + } + } + } + + bgpConfigs, errConf := c.gobgpConfigLister.List(labels.Everything()) + if errConf != nil { + klog.Errorf("failed to list gobgp configs, %v", errConf) + return errConf + } + for _, config := range bgpConfigs { + if err := c.config.KubeOvnClient.KubeovnV1().GobgpConfigs(config.Namespace).Delete(context.Background(), config.Name, metav1.DeleteOptions{}); err != nil { + if !k8serrors.IsNotFound(err) { + klog.Errorf("failed to delete gobgp config %s, %v", config.Name, err) + return err + } + } + } + + edgeRouters, errER := c.bgpEdgeRouterLister.List(labels.Everything()) + if errER != nil { + klog.Errorf("failed to list edge routers, %v", errER) + return errER + } + for _, edgeRouter := range edgeRouters { + if err := c.config.KubeOvnClient.KubeovnV1().BgpEdgeRouters(edgeRouter.Namespace).Delete(context.Background(), edgeRouter.Name, metav1.DeleteOptions{}); err != nil { + if !k8serrors.IsNotFound(err) { + klog.Errorf("failed to delete edge router %s, %v", edgeRouter.Name, err) + return err + } + } + } + + return nil +} + func (c *Controller) gcLogicalRouterPort() error { klog.Infof("start to gc logical router port") vpcs, err := c.vpcsLister.List(labels.Everything()) diff --git a/pkg/controller/gobgp_config.go b/pkg/controller/gobgp_config.go new file mode 100644 index 00000000000..fb302d0f225 --- /dev/null +++ b/pkg/controller/gobgp_config.go @@ -0,0 +1,723 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "reflect" + "slices" + "strings" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +type updateVerGobgpConfigObject struct { + key string + oldVer *kubeovnv1.GobgpConfig + newVer *kubeovnv1.GobgpConfig +} + +func (c *Controller) enqueueAddGobgpConfig(obj any) { + key := cache.MetaObjectToName(obj.(*kubeovnv1.GobgpConfig)).String() + klog.V(3).Infof("enqueue add gobgp-configuration %s", key) + c.addGobgpConfigQueue.Add(key) +} + +func (c *Controller) enqueueUpdateGobgpConfig(oldObj, newObj any) { + key := cache.MetaObjectToName(newObj.(*kubeovnv1.GobgpConfig)).String() + klog.V(3).Infof("enqueue update gobgp-configuration %s", key) + + if oldObj == nil || newObj == nil { + klog.Warningf("enqueue update gobgp-configuration %s, but old object is nil", key) + return + } + + oldGobgpConfig := oldObj.(*kubeovnv1.GobgpConfig) + newGobgpConfig := newObj.(*kubeovnv1.GobgpConfig) + + updateConfigVer := &updateVerGobgpConfigObject{ + key: key, + oldVer: oldGobgpConfig, + newVer: newGobgpConfig, + } + + if !newGobgpConfig.DeletionTimestamp.IsZero() { + c.deleteGobgpConfigQueue.Add(key) + return + } + + if !reflect.DeepEqual(oldGobgpConfig.Spec, newGobgpConfig.Spec) { + klog.Infof("enqueue update gobgp-config %s", key) + c.updateGobgpConfigQueue.Add(updateConfigVer) + } +} + +func (c *Controller) enqueueDeleteGobgpConfig(obj any) { + var gobgpConfig *kubeovnv1.GobgpConfig + + switch t := obj.(type) { + case *kubeovnv1.GobgpConfig: + gobgpConfig = t + case cache.DeletedFinalStateUnknown: + if v, ok := t.Obj.(*kubeovnv1.GobgpConfig); ok { + gobgpConfig = v + } + } + + if gobgpConfig == nil { + klog.Warning("enqueueDeleteGobgpConfig: object is not GobgpConfig") + return + } + + key := cache.MetaObjectToName(obj.(*kubeovnv1.GobgpConfig)).String() + + klog.V(3).Infof("enqueue delete gobgp-config %s", key) + c.deleteGobgpConfigQueue.Add(key) +} + +func (c *Controller) handleAddGobgpConfig(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.gobgpConfigKeyMutex.LockKey(key) + defer func() { _ = c.gobgpConfigKeyMutex.UnlockKey(key) }() + + cachedGobgpConfig, err := c.gobgpConfigLister.GobgpConfigs(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + if !cachedGobgpConfig.DeletionTimestamp.IsZero() { + c.deleteGobgpConfigQueue.Add(key) + return nil + } + + klog.V(3).Infof("debug gobgp-config %s", cachedGobgpConfig.Name) + gobgpConfig := cachedGobgpConfig.DeepCopy() + if gobgpConfig, err = c.initGobgpConfigStatus(gobgpConfig); err != nil { + klog.Error(err) + return err + } + + klog.Infof("reconciling gobgp-configuration %s for add", key) + + if controllerutil.AddFinalizer(gobgpConfig, util.KubeOVNControllerFinalizer) { + updatedGobgpConfig, err := c.config.KubeOvnClient.KubeovnV1().GobgpConfigs(gobgpConfig.Namespace). + Update(context.Background(), + gobgpConfig, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to add finalizer for gobgp-configuration %s/%s: %w", gobgpConfig.Namespace, gobgpConfig.Name, err) + klog.Error(err) + return err + } + gobgpConfig = updatedGobgpConfig + } + + pods, err := c.validateGobgpConfig(gobgpConfig) + if err != nil || pods == nil { + klog.Error(err) + return err + } + + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + klog.Infof("handle adding gobgp-config to pod %s", pod.Name) + if err = c.execUpdateBgpPolicy(key, pod, nil, gobgpConfig); err != nil { + klog.Error(err) + return err + } + } + + gobgpConfig.Status.Conditions.SetReady("ReconcileSuccess", gobgpConfig.Generation) + if _, err = c.updateGobgpConfigStatus(gobgpConfig); err != nil { + return err + } + klog.Infof("finished reconciling gobgp-config %s", key) + + return nil +} + +func (c *Controller) handleUpdateGobgpConfig(updatedObj *updateVerGobgpConfigObject) error { + key := updatedObj.key + + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.gobgpConfigKeyMutex.LockKey(key) + defer func() { _ = c.gobgpConfigKeyMutex.UnlockKey(key) }() + + cachedGobgpConfig, err := c.gobgpConfigLister.GobgpConfigs(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + if !cachedGobgpConfig.DeletionTimestamp.IsZero() { + c.deleteGobgpConfigQueue.Add(key) + return nil + } + + klog.Infof("reconciling gobgp-configs %s for update", key) + gobgpConfig := cachedGobgpConfig.DeepCopy() + + pods, err := c.validateGobgpConfig(gobgpConfig) + if err != nil || pods == nil { + klog.Error(err) + return err + } + + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + klog.Infof("handle adding gobgp-configs to pod %s", pod.Name) + if err = c.execUpdateBgpPolicy(key, pod, updatedObj.oldVer, updatedObj.newVer); err != nil { + klog.Error(err) + return err + } + } + + gobgpConfig.Status.Conditions.SetReady("ReconcileSuccess", gobgpConfig.Generation) + if _, err = c.updateGobgpConfigStatus(gobgpConfig); err != nil { + return err + } + klog.Infof("finished reconciling gobgp-configs %s", key) + + return nil +} + +func (c *Controller) handleDelGobgpConfig(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.gobgpConfigKeyMutex.LockKey(key) + defer func() { _ = c.gobgpConfigKeyMutex.UnlockKey(key) }() + + cachedGobgpConfig, err := c.gobgpConfigLister.GobgpConfigs(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + klog.Infof("reconciling gobgp-configs %s", key) + gobgpConfig := cachedGobgpConfig.DeepCopy() + + pods, err := c.validateGobgpConfig(gobgpConfig) + if err != nil || pods == nil { + klog.Error(err) + return err + } + + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + klog.Infof("handle deleting gobgp-configs %s", key) + if err = c.execUpdateBgpPolicy(key, pod, gobgpConfig, nil); err != nil { + klog.Error(err) + return err + } + } + + gobgpConfig = cachedGobgpConfig.DeepCopy() + if controllerutil.RemoveFinalizer(gobgpConfig, util.KubeOVNControllerFinalizer) { + if _, err = c.config.KubeOvnClient.KubeovnV1().GobgpConfigs(gobgpConfig.Namespace). + Update(context.Background(), gobgpConfig, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to remove finalizer from gobgp-configs %s: %w", key, err) + klog.Error(err) + } + } + + klog.Infof("finished reconciling gobgp-config %s", key) + + return nil +} + +func (c *Controller) execUpdateBgpPolicy(key string, pod *corev1.Pod, oldGobgpConfig, newGobgpConfig *kubeovnv1.GobgpConfig) error { + klog.Infof("execUpdateBgpPolicy %s", key) + + if pod.Name == "" { + err := fmt.Errorf("failed to get pod name %s", pod.Name) + klog.Error(err) + return err + } + + cmdArs := []string{} + if oldGobgpConfig != nil { + klog.Infof("execUpdateBgpPolicy %s", key) + + for _, neighbor := range oldGobgpConfig.Spec.Neighbors { + nbrIP := neighbor.Address + if len(nbrIP) == 0 { + klog.Warningf("neighbor address is empty for gobgp-config %s", key) + continue + } + // erase neighbor. + cmdArs = append(cmdArs, "--", "flush-neighbor-policy", nbrIP) + } + } else { + // if oldGobgpConfig is nil, it means this is the first time to update the bgp policy + // so we need to set default action to reject + cmdArs = append(cmdArs, "--", "set-default-action", "reject") + } + + if newGobgpConfig != nil { + for _, neighbor := range newGobgpConfig.Spec.Neighbors { + klog.Infof("new bgp config neighbor %v", neighbor) + nbrIP := neighbor.Address + if len(nbrIP) == 0 { + klog.Warningf("neighbor address is empty for gobgp-config %s", key) + continue + } + cmdArs = append(cmdArs, "--", "set-neighbor-policy", nbrIP) + + // toAdvertise + advMode := neighbor.ToAdvertise.Allowed.Mode + var advPrefixes []string + if advMode == "all" { + advPrefixes = []string{"0.0.0.0/0 0..32"} + } else { + advPrefixes = neighbor.ToAdvertise.Allowed.Prefixes + } + quoted := make([]string, len(advPrefixes)) + + for i, p := range advPrefixes { + quoted[i] = fmt.Sprintf("\"%s\"", p) + } + cmdArs = append(cmdArs, "--", "add-prefix", "out", nbrIP, strings.Join(quoted, ",")) + + // toReceive + recvMode := neighbor.ToReceive.Allowed.Mode + var recvPrefixes []string + if recvMode == "all" { + recvPrefixes = []string{"0.0.0.0/0 0..32"} + } else { + recvPrefixes = neighbor.ToReceive.Allowed.Prefixes + } + quoted = make([]string, len(recvPrefixes)) + for i, p := range recvPrefixes { + quoted[i] = fmt.Sprintf("\"%s\"", p) + } + cmdArs = append(cmdArs, "--", "add-prefix", "in", nbrIP, strings.Join(quoted, ",")) + } + } else { + // if newGobgpConfig is nil, it means the bgp policy is deleted + // so we need to set default action to accept + cmdArs = append(cmdArs, "--", "set-default-action", "accept") + } + + if err := c.execCmd(pod, cmdArs); err != nil { + klog.Error(err) + return err + } + + return nil +} + +func (c *Controller) validateGobgpConfig(gobgpConfig *kubeovnv1.GobgpConfig) ([]*corev1.Pod, error) { + klog.Infof("gobgpConfignamespace: %s name: %s", gobgpConfig.Namespace, gobgpConfig.Name) + ber, err := c.bgpEdgeRouterLister.BgpEdgeRouters(gobgpConfig.Namespace).Get(gobgpConfig.Spec.BgpEdgeRouter) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil, fmt.Errorf("bgp edge router %s not found: %w", gobgpConfig.Spec.BgpEdgeRouter, err) + } + } + berNeighbors := ber.Spec.BGP.Neighbors + gobgpNeighbors := gobgpConfig.Spec.Neighbors + neighborFlag := false + for _, gNeighbor := range gobgpNeighbors { + if containsNeighbor(berNeighbors, gNeighbor.Address) { + neighborFlag = true + break + } + } + + if !neighborFlag { + err = fmt.Errorf("no matching neighbor found in BgpEdgeRouter %s for GobgpConfig %s", gobgpConfig.Spec.BgpEdgeRouter, gobgpConfig.Name) + klog.Error(err) + return nil, err + } + + deploy, err := c.berDeploymentsLister.Deployments(gobgpConfig.Namespace).Get(gobgpConfig.Spec.BgpEdgeRouter) + if err != nil { + gobgpConfig.Status.Ready = false + msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) + gobgpConfig.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "BgpEdgeRouterDeployNotFound", msg, gobgpConfig.Generation) + _, _ = c.updateGobgpConfigStatus(gobgpConfig) + klog.Error(err) + return nil, err + } + + ready := util.DeploymentIsReady(deploy) + if !ready { + gobgpConfig.Status.Ready = false + msg := fmt.Sprintf("Waiting for %s %s to be ready", deploy.Kind, deploy.Name) + gobgpConfig.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "BgpEdgeRouterNotEnabled", msg, gobgpConfig.Generation) + _, _ = c.updateGobgpConfigStatus(gobgpConfig) + readyErr := fmt.Sprintf("Kind %s, Deployment %s is not ready", deploy.Kind, deploy.Name) + klog.Error(readyErr) + return nil, fmt.Errorf("%s", readyErr) + } + // get the pods of the deployment to collect the pod IPs + podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) + if err != nil { + err = fmt.Errorf("failed to get pod selector of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return nil, err + } + + pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) + if err != nil { + err = fmt.Errorf("failed to list pods of deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return nil, err + } + + if ready { + gobgpConfig.Status.Ready = true + } + + return pods, nil +} + +func (c *Controller) initGobgpConfigStatus(gobgpConfig *kubeovnv1.GobgpConfig) (*kubeovnv1.GobgpConfig, error) { + var err error + gobgpConfig, err = c.updateGobgpConfigStatus(gobgpConfig) + return gobgpConfig, err +} + +func (c *Controller) updateGobgpConfigStatus(gobgpConfig *kubeovnv1.GobgpConfig) (*kubeovnv1.GobgpConfig, error) { + if len(gobgpConfig.Status.Conditions) == 0 { + gobgpConfig.Status.Conditions.SetCondition(kubeovnv1.Init, corev1.ConditionUnknown, "Processing", "", gobgpConfig.Generation) + } + + updatedGobgpConfig, err := c.config.KubeOvnClient.KubeovnV1().GobgpConfigs(gobgpConfig.Namespace). + UpdateStatus(context.Background(), gobgpConfig, metav1.UpdateOptions{}) + if err != nil { + err = fmt.Errorf("failed to update status of gobgp-config %s/%s: %w", gobgpConfig.Namespace, gobgpConfig.Name, err) + klog.Error(err) + return nil, err + } + + return updatedGobgpConfig, nil +} + +func (c *Controller) execGetCmd(pod *corev1.Pod, cmdArs []string) (string, error) { + cmd := fmt.Sprintf("bash /kube-ovn/update-bgp-policy.sh %s", strings.Join(cmdArs, " ")) + + klog.Infof("exec command : %s", cmd) + stdOutput, errOutput, err := util.ExecuteCommandInContainer(c.config.KubeClient, c.config.KubeRestConfig, pod.Namespace, pod.Name, "bgp-router-speaker", []string{"/bin/bash", "-c", cmd}...) + if err != nil { + if len(errOutput) > 0 { + klog.Errorf("failed to ExecuteCommandInContainer, errOutput: %v", errOutput) + } + if len(stdOutput) > 0 { + klog.Infof("failed to ExecuteCommandInContainer, stdOutput: %v", stdOutput) + } + klog.Error(err) + return "", err + } + + cmdSuccess := false + if len(stdOutput) > 0 { + klog.Infof("ExecuteCommandInContainer stdOutput: %v", stdOutput) + if strings.Contains(stdOutput, "Update bgp policy completed successfully") { + cmdSuccess = true + } + } + + if len(errOutput) > 0 && !cmdSuccess { + klog.Errorf("failed to ExecuteCommandInContainer errOutput: %v", errOutput) + return "", errors.New(errOutput) + } + + return stdOutput, nil +} + +func (c *Controller) execCmd(pod *corev1.Pod, cmdArs []string) error { + cmd := fmt.Sprintf("bash /kube-ovn/update-bgp-policy.sh --batch %s", strings.Join(cmdArs, " ")) + + klog.Infof("exec command : %s", cmd) + stdOutput, errOutput, err := util.ExecuteCommandInContainer(c.config.KubeClient, c.config.KubeRestConfig, pod.Namespace, pod.Name, "bgp-router-speaker", []string{"/bin/bash", "-c", cmd}...) + if err != nil { + if len(errOutput) > 0 { + klog.Errorf("failed to ExecuteCommandInContainer, errOutput: %v", errOutput) + } + if len(stdOutput) > 0 { + klog.Infof("failed to ExecuteCommandInContainer, stdOutput: %v", stdOutput) + } + klog.Error(err) + return err + } + + cmdSuccess := false + if len(stdOutput) > 0 { + klog.Infof("ExecuteCommandInContainer stdOutput: %v", stdOutput) + if strings.Contains(stdOutput, "Update bgp policy completed successfully") { + cmdSuccess = true + } + } + + if len(errOutput) > 0 && !cmdSuccess { + klog.Errorf("failed to ExecuteCommandInContainer errOutput: %v", errOutput) + return errors.New(errOutput) + } + + return nil +} + +func containsNeighbor(neighbors []string, address string) bool { + return slices.Contains(neighbors, address) +} + +func (c *Controller) resyncBgpPolicyRules() { + klog.Info("resync bgp edge router") + + // resync all bgp edge routers + gobgpConfigs, err := c.gobgpConfigLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list bgp edge routers: %v", err) + return + } + + for _, gobgpConfig := range gobgpConfigs { + if err := c.syncGobgpPolicy(gobgpConfig); err != nil { + klog.Errorf("failed to sync advertised routes for bgp edge router %s: %v", gobgpConfig.Name, err) + continue + } + klog.Infof("resync bgp edge router %s", gobgpConfig.Name) + } +} + +func (c *Controller) syncGobgpPolicy(gobgpConfig *kubeovnv1.GobgpConfig) error { + key := cache.MetaObjectToName(gobgpConfig).String() + + c.gobgpConfigKeyMutex.LockKey(key) + defer func() { _ = c.gobgpConfigKeyMutex.UnlockKey(key) }() + + if !gobgpConfig.DeletionTimestamp.IsZero() { + c.deleteGobgpConfigQueue.Add(key) + return nil + } + klog.Infof("reconciling bgp-edge-router %s", key) + // Deep copy because we might mutate Status below. + cachedGobgpConfig := gobgpConfig.DeepCopy() + + pods, err := c.validateGobgpConfig(cachedGobgpConfig) + if err != nil || pods == nil { + klog.Error(err) + return err + } + + for _, pod := range pods { + if len(pod.Status.PodIPs) == 0 { + continue + } + + // execGetBgpPolicy + // if + cmdArs := []string{"list-global-policy"} + output, err := c.execGetCmd(pod, cmdArs) + if err != nil { + klog.Error(err) + return err + } + + validate, err := c.validateSyncGobgpConfig(output, cachedGobgpConfig) + if !validate { + err = c.execUpdateBgpPolicy(key, pod, cachedGobgpConfig, cachedGobgpConfig) + if err != nil { + return err + } + } else if err != nil { + return err + } + + // klog.Infof("router pod %s/%s policy: %v", pod.Namespace, pod.Name, gobgpConfig) + } + + klog.Infof("finished sync bgp-edge-router %s advertised routes", key) + return nil +} + +func (c *Controller) validateSyncGobgpConfig(output string, gobgpConfig *kubeovnv1.GobgpConfig) (bool, error) { + klog.Infof("output: %s", output) + // Parse the output to verify that all neighbors, statements, and prefixes in gobgpConfig.Spec.Neighbors exist in the output. + for _, neighbor := range gobgpConfig.Spec.Neighbors { + nbrIP := neighbor.Address + if nbrIP == "" { + klog.Warningf("neighbor address is empty for gobgpConfig %s", gobgpConfig.Name) + return false, errors.New("neighbor address is empty") + } + + // Check Import policy statement and prefix + var inPrefixes, outPrefixes []string + inPrefixName := fmt.Sprintf("prefix-%s-in", nbrIP) + outPrefixName := fmt.Sprintf("prefix-%s-out", nbrIP) + + if neighbor.ToReceive.Allowed.Mode == "all" || neighbor.ToReceive.Allowed.Mode == "filtered" { + importPolicy := fmt.Sprintf("policy-%s-in", nbrIP) + importStmt := fmt.Sprintf("stmt-%s-in", nbrIP) + importPrefix := fmt.Sprintf("prefix-%s-in", nbrIP) + if !strings.Contains(output, importPolicy) || + !strings.Contains(output, importStmt) || + !strings.Contains(output, importPrefix) { + klog.Warningf("missing import policy/statement/prefix for neighbor %s", nbrIP) + return false, nil + } + } + + if neighbor.ToAdvertise.Allowed.Mode == "all" || neighbor.ToAdvertise.Allowed.Mode == "filtered" { + // Check Export policy statement and prefix + exportPolicy := fmt.Sprintf("policy-%s-out", nbrIP) + exportStmt := fmt.Sprintf("stmt-%s-out", nbrIP) + exportPrefix := fmt.Sprintf("prefix-%s-out", nbrIP) + if !strings.Contains(output, exportPolicy) || + !strings.Contains(output, exportStmt) || + !strings.Contains(output, exportPrefix) { + klog.Warningf("missing export policy/statement/prefix for neighbor %s", nbrIP) + return false, nil + } + } + + // Parse global policy prefix lines for this neighbor + // Parse only the lines after "=== Global Policy Prefix ===" + + lines := strings.Split(output, "\n") + startIdx := 0 + for i, line := range lines { + if strings.Contains(line, "=== Policy Prefix ===") { + startIdx = i + 2 + break + } + } + + var inDir bool + if strings.HasPrefix(lines[startIdx], inPrefixName) { + inDir = true + } else { + inDir = false + } + + for _, line := range lines[startIdx:] { + line = strings.TrimSpace(line) + // klog.Infof("line: %v, startIndex: %d", line, startIdx) + + if strings.Contains(line, "Update bgp policy completed successfully") { + break + } + + if strings.HasPrefix(line, inPrefixName) || inDir { + // klog.Infof("line: %v", line) + prefixPart := strings.TrimPrefix(line, inPrefixName+" ") + // klog.Infof("prefixPart: %v", prefixPart) + inPrefixes = append(inPrefixes, prefixPart) + if strings.HasPrefix(lines[startIdx+1], outPrefixName) { + inDir = false + } + } else if strings.HasPrefix(line, outPrefixName) || !inDir { + klog.Infof("line: %v", line) + prefixPart := strings.TrimPrefix(line, outPrefixName+" ") + klog.Infof("prefixPart: %v", prefixPart) + outPrefixes = append(outPrefixes, prefixPart) + if strings.HasPrefix(lines[startIdx+1], inPrefixName) { + inDir = true + } + } + } + klog.Warningf("inPrefixes: %v, outPrefixes: %v", inPrefixes, outPrefixes) + + // Check advertised prefixes (out) + if neighbor.ToAdvertise.Allowed.Mode == "filtered" { + for _, p := range neighbor.ToAdvertise.Allowed.Prefixes { + found := false + for _, out := range outPrefixes { + if strings.Contains(out, p) { + found = true + break + } + } + if !found { + klog.Warningf("missing advertised prefix %s for neighbor %s", p, nbrIP) + return false, nil + } + } + } else { + // Mode "all": should contain 0.0.0.0/0 0..32 + found := false + for _, out := range outPrefixes { + if strings.Contains(out, "0.0.0.0/0") && strings.Contains(out, "0..32") { + found = true + break + } + } + if !found { + klog.Warningf("missing advertised prefix 0.0.0.0/0 0..32 for neighbor %s", nbrIP) + return false, nil + } + } + + // Check received prefixes (in) + if neighbor.ToReceive.Allowed.Mode == "filtered" { + for _, p := range neighbor.ToReceive.Allowed.Prefixes { + found := false + for _, in := range inPrefixes { + if strings.Contains(in, p) { + found = true + break + } + } + if !found { + klog.Warningf("missing received prefix %s for neighbor %s", p, nbrIP) + return false, nil + } + } + } else { + // Mode "all": should contain 0.0.0.0/0 0..32 + found := false + for _, in := range inPrefixes { + if strings.Contains(in, "0.0.0.0/0") && strings.Contains(in, "0..32") { + found = true + break + } + } + if !found { + klog.Warningf("missing received prefix 0.0.0.0/0 0..32 for neighbor %s", nbrIP) + return false, nil + } + } + } + klog.Infof("Sync is not needed.") + return true, nil +} diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 8197c315eaa..2c1bb62c59c 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -235,6 +235,10 @@ func (c *Controller) enqueueAddPod(obj any) { if err = c.handlePodEventForVpcEgressGateway(p); err != nil { klog.Errorf("failed to handle pod event for vpc egress gateway: %v", err) } + + if err = c.handlePodEventForBgpEdgeRouter(p); err != nil { + klog.Errorf("failed to handle pod event for vpc egress gateway: %v", err) + } } func (c *Controller) enqueueDeletePod(obj any) { @@ -376,6 +380,9 @@ func (c *Controller) enqueueUpdatePod(oldObj, newObj any) { klog.Errorf("failed to handle pod event for vpc egress gateway: %v", err) } + if err = c.handlePodEventForBgpEdgeRouter(newPod); err != nil { + klog.Errorf("failed to handle pod event for bgp edge router: %v", err) + } // do not delete statefulset pod unless ownerReferences is deleted if isStateful && isStatefulSetPodToDel(c.config.KubeClient, newPod, statefulSetName, statefulSetUID) { go func() { diff --git a/pkg/controller/vpc_egress_gateway.go b/pkg/controller/vpc_egress_gateway.go index 6612f5ff57f..20a8e99ff06 100644 --- a/pkg/controller/vpc_egress_gateway.go +++ b/pkg/controller/vpc_egress_gateway.go @@ -2,6 +2,7 @@ package controller import ( "context" + "errors" "fmt" "maps" "reflect" @@ -240,9 +241,13 @@ func (c *Controller) updateVpcEgressGatewayStatus(gw *kubeovnv1.VpcEgressGateway // create or update vpc egress gateway workload func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGateway, vpc *kubeovnv1.Vpc, bfdIP, bfdIPv4, bfdIPv6 string) (string, set.Set[string], set.Set[string], *appsv1.Deployment, error) { image := c.config.Image + bgpImage := c.config.Image if gw.Spec.Image != "" { image = gw.Spec.Image } + if gw.Spec.BGP.Image != "" { + bgpImage = gw.Spec.BGP.Image + } if image == "" { err := fmt.Errorf("no image specified for vpc egress gateway %s/%s", gw.Namespace, gw.Name) klog.Error(err) @@ -362,7 +367,7 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa annotations[util.LogicalSwitchAnnotation] = intSubnet.Name if len(gw.Spec.InternalIPs) != 0 { // set internal IPs - annotations[util.IPPoolAnnotation] = strings.Join(gw.Spec.InternalIPs, ";") + annotations[util.IPPoolAnnotation] = strings.Join(gw.Spec.InternalIPs, ";") // is it ; okay? } if len(gw.Spec.ExternalIPs) != 0 { // set external IPs @@ -476,6 +481,17 @@ func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGa deploy.Spec.Template.Spec.Containers[0] = container } + // bgp sidecar container logic + if gw.Spec.BGP.Enabled { + // run BGP in the gateway container + bgpContainer, err := vpcEgressGatewayContainerBGP(bgpImage, gw.Name, &gw.Spec.BGP) + if err != nil { + klog.Errorf("failed to create a BGP speaker container for gateway %s: %v", gw.Name, err) + return "", nil, nil, nil, err + } + deploy.Spec.Template.Spec.Containers = append(deploy.Spec.Template.Spec.Containers, *bgpContainer) + } + // generate hash for the workload to determine whether to update the existing workload or not hash, err := util.Sha256HashObject(deploy) if err != nil { @@ -1036,3 +1052,94 @@ func (c *Controller) handlePodEventForVpcEgressGateway(pod *corev1.Pod) error { } return nil } + +func vpcEgressGatewayContainerBGP(speakerImage, gatewayName string, speakerParams *kubeovnv1.VpcEgressGatewayBGPConfig) (*corev1.Container, error) { + if speakerImage == "" { + return nil, errors.New("BGP speaker image must be specified") + } + if speakerParams == nil { + return nil, errors.New("BGP config must not be nil") + } + if speakerParams.ASN == 0 { + return nil, errors.New("ASN not set, but must be non-zero value") + } + if speakerParams.RemoteASN == 0 { + return nil, errors.New("remote ASN not set, but must be non-zero value") + } + if len(speakerParams.Neighbors) == 0 { + return nil, errors.New("no BGP neighbors specified") + } + + args := []string{} + if speakerParams.EdgeRouterMode { + args = append(args, "--edge-router-mode=true") + } + if speakerParams.RouterID != "" { + args = append(args, "--router-id="+speakerParams.RouterID) + } + if speakerParams.Password != "" { + args = append(args, "--auth-password="+speakerParams.Password) + } + if speakerParams.EnableGracefulRestart { + args = append(args, "--graceful-restart") + } + if speakerParams.HoldTime != (metav1.Duration{}) { + args = append(args, "--holdtime="+speakerParams.HoldTime.Duration.String()) + } + + args = append(args, fmt.Sprintf("--cluster-as=%d", speakerParams.ASN)) + args = append(args, fmt.Sprintf("--neighbor-as=%d", speakerParams.RemoteASN)) + + var neighIPv4, neighIPv6 []string + for _, neighbor := range speakerParams.Neighbors { + switch util.CheckProtocol(neighbor) { + case kubeovnv1.ProtocolIPv4: + neighIPv4 = append(neighIPv4, neighbor) + case kubeovnv1.ProtocolIPv6: + neighIPv6 = append(neighIPv6, neighbor) + default: + return nil, fmt.Errorf("unsupported protocol for peer %s", neighbor) + } + } + if len(neighIPv4) > 0 { + args = append(args, "--neighbor-address="+strings.Join(neighIPv4, ",")) + } + if len(neighIPv6) > 0 { + args = append(args, "--neighbor-ipv6-address="+strings.Join(neighIPv6, ",")) + } + + args = append(args, speakerParams.ExtraArgs...) + + container := &corev1.Container{ + Name: "vpc-egress-gw-speaker", + Image: speakerImage, + Command: []string{"/kube-ovn/kube-ovn-speaker"}, + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + { + Name: "EGRESS_GATEWAY_NAME", + Value: gatewayName, + }, + { + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + }, + Args: args, + // bgp need to add/remove fib, it needs root user + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](0), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_BIND_SERVICE", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, + } + + return container, nil +} diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go index 1b6fe901470..4038692ae81 100644 --- a/pkg/ovs/ovn.go +++ b/pkg/ovs/ovn.go @@ -46,8 +46,11 @@ const ( OVSDBWaitTimeout = 0 - ExternalIDVendor = "vendor" - ExternalIDVpcEgressGateway = "vpc-egress-gateway" + ExternalIDVendor = "vendor" + ExternalIDVpcEgressGateway = "vpc-egress-gateway" + ExternalIDBgpEdgeRouter = "bgp-edge-router" + ExternalIDBgpEdgeRouterAdvertisement = "bgp-edge-router-advertisement" + ExternalIDGobgpConfig = "gobgp-config" ) // NewLegacyClient init a legacy ovn client diff --git a/pkg/speaker/config.go b/pkg/speaker/config.go index a07d21c97ee..07b63d99295 100644 --- a/pkg/speaker/config.go +++ b/pkg/speaker/config.go @@ -2,6 +2,7 @@ package speaker import ( "context" + "encoding/json" "errors" "flag" "fmt" @@ -65,8 +66,10 @@ type Configuration struct { KubeClient kubernetes.Interface KubeOvnClient clientset.Interface - PprofPort int32 - LogPerm string + PprofPort int32 + LogPerm string + EdgeRouterMode bool + RouteServerClient bool } func ParseFlags() (*Configuration, error) { @@ -94,6 +97,8 @@ func ParseFlags() (*Configuration, error) { argNatGwMode = pflag.BoolP("nat-gw-mode", "", false, "Make the BGP speaker announce EIPs from inside a NAT gateway, Pod IP/Service/Subnet announcements will be disabled") argEnableMetrics = pflag.BoolP("enable-metrics", "", true, "Whether to support metrics query") argLogPerm = pflag.String("log-perm", "640", "The permission for the log file") + argEdgeRouterMode = pflag.BoolP("edge-router-mode", "", false, "Make the BGP speaker announce inside subnet and get routes from the outside, work as edge router") + argRouteServerClient = pflag.BoolP("route-server-client", "", false, "Make the BGP speaker policy route, work as route server client") ) klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) klog.InitFlags(klogFlags) @@ -161,6 +166,8 @@ func ParseFlags() (*Configuration, error) { NatGwMode: *argNatGwMode, EnableMetrics: *argEnableMetrics, LogPerm: *argLogPerm, + EdgeRouterMode: *argEdgeRouterMode, + RouteServerClient: *argRouteServerClient, } if *argNeighborAddress != "" { @@ -181,11 +188,19 @@ func ParseFlags() (*Configuration, error) { } if config.RouterID == "" { - if podIPv4 != "" { - config.RouterID = podIPv4 - } else { - config.RouterID = podIPv6 + externalIP, err := GetExternalIP() + if err != nil || externalIP == "" { + klog.Warningf("failed to get external IP: %v", err) + return nil, err } + config.RouterID = externalIP + klog.Infof("using external IP %s as router ID", config.RouterID) + + // if podIPv4 != "" { + // config.RouterID = podIPv4 + // } else { + // config.RouterID = podIPv6 + // } if config.RouterID == "" { return nil, errors.New("no router id or POD_IPS") } @@ -365,3 +380,32 @@ func (config *Configuration) initBgpServer() error { config.BgpServer = s return nil } + +func GetExternalIP() (string, error) { + raw := os.Getenv("MULTI_NET_STATUS") + if raw == "" { + return "", errors.New("MULTI_NET_STATUS annotation is empty") + } + + type networkStatusEntry struct { + Name string `json:"name"` + Interface string `json:"interface"` + IPs []string `json:"ips"` + Default bool `json:"default"` + DNS struct{} `json:"dns"` + } + + var entries []networkStatusEntry + if err := json.Unmarshal([]byte(raw), &entries); err != nil { + return "", err + } + + for _, e := range entries { + // search for CNI network name is not "kube-ovn" + if e.Name != "kube-ovn" && len(e.IPs) > 0 { + return e.IPs[0], nil + } + } + + return "", errors.New("non–kube-ovn interface not found") +} diff --git a/pkg/speaker/controller.go b/pkg/speaker/controller.go index 294e124b34e..bc520fd177b 100644 --- a/pkg/speaker/controller.go +++ b/pkg/speaker/controller.go @@ -44,6 +44,8 @@ type Controller struct { informerFactory kubeinformers.SharedInformerFactory kubeovnInformerFactory kubeovninformer.SharedInformerFactory recorder record.EventRecorder + + routerSyncer *RouteSyncer } func NewController(config *Configuration) *Controller { @@ -69,6 +71,8 @@ func NewController(config *Configuration) *Controller { eipInformer := kubeovnInformerFactory.Kubeovn().V1().IptablesEIPs() natgatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcNatGateways() + routerSyncer := NewRouteSyncer(time.Second*60, config) + controller := &Controller{ config: config, @@ -86,6 +90,8 @@ func NewController(config *Configuration) *Controller { informerFactory: informerFactory, kubeovnInformerFactory: kubeovnInformerFactory, recorder: recorder, + + routerSyncer: routerSyncer, } return controller @@ -102,7 +108,13 @@ func (c *Controller) Run(stopCh <-chan struct{}) { } klog.Info("Started workers") - go wait.Until(c.Reconcile, 5*time.Second, stopCh) + if c.config.EdgeRouterMode { + // Start route syncer work + // run every c.routerSyncer.injectedRoutesSyncPeriod + c.routerSyncer.Run(stopCh) + } else { + go wait.Until(c.Reconcile, 5*time.Second, stopCh) + } <-stopCh klog.Info("Shutting down workers") diff --git a/pkg/speaker/route_syncer.go b/pkg/speaker/route_syncer.go new file mode 100644 index 00000000000..bf7759bf68f --- /dev/null +++ b/pkg/speaker/route_syncer.go @@ -0,0 +1,325 @@ +package speaker + +import ( + "context" + "errors" + "fmt" + "net" + "strconv" + "sync" + "time" + + "k8s.io/klog/v2" + + gobgpapi "github.com/osrg/gobgp/v3/api" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netlink/nl" +) + +// RTPROT_BGP is the protocol number for BGP routes in the kernel's routing table. +// Get from https://github.com/torvalds/linux/blob/master/include/uapi/linux/rtnetlink.h#L312 +const ( + rtprotKernel = 2 // Route installed by kernel + rtprotBgp = 186 // BGP Routes +) + +// RouteSyncer is responsible for watching BGP events and handling them. +// It listens for BGP updates, injects routes into the local route table, and syncs them to the kernel's routing table. +// RouteSyncer is a struct that holds all of the information needed for syncing routes to the kernel's routing table +type RouteSyncer struct { + config *Configuration + routeTableStateMap map[string]*netlink.Route + injectedRoutesSyncPeriod time.Duration + mutex sync.Mutex + routeReplacer func(route *netlink.Route) error +} + +// NewRouteSyncer creates a new routeSyncer that, when run, +// will sync routes kept in its local state table every syncPeriod +func NewRouteSyncer(syncPeriod time.Duration, config *Configuration) *RouteSyncer { + rs := RouteSyncer{} + rs.config = config + rs.routeTableStateMap = make(map[string]*netlink.Route) + rs.injectedRoutesSyncPeriod = syncPeriod + rs.mutex = sync.Mutex{} + // We substitute the RouteReplace function here so that we can easily monkey patch it in our unit tests + rs.routeReplacer = netlink.RouteReplace + // Start to watch updates from the GoBGP server + rs.watchBgpUpdates() + + return &rs +} + +// update path cache when a new path is added or an existing path is updated from gobgp server +func (rs *RouteSyncer) watchBgpUpdates() { + pathWatch := func(r *gobgpapi.WatchEventResponse) { + if table := r.GetTable(); table != nil { + for _, path := range table.Paths { + if path.Family.Afi == gobgpapi.Family_AFI_IP || + path.Family.Afi == gobgpapi.Family_AFI_IP6 || + path.Family.Safi == gobgpapi.Family_SAFI_UNICAST { + if path.NeighborIp == "" { + return + } + klog.Infof("Processing bgp route advertisement from peer: %s", path.NeighborIp) + if err := rs.injectRoute(path); err != nil { + klog.Errorf("failed to inject routes due to: %v", err) + } + } + } + } + } + err := rs.config.BgpServer.WatchEvent(context.Background(), &gobgpapi.WatchEventRequest{ + Table: &gobgpapi.WatchEventRequest_Table{ + Filters: []*gobgpapi.WatchEventRequest_Table_Filter{ + { + Type: gobgpapi.WatchEventRequest_Table_Filter_BEST, + }, + }, + }, + }, pathWatch) + if err != nil { + klog.Errorf("failed to register monitor global routing table callback due to: %v", err) + } +} + +// addInjectedRoute adds a route to the route map that is regularly synced to the kernel's routing table +func (rs *RouteSyncer) AddInjectedRoute(dst *net.IPNet, route *netlink.Route) { + rs.mutex.Lock() + defer rs.mutex.Unlock() + klog.Infof("Adding route for destination: %s", dst) + rs.routeTableStateMap[dst.String()] = route +} + +// delInjectedRoute delete a route from the route map that is regularly synced to the kernel's routing table +func (rs *RouteSyncer) DelInjectedRoute(dst *net.IPNet) { + rs.mutex.Lock() + defer rs.mutex.Unlock() + if _, ok := rs.routeTableStateMap[dst.String()]; ok { + klog.Infof("Removing route for destination: %s", dst) + delete(rs.routeTableStateMap, dst.String()) + } +} + +// syncLocalRouteTable iterates over the local route state map and syncs all routes to the kernel's routing table +func (rs *RouteSyncer) SyncLocalRouteTable() (*netlink.Route, error) { + rs.mutex.Lock() + defer rs.mutex.Unlock() + klog.Infof("Running local route table synchronization") + for _, route := range rs.routeTableStateMap { + klog.Infof("Syncing route: %s -> %s via %s", route.Src, route.Dst, route.Gw) + err := rs.routeReplacer(route) + if err != nil { + return route, err + } + } + return nil, nil +} + +// run starts a goroutine that calls syncLocalRouteTable on interval injectedRoutesSyncPeriod +func (rs *RouteSyncer) Run(stopCh <-chan struct{}) { + // Start route synchronization routine + go func(stopCh <-chan struct{}) { + t := time.NewTicker(rs.injectedRoutesSyncPeriod) + defer t.Stop() + for { + select { + case <-t.C: + _, err := rs.SyncLocalRouteTable() + if err != nil { + klog.Errorf("route could not be replaced due to: %v", err) + } + case <-stopCh: + klog.Infof("Shutting down local route synchronization") + return + } + } + }(stopCh) +} + +// Delete the route from the kernel's routing table, and remove it from the local state map +func (rs *RouteSyncer) injectRoute(path *gobgpapi.Path) error { + klog.Infof("injectRoute Path Looks Like: %s", path.String()) + var route *netlink.Route + var link netlink.Link + + // TODO: This is a hardcoded link name, which is not ideal. + // Find the link by name, which is hardcoded to "net1" in this example. + // In a real-world scenario, you might want to make this configurable or discover it dynamically + // based on your network setup. + link, linkErr := netlink.LinkByName("net1") + if linkErr != nil { + klog.Fatalf("Failed to find interface: %v", linkErr) + return linkErr + } + + dst, nextHop, pathErr := rs.ParsePath(path) + if pathErr != nil { + return pathErr + } + + // If path is same with external interface subnet do not add + if err := rs.checkExistingKernelRoute(dst); err != nil { + // If it's already a kernel route, just log and skip + klog.Infof("Skipping BGP route injection for %s: %v", dst.String(), err) + return nil + } + + // If the path we've received from GoBGP is a withdrawal, we should clean up any lingering routes that may exist + // on the host (rather than creating a new one or updating an existing one), and then return. + if path.IsWithdraw { + klog.Infof("Removing route: '%s via %s' from peer in the routing table", dst, nextHop) + + // Delete route from state map so that it doesn't get re-synced after deletion + rs.DelInjectedRoute(dst) + return rs.DeleteByDestination(dst) + } + + selectBestAddress := func(addrs []netlink.Addr) net.IP { + var bestAddr net.IP + bestScope := 1000 // Set a high initial scope value + for _, addr := range addrs { + // Scope values are defined as follows: + // RT_SCOPE_UNIVERSE(0) > RT_SCOPE_SITE(200) > RT_SCOPE_LINK(253) > RT_SCOPE_HOST(254) + if addr.Scope < bestScope { + bestScope = addr.Scope + bestAddr = addr.IP + } + } + // If no best address was found, return the first address in the list + if bestAddr == nil && len(addrs) > 0 { + bestAddr = addrs[0].IP + } + return bestAddr + } + // Use external interface for destination routing + // Determine the address family based on the destination IP + family := netlink.FAMILY_V4 + if dst.IP.To4() == nil { + family = netlink.FAMILY_V6 + } + // Get the list of addresses for the link + addrs, addrsErr := netlink.AddrList(link, family) + if addrsErr != nil { + klog.Errorf("failed to get addresses for interface %s: %s", + link.Attrs().Name, addrsErr) + return addrsErr + } + if len(addrs) == 0 { + klog.Errorf("no addresses found on interface %s", + link.Attrs().Name) + return errors.New("no addresses found on interface") + } + // If we have multiple addresses, we need to select the best one + bestIPForFamily := selectBestAddress(addrs) + + route = &netlink.Route{ + LinkIndex: link.Attrs().Index, + Src: bestIPForFamily, + Dst: dst, + Protocol: rtprotBgp, + Gw: nextHop, + } + + // We have our route configured, let's add it to the host's routing table + klog.Infof("Inject route: '%s via %s' from peer to routing table", dst, nextHop) + rs.AddInjectedRoute(dst, route) + // Immediately sync the local route table regardless of timer + _, syncLocalRouteTableErr := rs.SyncLocalRouteTable() + return syncLocalRouteTableErr +} + +// ParseNextHop takes in a GoBGP Path and parses out the destination's next hop from its attributes. If it +// can't parse a next hop IP from the GoBGP Path, it returns an error. +func (rs *RouteSyncer) ParseNextHop(path *gobgpapi.Path) (net.IP, error) { + for _, pAttr := range path.GetPattrs() { + unmarshalNew, err := pAttr.UnmarshalNew() + if err != nil { + return nil, fmt.Errorf("failed to unmarshal path attribute: %w", err) + } + switch t := unmarshalNew.(type) { + case *gobgpapi.NextHopAttribute: + // This is the primary way that we receive NextHops and happens when both the client and the server exchange + // next hops on the same IP family that they negotiated BGP on + nextHopIP := net.ParseIP(t.NextHop) + if nextHopIP != nil && (nextHopIP.To4() != nil || nextHopIP.To16() != nil) { + return nextHopIP, nil + } + return nil, fmt.Errorf("invalid nextHop address: %s", t.NextHop) + case *gobgpapi.MpReachNLRIAttribute: + // in the case where the server and the client are exchanging next-hops that don't relate to their primary + // IP family, we get MpReachNLRIAttribute instead of NextHopAttributes + // TODO: here we only take the first next hop, at some point in the future it would probably be best to + // consider multiple next hops + nextHopIP := net.ParseIP(t.NextHops[0]) + if nextHopIP != nil && (nextHopIP.To4() != nil || nextHopIP.To16() != nil) { + return nextHopIP, nil + } + return nil, fmt.Errorf("invalid nextHop address: %s", t.NextHops[0]) + } + } + return nil, fmt.Errorf("could not parse next hop received from GoBGP for path: %s", path) +} + +// ParsePath takes in a GoBGP Path and parses out the destination subnet and the next hop from its attributes. +// If successful, it will return the destination of the BGP path as a subnet form and the next hop. If it +// can't parse the destination or the next hop IP, it returns an error. +func (rs *RouteSyncer) ParsePath(path *gobgpapi.Path) (*net.IPNet, net.IP, error) { + nextHop, err := rs.ParseNextHop(path) + if err != nil { + return nil, nil, err + } + + nlri := path.GetNlri() + var prefix gobgpapi.IPAddressPrefix + err = nlri.UnmarshalTo(&prefix) + if err != nil { + return nil, nil, errors.New("invalid nlri in advertised path") + } + dstSubnet, err := netlink.ParseIPNet(prefix.Prefix + "/" + strconv.FormatUint(uint64(prefix.PrefixLen), 10)) + if err != nil { + return nil, nil, errors.New("couldn't parse IP subnet from nlri advertised path") + } + return dstSubnet, nextHop, nil +} + +// DeleteByDestination attempts to safely find all routes based upon its destination subnet and delete them +func (rs *RouteSyncer) DeleteByDestination(destinationSubnet *net.IPNet) error { + routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, &netlink.Route{ + Dst: destinationSubnet, Protocol: rtprotBgp, + }, netlink.RT_FILTER_DST|netlink.RT_FILTER_PROTOCOL) + if err != nil { + return fmt.Errorf("failed to get routes from netlink: %w", err) + } + for i, r := range routes { + klog.Infof("Found route to remove: %s", r.String()) + if err = netlink.RouteDel(&routes[i]); err != nil { + return fmt.Errorf("failed to remove route due to %w", err) + } + } + return nil +} + +// checks if a route with the same destination already exists +// with protocol "kernel" and returns an error if found +func (rs *RouteSyncer) checkExistingKernelRoute(dst *net.IPNet) error { + // Get existing routes for the destination + routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, &netlink.Route{ + Dst: dst, + Protocol: rtprotKernel, + }, netlink.RT_FILTER_DST|netlink.RT_FILTER_PROTOCOL) + if err != nil { + klog.Errorf("Failed to get existing routes for destination %s: %v", dst.String(), err) + return nil // Don't block route injection on query failure + } + + // If we found any kernel routes with the same destination, skip injection + if len(routes) > 0 { + for _, existingRoute := range routes { + klog.Infof("Found existing kernel route: %s", existingRoute.String()) + } + return fmt.Errorf("destination %s already exists as kernel protocol route", dst.String()) + } + + return nil +} diff --git a/pkg/util/const.go b/pkg/util/const.go index 4cf39564fea..ee5655cefab 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -113,8 +113,11 @@ const ( NetworkPolicyLogAnnotation = "ovn.kubernetes.io/enable_log" ACLActionsLogAnnotation = "ovn.kubernetes.io/log_acl_actions" - VpcEgressGatewayLabel = "ovn.kubernetes.io/vpc-egress-gateway" - GenerateHashAnnotation = "ovn.kubernetes.io/generate-hash" + VpcEgressGatewayLabel = "ovn.kubernetes.io/vpc-egress-gateway" + BgpEdgeRouterLabel = "ovn.kubernetes.io/bgp-edge-router" + BgpEdgeRouterAdvertisementLabel = "ovn.kubernetes.io/bgp-edge-router-advertisement" + GobgpConfigLabel = "ovn.kubernetes.io/gobgp-config" + GenerateHashAnnotation = "ovn.kubernetes.io/generate-hash" VpcLastName = "ovn.kubernetes.io/last_vpc_name" VpcLastPolicies = "ovn.kubernetes.io/last_policies" diff --git a/test/e2e/edge-router/e2e_test.go b/test/e2e/edge-router/e2e_test.go new file mode 100644 index 00000000000..e3406b53c3e --- /dev/null +++ b/test/e2e/edge-router/e2e_test.go @@ -0,0 +1,538 @@ +package multus + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "maps" + "math/rand/v2" + "net" + "reflect" + + // "slices" + "strconv" + "strings" + "testing" + + // "time" + + dockernetwork "github.com/docker/docker/api/types/network" + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/component-base/logs" + "k8s.io/klog/v2" + commontest "k8s.io/kubernetes/test/e2e/common" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/test/e2e/framework" + "github.com/kubeovn/kube-ovn/test/e2e/framework/docker" + + // "github.com/kubeovn/kube-ovn/test/e2e/framework/iproute" + "github.com/kubeovn/kube-ovn/test/e2e/framework/kind" +) + +type GlobalConfig struct { + ASN uint32 `json:"asn"` + RouterID string `json:"router_id"` + ListenPort int `json:"listen_port"` + ListenAddresses []string `json:"listen_addresses"` +} + +type NeighborMessages struct { + Received struct { + Notification uint64 `json:"notification,omitempty"` + Open uint64 `json:"open,omitempty"` + Update uint64 `json:"update,omitempty"` + Keepalive uint64 `json:"keepalive,omitempty"` + Total uint64 `json:"total,omitempty"` + } `json:"received"` + Sent struct { + Open uint64 `json:"open,omitempty"` + Update uint64 `json:"update,omitempty"` + Keepalive uint64 `json:"keepalive,omitempty"` + Total uint64 `json:"total,omitempty"` + } `json:"sent"` +} + +type NeighborConfig struct { + LocalASN uint32 `json:"local_asn"` + NeighborAddress string `json:"neighbor_address"` + PeerASN uint32 `json:"peer_asn"` + Type int `json:"type"` +} + +type NeighborState struct { + LocalASN uint32 `json:"local_asn"` + Messages NeighborMessages `json:"messages"` + NeighborAddress string `json:"neighbor_address"` + PeerASN uint32 `json:"peer_asn"` + Type int `json:"type"` + SessionState int `json:"session_state"` + RouterID string `json:"router_id,omitempty"` +} + +type NeighborEntry struct { + Conf NeighborConfig `json:"conf"` + State NeighborState `json:"state"` +} + +func init() { + klog.SetOutput(ginkgo.GinkgoWriter) + + // Register flags. + config.CopyFlags(config.Flags, flag.CommandLine) + k8sframework.RegisterCommonFlags(flag.CommandLine) + k8sframework.RegisterClusterFlags(flag.CommandLine) +} + +func TestE2E(t *testing.T) { + k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) + + logs.InitLogs() + defer logs.FlushLogs() + klog.EnableContextualLogging(true) + + gomega.RegisterFailHandler(k8sframework.Fail) + + // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins + suiteConfig, reporterConfig := k8sframework.CreateGinkgoConfig() + klog.Infof("Starting e2e run %q on Ginkgo node %d", k8sframework.RunID, suiteConfig.ParallelProcess) + ginkgo.RunSpecs(t, "Kube-OVN e2e suite", suiteConfig, reporterConfig) +} + +const ( + kindNetwork = "kind" + +// controlPlaneLabel = "node-role.kubernetes.io/control-plane" +) + +var clusterName string + +var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { + // Reference common test to make the import valid. + commontest.CurrentSuite = commontest.E2E + + cs, err := k8sframework.LoadClientset() + framework.ExpectNoError(err) + + ginkgo.By("Getting k8s nodes") + k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) + framework.ExpectNoError(err) + + var ok bool + if clusterName, ok = kind.IsKindProvided(k8sNodes.Items[0].Spec.ProviderID); !ok { + ginkgo.Fail("vpc-egress-gateway spec only runs on kind clusters") + } + + return []byte(clusterName) +}, func(data []byte) { + clusterName = string(data) +}) + +var _ = framework.Describe("[group:ber]", func() { + f := framework.NewDefaultFramework("ber") + + var vpcClient *framework.VpcClient + var subnetClient *framework.SubnetClient + var nadClient *framework.NetworkAttachmentDefinitionClient + var nadName, externalSubnetName, namespaceName string + var schedulableNodes []corev1.Node + + var replicas int32 + ginkgo.BeforeEach(func() { + namespaceName = f.Namespace.Name + nadName = "nad-" + framework.RandomSuffix() + externalSubnetName = "ext-" + framework.RandomSuffix() + vpcClient = f.VpcClient() + subnetClient = f.SubnetClient() + nadClient = f.NetworkAttachmentDefinitionClient() + + nodeList, err := e2enode.GetReadyNodesIncludingTainted(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + framework.ExpectNotEmpty(nodeList.Items) + + nodeList, err = e2enode.GetReadySchedulableNodes(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + framework.ExpectNotEmpty(nodeList.Items) + schedulableNodes = nodeList.Items + + replicas = min(int32(len(schedulableNodes)), 3) + }) + + framework.ConformanceIt("should be able to create edge-router with macvlan", func() { + provider := fmt.Sprintf("%s.%s", nadName, namespaceName) + + ginkgo.By("Creating network attachment definition " + nadName) + nad := framework.MakeMacvlanNetworkAttachmentDefinition(nadName, namespaceName, "eth0", "bridge", provider, nil) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting network attachment definition " + nadName) + nadClient.Delete(nadName) + }) + nad = nadClient.Create(nad) + framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + vpcName := "vpc-" + framework.RandomSuffix() + vpcCidr := framework.RandomCIDR(f.ClusterIPFamily) + bfdIP := framework.RandomIPs(vpcCidr, ";", 1) + ginkgo.By("Creating vpc " + vpcName + ", enabling BFD Port with IP " + bfdIP + " for VPC " + vpcName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting vpc " + vpcName) + vpcClient.DeleteSync(vpcName) + }) + vpc := &apiv1.Vpc{ + ObjectMeta: metav1.ObjectMeta{ + Name: vpcName, + }, + Spec: apiv1.VpcSpec{ + BFDPort: &apiv1.BFDPort{ + Enabled: true, + IP: bfdIP, + }, + }, + } + vpc = vpcClient.CreateSync(vpc) + framework.ExpectNotEmpty(vpc.Status.BFDPort.Name) + + internalSubnetName := "int-" + framework.RandomSuffix() + ginkgo.By("Creating internal subnet " + internalSubnetName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting internal subnet " + internalSubnetName) + subnetClient.DeleteSync(internalSubnetName) + }) + cidr := framework.RandomCIDR(f.ClusterIPFamily) + internalSubnet := framework.MakeSubnet(internalSubnetName, "", cidr, "", vpcName, "", nil, nil, nil) + _ = subnetClient.CreateSync(internalSubnet) + + ginkgo.By("Getting docker network " + kindNetwork) + network, err := docker.NetworkInspect(kindNetwork) + framework.ExpectNoError(err, "getting docker network "+kindNetwork) + + externalSubnet := generateSubnetFromDockerNetwork(externalSubnetName, network, f.HasIPv4(), f.HasIPv6()) + externalSubnet.Spec.Provider = provider + + ginkgo.By("Creating macvlan subnet " + externalSubnetName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting external subnet " + externalSubnetName) + subnetClient.DeleteSync(externalSubnetName) + }) + _ = subnetClient.CreateSync(externalSubnet) + + berTest(f, true, provider, nadName, vpcName, internalSubnetName, externalSubnetName, replicas) + }) +}) + +func generateSubnetFromDockerNetwork(subnetName string, network *dockernetwork.Inspect, ipv4, ipv6 bool) *apiv1.Subnet { + ginkgo.GinkgoHelper() + + ginkgo.By("Generating subnet configuration from docker network " + network.Name) + var cidrV4, cidrV6, gatewayV4, gatewayV6 string + for _, config := range network.IPAM.Config { + switch util.CheckProtocol(config.Subnet) { + case apiv1.ProtocolIPv4: + if ipv4 { + cidrV4 = config.Subnet + gatewayV4 = config.Gateway + } + case apiv1.ProtocolIPv6: + if ipv6 { + cidrV6 = config.Subnet + if gatewayV6 = config.Gateway; gatewayV6 == "" { + var err error + gatewayV6, err = util.FirstIP(cidrV6) + framework.ExpectNoError(err) + } + } + } + } + + cidr := make([]string, 0, 2) + gateway := make([]string, 0, 2) + if ipv4 { + cidr = append(cidr, cidrV4) + gateway = append(gateway, gatewayV4) + } + if ipv6 { + cidr = append(cidr, cidrV6) + gateway = append(gateway, gatewayV6) + } + + excludeIPs := make([]string, 0, len(network.Containers)*2) + for _, container := range network.Containers { + if container.IPv4Address != "" && ipv4 { + excludeIPs = append(excludeIPs, strings.Split(container.IPv4Address, "/")[0]) + } + if container.IPv6Address != "" && ipv6 { + excludeIPs = append(excludeIPs, strings.Split(container.IPv6Address, "/")[0]) + } + } + + return framework.MakeSubnet(subnetName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", "", excludeIPs, nil, nil) +} + +func checkEgressAccess(f *framework.Framework, namespaceName, svrPodName, image, svrPort string, svrIPs, extIPs []string, intIPs map[string][]string, subnetName, nodeName string, snat bool) { + ginkgo.GinkgoHelper() + + podName := "pod-" + framework.RandomSuffix() + ginkgo.By("Creating client pod " + podName + " within subnet " + subnetName) + labels := map[string]string{"snat": strconv.FormatBool(snat)} + annotations := map[string]string{util.LogicalSwitchAnnotation: subnetName} + pod := framework.MakePrivilegedPod(namespaceName, podName, labels, annotations, image, []string{"sleep", "infinity"}, nil) + pod.Spec.NodeName = nodeName + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting pod " + podName) + f.PodClient().DeleteSync(podName) + }) + pod = f.PodClient().CreateSync(pod) + + if !snat { + // skip egress route check if SNAT is enabled + // traceroute does not work for pods selected by the selectors + var hops []string + if nodeName == "" { + for ips := range maps.Values(intIPs) { + hops = append(hops, ips...) + } + } else { + hops = intIPs[nodeName] + } + framework.CheckPodEgressRoutes(pod.Namespace, pod.Name, f.HasIPv4(), f.HasIPv6(), 2, hops) + } + + if !snat { + podIPv4, podIPv6 := util.SplitIpsByProtocol(util.PodIPs(*pod)) + hopsIPv4, hopsIPv6 := util.SplitIpsByProtocol(extIPs) + addEcmpRoutes(namespaceName, svrPodName, podIPv4, hopsIPv4) + addEcmpRoutes(namespaceName, svrPodName, podIPv6, hopsIPv6) + } + + expectedClientIPs := extIPs + if !snat { + expectedClientIPs = util.PodIPs(*pod) + } + for _, svrIP := range svrIPs { + protocol := strings.ToLower(util.CheckProtocol(svrIP)) + ginkgo.By("Checking connection from " + pod.Name + " to " + svrIP + " via " + protocol) + cmd := fmt.Sprintf("curl -q -s --connect-timeout 2 --max-time 2 %s/clientip", net.JoinHostPort(svrIP, svrPort)) + ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, cmd, pod.Namespace, pod.Name)) + output := e2epodoutput.RunHostCmdOrDie(pod.Namespace, pod.Name, cmd) + clientIP, _, err := net.SplitHostPort(strings.TrimSpace(output)) + framework.ExpectNoError(err) + framework.ExpectContainElement(expectedClientIPs, clientIP) + } +} + +func addEcmpRoutes(namespaceName, podName string, destinations, nextHops []string) { + ginkgo.GinkgoHelper() + + if len(destinations) == 0 || len(nextHops) == 0 { + return + } + + var args string + if len(nextHops) == 1 { + args = " via " + nextHops[0] + } else { + for _, ip := range nextHops { + args += fmt.Sprintf(" nexthop via %s dev net1 weight 1", ip) + } + } + for _, dst := range destinations { + cmd := fmt.Sprintf("ip route add %s%s", dst, args) + output, err := e2epodoutput.RunHostCmd(namespaceName, podName, cmd) + framework.ExpectNoError(err, output) + } +} + +func parseGobgpOutput(output string) (*GlobalConfig, []NeighborEntry, error) { + lines := strings.Split(strings.TrimSpace(output), "\n") + + var globalJSON, neighborJSON string + var foundGlobal bool + + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + // global config starts with { + if strings.HasPrefix(line, "{") && !foundGlobal { + globalJSON = line + foundGlobal = true + } else if strings.HasPrefix(line, "[") { + // neighbor config starts with [ + neighborJSON = line + } + } + + if globalJSON == "" || neighborJSON == "" { + return nil, nil, errors.New("failed to extract JSON parts from output") + } + + // GlobalConfig + var global GlobalConfig + if err := json.Unmarshal([]byte(globalJSON), &global); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal global config: %w", err) + } + + // NeighborEntry + var neighbors []NeighborEntry + if err := json.Unmarshal([]byte(neighborJSON), &neighbors); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal neighbor config: %w", err) + } + + return &global, neighbors, nil +} + +func checkBgpInitSetting(ber *apiv1.BgpEdgeRouter, output string) { + ginkgo.GinkgoHelper() + + global, neighbors, err := parseGobgpOutput(output) + framework.ExpectNoError(err, "parsing gobgp output") + + remoteAsn := ber.Spec.BGP.RemoteASN + + framework.ExpectEqual(global.ASN, ber.Spec.BGP.ASN, "global ASN mismatch") + if ber.Spec.BGP.RouterID != "" { + framework.ExpectEqual(global.RouterID, ber.Spec.BGP.RouterID, "global Router ID mismatch") + } + // TODO check when ber.Spec.BGP.RouterID is empty + + for _, berNeighborAddr := range ber.Spec.BGP.Neighbors { + matchFound := false + for _, neighbor := range neighbors { + if neighbor.Conf.NeighborAddress == berNeighborAddr { + matchFound = true + framework.ExpectEqual(neighbor.Conf.PeerASN, remoteAsn, "neighbor %s peer ASN %d mismatch", berNeighborAddr, remoteAsn) + break + } + } + if !matchFound { + framework.Failf("neighbor address %s not found", berNeighborAddr) + } + } +} + +func berTest(f *framework.Framework, bfd bool, provider, nadName, vpcName, internalSubnetName, externalSubnetName string, replicas int32) { + ginkgo.GinkgoHelper() + + namespaceName := f.Namespace.Name + forwardSubnetName := "forward-" + framework.RandomSuffix() + subnetClient := f.SubnetClient() + berClient := f.BgpEdgeRouterClient() + deployClient := f.DeploymentClient() + podClient := f.PodClient() + + var forwardSubnet *apiv1.Subnet + ginkgo.By("Creating subnet " + forwardSubnetName) + cidr := framework.RandomCIDR(f.ClusterIPFamily) + subnet := framework.MakeSubnet(forwardSubnetName, "", cidr, "", vpcName, "", nil, nil, nil) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting subnet " + forwardSubnetName) + subnetClient.DeleteSync(forwardSubnetName) + }) + _ = subnetClient.CreateSync(subnet) + forwardSubnet = subnet + + berName := "ber-" + framework.RandomSuffix() + ginkgo.By("Creating bgp edge router " + berName) + ber := framework.MakeBgpEdgeRouter(namespaceName, berName, vpcName, replicas, internalSubnetName, externalSubnetName, forwardSubnetName) + if rand.Int32N(2) == 0 { + ber.Spec.Prefix = fmt.Sprintf("e2e-%s-", framework.RandomSuffix()) + } + ber.Spec.BFD.Enabled = bfd + if vpcName == util.DefaultVpc { + ber.Spec.VPC = "" // test whether the ber works without specifying VPC + ber.Spec.TrafficPolicy = apiv1.TrafficPolicyLocal + } + + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting bgp edge router " + berName) + berClient.DeleteSync(berName) + }) + ber = berClient.CreateSync(ber) + + ginkgo.By("Validating bgp edge router status") + framework.ExpectTrue(ber.Status.Ready) + framework.ExpectEqual(ber.Status.Phase, apiv1.PhaseCompleted) + framework.ExpectHaveLen(ber.Status.InternalIPs, int(replicas)) + framework.ExpectHaveLen(ber.Status.ExternalIPs, int(replicas)) + + ginkgo.By("Validating bgp edge router workload") + framework.ExpectEqual(ber.Status.Workload.Name, ber.Spec.Prefix+ber.Name) + deploy := deployClient.Get(ber.Status.Workload.Name) + framework.ExpectEqual(deploy.Status.Replicas, replicas) + framework.ExpectEqual(deploy.Status.ReadyReplicas, replicas) + gvk := appsv1.SchemeGroupVersion.WithKind(reflect.TypeOf(deploy).Elem().Name()) + framework.ExpectEqual(ber.Status.Workload.APIVersion, gvk.GroupVersion().String()) + framework.ExpectEqual(ber.Status.Workload.Kind, gvk.Kind) + framework.ExpectHaveLen(ber.Status.Workload.Nodes, int(replicas)) + workloadPods, err := deployClient.GetPods(deploy) + framework.ExpectNoError(err) + framework.ExpectHaveLen(workloadPods.Items, int(replicas)) + podNodes := make([]string, 0, len(workloadPods.Items)) + intIPs := make(map[string][]string, len(workloadPods.Items)) + for _, pod := range workloadPods.Items { + framework.ExpectNotContainElement(podNodes, pod.Spec.NodeName) + podNodes = append(podNodes, pod.Spec.NodeName) + intIPs[pod.Spec.NodeName] = util.PodIPs(pod) + // exec and list + ginkgo.By("Checking bgp setting " + pod.Name) + cmd := "gobgp global -j && gobgp neighbor -j" + ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, cmd, pod.Namespace, pod.Name)) + output := e2epodoutput.RunHostCmdOrDie(pod.Namespace, pod.Name, cmd) + checkBgpInitSetting(ber, output) + } + framework.ExpectConsistOf(ber.Status.Workload.Nodes, podNodes) + + // TODO + // Add route advertisement + + // Add bgp policy + + svrPodName := "svr-" + framework.RandomSuffix() + ginkgo.By("Creating netexec server pod " + svrPodName) + routes := util.NewPodRoutes() + dstV4, dstV6 := util.SplitStringIP(forwardSubnet.Spec.CIDRBlock) + gwV4, gwV6 := util.SplitStringIP(ber.Status.ExternalIPs[0]) + routes.Add(provider, dstV4, gwV4) + routes.Add(provider, dstV6, gwV6) + annotations, err := routes.ToAnnotations() + framework.ExpectNoError(err) + attachmentNetworkName := fmt.Sprintf("%s/%s", namespaceName, nadName) + annotations[nadv1.NetworkAttachmentAnnot] = attachmentNetworkName + port := strconv.Itoa(8000 + rand.IntN(1000)) + args := []string{"netexec", "--http-port", port} + svrPod := framework.MakePrivilegedPod(namespaceName, svrPodName, nil, annotations, framework.AgnhostImage, nil, args) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting pod " + svrPodName) + podClient.DeleteSync(svrPodName) + }) + svrPod = podClient.CreateSync(svrPod) + svrIPs, err := util.PodAttachmentIPs(svrPod, attachmentNetworkName) + framework.ExpectNoError(err) + + image := workloadPods.Items[0].Spec.Containers[0].Image + extIPs := make([]string, 0, len(ber.Status.ExternalIPs)*2) + for _, ips := range ber.Status.ExternalIPs { + extIPs = append(extIPs, strings.Split(ips, ",")...) + } + + var nodeName string + if ber.Spec.TrafficPolicy == apiv1.TrafficPolicyLocal { + nodeName = ber.Status.Workload.Nodes[0] + } + checkEgressAccess(f, namespaceName, svrPodName, image, port, svrIPs, extIPs, intIPs, forwardSubnetName, nodeName, false) +} diff --git a/test/e2e/framework/bgp-edge-router.go b/test/e2e/framework/bgp-edge-router.go new file mode 100644 index 00000000000..026d335cc49 --- /dev/null +++ b/test/e2e/framework/bgp-edge-router.go @@ -0,0 +1,213 @@ +package framework + +import ( + "context" + "errors" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + v1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// BgpEdgeRouterClient is a struct for bgp edge router client. +type BgpEdgeRouterClient struct { + f *Framework + namespace string + v1.BgpEdgeRouterInterface +} + +func NewBgpEdgeRouterClient(cs clientset.Interface, namespapce string) *BgpEdgeRouterClient { + return &BgpEdgeRouterClient{ + namespace: namespapce, + BgpEdgeRouterInterface: cs.KubeovnV1().BgpEdgeRouters(namespapce), + } +} + +func (f *Framework) BgpEdgeRouterClient() *BgpEdgeRouterClient { + return &BgpEdgeRouterClient{ + f: f, + namespace: f.Namespace.Name, + BgpEdgeRouterInterface: f.KubeOVNClientSet.KubeovnV1().BgpEdgeRouters(f.Namespace.Name), + } +} + +func (f *Framework) BgpEdgeRouterClientNS(namespapce string) *BgpEdgeRouterClient { + return &BgpEdgeRouterClient{ + f: f, + namespace: namespapce, + BgpEdgeRouterInterface: f.KubeOVNClientSet.KubeovnV1().BgpEdgeRouters(namespapce), + } +} + +func (c *BgpEdgeRouterClient) Get(name string) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + router, err := c.BgpEdgeRouterInterface.Get(context.TODO(), name, metav1.GetOptions{}) + ExpectNoError(err) + return router +} + +// Create creates a new bgp-edge-router according to the framework specifications +func (c *BgpEdgeRouterClient) Create(router *apiv1.BgpEdgeRouter) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + g, err := c.BgpEdgeRouterInterface.Create(context.TODO(), router, metav1.CreateOptions{}) + ExpectNoError(err, "Error creating bgp-edge-router") + return g.DeepCopy() +} + +// CreateSync creates a new bgp-edge-router according to the framework specifications, and waits for it to be ready. +func (c *BgpEdgeRouterClient) CreateSync(router *apiv1.BgpEdgeRouter) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + _ = c.Create(router) + return c.WaitUntil(router.Name, func(g *apiv1.BgpEdgeRouter) (bool, error) { + return g.Ready(), nil + }, "Ready", 2*time.Second, timeout) +} + +// Patch patches the router +func (c *BgpEdgeRouterClient) Patch(original, modified *apiv1.BgpEdgeRouter) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + + patch, err := util.GenerateMergePatchPayload(original, modified) + ExpectNoError(err) + + var patchedRouter *apiv1.BgpEdgeRouter + err = wait.PollUntilContextTimeout(context.Background(), 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + g, err := c.BgpEdgeRouterInterface.Patch(ctx, original.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "") + if err != nil { + return handleWaitingAPIError(err, false, "patch bgp-edge-router %s/%s", original.Namespace, original.Name) + } + patchedRouter = g + return true, nil + }) + if err == nil { + return patchedRouter.DeepCopy() + } + + if errors.Is(err, context.DeadlineExceeded) { + Failf("timed out while retrying to patch bgp-edge-router %s/%s", original.Namespace, original.Name) + } + Failf("error occurred while retrying to patch bgp-edge-router %s/%s: %v", original.Namespace, original.Name, err) + + return nil +} + +// PatchSync patches the router and waits the router to meet the condition +func (c *BgpEdgeRouterClient) PatchSync(original, modified *apiv1.BgpEdgeRouter) *apiv1.BgpEdgeRouter { + ginkgo.GinkgoHelper() + _ = c.Patch(original, modified) + return c.WaitUntil(original.Name, func(g *apiv1.BgpEdgeRouter) (bool, error) { + return g.Ready(), nil + }, "Ready", 2*time.Second, timeout) +} + +// Delete deletes a bgp-edge-router if the bgp-edge-router exists +func (c *BgpEdgeRouterClient) Delete(name string) { + ginkgo.GinkgoHelper() + err := c.BgpEdgeRouterInterface.Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + Failf("Failed to delete bgp-edge-router %s/%s: %v", c.namespace, name, err) + } +} + +// DeleteSync deletes the bgp-edge-router and waits for the bgp-edge-router to disappear for `timeout`. +// If the bgp-edge-router doesn't disappear before the timeout, it will fail the test. +func (c *BgpEdgeRouterClient) DeleteSync(name string) { + ginkgo.GinkgoHelper() + c.Delete(name) + gomega.Expect(c.WaitToDisappear(name, 2*time.Second, timeout)).To(gomega.Succeed(), "wait for bgp-edge-router %s/%s to disappear", c.namespace, name) +} + +// WaitUntil waits the given timeout duration for the specified condition to be met. +func (c *BgpEdgeRouterClient) WaitUntil(name string, cond func(g *apiv1.BgpEdgeRouter) (bool, error), condDesc string, interval, timeout time.Duration) *apiv1.BgpEdgeRouter { + var router *apiv1.BgpEdgeRouter + err := wait.PollUntilContextTimeout(context.Background(), interval, timeout, false, func(_ context.Context) (bool, error) { + Logf("Waiting for bgp-edge-router %s/%s to meet condition %q", c.namespace, name, condDesc) + router = c.Get(name).DeepCopy() + met, err := cond(router) + if err != nil { + return false, fmt.Errorf("failed to check condition for bgp-edge-router %s/%s: %w", c.namespace, name, err) + } + if met { + Logf("bgp-edge-router %s/%s met condition %q", c.namespace, name, condDesc) + } else { + Logf("bgp-edge-router %s/%s not met condition %q", c.namespace, name, condDesc) + } + return met, nil + }) + if err == nil { + return router + } + + if errors.Is(err, context.DeadlineExceeded) { + Failf("timed out while waiting for bgp-edge-router %s/%s to meet condition %q", c.namespace, name, condDesc) + } + Failf("error occurred while waiting for bgp-edge-router %s/%s to meet condition %q: %v", c.namespace, name, condDesc, err) + + return nil +} + +// WaitToDisappear waits the given timeout duration for the specified bgp-edge-router to disappear. +func (c *BgpEdgeRouterClient) WaitToDisappear(name string, _, timeout time.Duration) error { + err := framework.Gomega().Eventually(context.Background(), framework.HandleRetry(func(ctx context.Context) (*apiv1.BgpEdgeRouter, error) { + svc, err := c.BgpEdgeRouterInterface.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, nil + } + return svc, err + })).WithTimeout(timeout).Should(gomega.BeNil()) + if err != nil { + return fmt.Errorf("expected vpc-egress-gateway %s/%s to not be found: %w", c.namespace, name, err) + } + return nil +} + +func MakeBgpEdgeRouter(namespace, name, vpc string, replicas int32, internalSubnet, externalSubnet, forwardSubnet string) *apiv1.BgpEdgeRouter { + return &apiv1.BgpEdgeRouter{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: apiv1.BgpEdgeRouterSpec{ + Replicas: replicas, + VPC: vpc, + InternalSubnet: internalSubnet, + ExternalSubnet: externalSubnet, + BFD: apiv1.BgpEdgeRouterBFDConfig{ + Enabled: true, + MinRX: 300, + MinTX: 300, + Multiplier: 3, + }, + Policies: []apiv1.BgpEdgeRouterPolicy{ + { + SNAT: false, + Subnets: []string{ + forwardSubnet, + }, + }, + }, + BGP: apiv1.BgpEdgeRouterBGPConfig{ + Enabled: true, + ASN: 65000, + EdgeRouterMode: true, + RemoteASN: 65100, + Neighbors: []string{ + "192.168.1.1", + }, + EnableGracefulRestart: true, + }, + }, + } +} diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index c829cf194b0..e6f534faee9 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -102,6 +102,37 @@ func RandomCIDR(family string) string { } } +func CIDRGatewayIP(cidr string) string { + ginkgo.GinkgoHelper() + + if cidr == "" { + Failf("cidr is empty") + return "" + } + + _, ipNet, err := net.ParseCIDR(cidr) + ExpectNoError(err) + + ip := ipNet.IP + // ones, _ := ipNet.Mask.Size() + switch { + case ip.To4() != nil: + // IPv4 + ip = ip.To4() + ip[3] = 1 + + return ip.String() + case ip.To16() == nil: + // IPv4-mapped IPv6 + ip = ip.To16() + ip[15] = 1 + return ip.String() + default: + Failf("invalid cidr: %s", cidr) + return "" + } +} + func sortIPs(ips []string) { ginkgo.GinkgoHelper()